@sigrun/sigrun 0.1.5 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +136 -0
- package/README.md +352 -36
- package/THIRD_PARTY_NOTICES.md +23 -0
- package/dist/blocks/schema.js +9 -2
- package/dist/blocks/validate.js +57 -8
- package/dist/cli.js +26 -2
- package/dist/config.js +67 -1
- package/dist/extension.d.ts +189 -15
- package/dist/harness/agent-env.js +62 -0
- package/dist/harness/ambient.js +39 -185
- package/dist/harness/blocks/entries.js +47 -0
- package/dist/harness/blocks/state.js +58 -0
- package/dist/harness/blocks/tools.js +274 -0
- package/dist/harness/boot.js +253 -20
- package/dist/harness/builtins.js +15 -11
- package/dist/harness/event-bus.js +105 -10
- package/dist/harness/identity/accounts.js +186 -0
- package/dist/harness/identity/groups.js +14 -0
- package/dist/harness/identity/index.js +13 -6
- package/dist/harness/identity/json-store.js +15 -4
- package/dist/harness/identity/setup-code-check.js +9 -0
- package/dist/harness/identity/tokens.js +28 -5
- package/dist/harness/identity/users.js +111 -12
- package/dist/harness/interactions.js +292 -136
- package/dist/harness/loader.js +271 -17
- package/dist/harness/packages.js +1 -1
- package/dist/harness/reserved-sections.js +21 -0
- package/dist/harness/seam-host.js +80 -6
- package/dist/harness/settings.js +63 -2
- package/dist/harness/stores.js +241 -4
- package/dist/harness/user-services.js +391 -0
- package/dist/harness/user-tools.js +187 -0
- package/dist/harness/workspace.js +61 -55
- package/dist/jobs/fire.js +10 -16
- package/dist/sandbox/availability.js +162 -0
- package/dist/sandbox/command-observer.js +59 -0
- package/dist/sandbox/grants.js +159 -0
- package/dist/sandbox/helper.js +29 -0
- package/dist/sandbox/isolation.js +29 -0
- package/dist/sandbox/launch.js +20 -0
- package/dist/sandbox/os-account.js +49 -0
- package/dist/sandbox/policy.js +253 -0
- package/dist/sandbox/pool.js +548 -0
- package/dist/sandbox/protocol.js +167 -0
- package/dist/sandbox/sandboxed-env.js +245 -0
- package/dist/sandbox/unavailable-env.js +49 -0
- package/dist/sandbox/worker.js +281 -0
- package/dist/serve.js +82 -7
- package/dist/server/auth/access-principal.js +16 -0
- package/dist/server/auth/pipeline.js +47 -33
- package/dist/server/blocks/answers.js +94 -0
- package/dist/server/blocks/chat-surfaces.js +393 -0
- package/dist/server/blocks/notes.js +216 -0
- package/dist/server/blocks/summaries.js +328 -0
- package/dist/server/commands.js +5 -0
- package/dist/server/content-parts-lazy.js +13 -0
- package/dist/server/conversation-handler.js +2 -5
- package/dist/server/conversation-ops.js +138 -37
- package/dist/server/helpers.js +18 -6
- package/dist/server/host-bridge.js +40 -20
- package/dist/server/images/cache.js +205 -0
- package/dist/server/images/convert.js +13 -0
- package/dist/server/images/fetch.js +302 -0
- package/dist/server/images/ranges.js +42 -0
- package/dist/server/images/rewrite.js +94 -0
- package/dist/server/images/sign.js +84 -0
- package/dist/server/images/sniff.js +62 -0
- package/dist/server/index.js +144 -4
- package/dist/server/login/flows.js +126 -0
- package/dist/server/login/http.js +31 -0
- package/dist/server/login/resolve.js +133 -0
- package/dist/server/login/setup-claim.js +55 -0
- package/dist/server/login/usernames.js +15 -0
- package/dist/server/ownership.js +13 -0
- package/dist/server/routes/admin.js +58 -0
- package/dist/server/routes/auth.js +88 -2
- package/dist/server/routes/conversations.js +3 -6
- package/dist/server/routes/events.js +28 -3
- package/dist/server/routes/extensions.js +37 -5
- package/dist/server/routes/images.js +267 -0
- package/dist/server/routes/interactions.js +57 -68
- package/dist/server/routes/jobs.js +2 -8
- package/dist/server/routes/login.js +223 -0
- package/dist/server/routes/me.js +90 -0
- package/dist/server/routes/meta.js +15 -0
- package/dist/server/routes/onboarding.js +94 -84
- package/dist/server/routes/settings-sections.js +190 -0
- package/dist/server/routes/settings.js +16 -4
- package/dist/server/routes/submissions.js +1 -1
- package/dist/server/routes/surfaces.js +175 -23
- package/dist/server/routes/users.js +116 -20
- package/dist/server/schema-fixture.js +213 -0
- package/dist/server/session-meta.js +3 -0
- package/dist/server/shutdown.js +11 -4
- package/dist/server/static.js +18 -0
- package/dist/server/surfaces/dedup.js +9 -2
- package/dist/server/surfaces/service.js +181 -13
- package/dist/server/surfaces/settings-surfaces.js +597 -0
- package/dist/server/turns/submit-turn.js +26 -0
- package/dist/server/views.js +16 -0
- package/dist/settings/core-sections.js +532 -0
- package/dist/settings/pending.js +181 -0
- package/dist/settings/registry.js +453 -0
- package/dist/skills/index.js +6 -0
- package/node_modules/@sigrun/shared/CHANGELOG.md +15 -0
- package/node_modules/@sigrun/shared/dist/access.d.ts +4 -2
- package/node_modules/@sigrun/shared/dist/access.js +11 -4
- package/node_modules/@sigrun/shared/dist/ambient.d.ts +17 -74
- package/node_modules/@sigrun/shared/dist/ambient.js +8 -6
- package/node_modules/@sigrun/shared/dist/blocks/text.js +16 -2
- package/node_modules/@sigrun/shared/dist/blocks/transport.d.ts +4 -0
- package/node_modules/@sigrun/shared/dist/blocks/types.d.ts +21 -4
- package/node_modules/@sigrun/shared/dist/interactions.d.ts +22 -48
- package/node_modules/@sigrun/shared/dist/interactions.js +9 -7
- package/node_modules/@sigrun/shared/dist/settings.d.ts +27 -2
- package/node_modules/@sigrun/shared/dist/themes.d.ts +2 -3
- package/node_modules/@sigrun/shared/dist/themes.js +2 -3
- package/node_modules/@sigrun/shared/dist/turns.d.ts +14 -0
- package/node_modules/@sigrun/shared/dist/turns.js +53 -3
- package/node_modules/@sigrun/shared/dist/views.d.ts +175 -12
- package/node_modules/@sigrun/shared/dist/views.js +16 -0
- package/node_modules/@sigrun/shared/package.json +1 -1
- package/package.json +13 -8
- package/webui/dist/assets/index-CxB969qp.js +55 -0
- package/webui/dist/assets/index-yqkL6BSS.css +1 -0
- package/webui/dist/index.html +2 -2
- package/webui/dist/assets/index-B4-GYqEb.js +0 -50
- package/webui/dist/assets/index-BiSMljer.css +0 -1
package/dist/server/helpers.js
CHANGED
|
@@ -8,6 +8,7 @@ import { existsSync, readFileSync } from "node:fs";
|
|
|
8
8
|
import { homedir } from "node:os";
|
|
9
9
|
import { isAbsolute, resolve } from "node:path";
|
|
10
10
|
import { effectiveServerSection, } from "../config.js";
|
|
11
|
+
import { pendingRestart } from "../settings/pending.js";
|
|
11
12
|
import { DEFAULT_AUTH_KEY } from "../harness/auth.js";
|
|
12
13
|
import { OPENAI_API_KEY_ENV } from "../harness/providers/index.js";
|
|
13
14
|
import { SessionMetaDoc } from "./session-meta.js";
|
|
@@ -198,12 +199,15 @@ export const ID_PARAMS_SCHEMA = {
|
|
|
198
199
|
/** The settings response over the live store + registry snapshot. */
|
|
199
200
|
/**
|
|
200
201
|
* The `/v1/settings` payload (GET and the post-PATCH readback): the
|
|
201
|
-
* file's effective contents, the registry-computed `toolRoster`,
|
|
202
|
-
*
|
|
203
|
-
*
|
|
204
|
-
*
|
|
202
|
+
* file's effective contents, the registry-computed `toolRoster`, the
|
|
203
|
+
* `effective` block (spec §6.6), and the `pendingRestart` diff
|
|
204
|
+
* (UI-blocks U4 §13.2) — every server key's resolved value and source,
|
|
205
|
+
* computed PER REQUEST over the fresh file read (a PATCH lands as a
|
|
206
|
+
* `file` source on the next GET) and the boot env/config. A PATCH
|
|
207
|
+
* response additionally carries the write's own `applied`/`pending`
|
|
208
|
+
* classification (§13.1) via the optional `write` argument.
|
|
205
209
|
*/
|
|
206
|
-
export function settingsResponse(opened, config, env) {
|
|
210
|
+
export function settingsResponse(opened, config, env, write) {
|
|
207
211
|
const names = opened.registry
|
|
208
212
|
.snapshot()
|
|
209
213
|
.tools()
|
|
@@ -215,6 +219,8 @@ export function settingsResponse(opened, config, env) {
|
|
|
215
219
|
...file,
|
|
216
220
|
toolRoster: [...new Set(names)],
|
|
217
221
|
effective: effectiveServerSection(env, file.server, config),
|
|
222
|
+
pendingRestart: pendingRestart(file, opened.settingsBootSnapshot),
|
|
223
|
+
...(write === undefined ? {} : { applied: write.applied, pending: write.pending }),
|
|
218
224
|
};
|
|
219
225
|
}
|
|
220
226
|
/** Re-brand a route id as a ConversationId (the branded Id type is a number). */
|
|
@@ -241,12 +247,18 @@ export async function readView(conversation) {
|
|
|
241
247
|
* copied state (`fork: "current"`), so the caller's identity OVERWRITES
|
|
242
248
|
* the source's.
|
|
243
249
|
*/
|
|
244
|
-
export async function stampSessionOwner(conversation, owner) {
|
|
250
|
+
export async function stampSessionOwner(conversation, owner, opened) {
|
|
245
251
|
if (owner === undefined)
|
|
246
252
|
return;
|
|
253
|
+
await stampOwnerInCommit(conversation, owner, opened);
|
|
254
|
+
}
|
|
255
|
+
/** Stamp/replace an owner and rewrite per-user selection in ONE commit. */
|
|
256
|
+
export async function stampOwnerInCommit(conversation, owner, opened) {
|
|
247
257
|
await conversation.commit(async (tx) => {
|
|
248
258
|
const draft = await tx.doc(SessionMetaDoc, conversation.id);
|
|
259
|
+
const previous = draft.owner;
|
|
249
260
|
draft.owner = owner;
|
|
261
|
+
await opened.rewriteUserSelection(tx, conversation.id, previous, owner);
|
|
250
262
|
}, BACKGROUND_CONTEXT);
|
|
251
263
|
}
|
|
252
264
|
/**
|
|
@@ -16,25 +16,22 @@
|
|
|
16
16
|
* superuser's HTTP list does: every row visible, `access: "write"`.
|
|
17
17
|
*/
|
|
18
18
|
import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context";
|
|
19
|
-
import { accessOf as accessOfShared
|
|
20
|
-
import {
|
|
19
|
+
import { accessOf as accessOfShared } from "@sigrun/shared/access";
|
|
20
|
+
import { effectiveAdmin } from "../harness/identity/groups.js";
|
|
21
|
+
import { userWire } from "./routes/users.js";
|
|
22
|
+
import { accessPrincipalOf } from "./auth/access-principal.js";
|
|
21
23
|
import { abortOp, compactOp, configureOp, ConversationOpError, createConversationOp, forkOp, listConversationsOp, patchMetaOp, readFileOp, readMetaOp, submitOp, uploadOp, } from "./conversation-ops.js";
|
|
22
24
|
import { listModels } from "./helpers.js";
|
|
23
25
|
import { SessionMetaDoc } from "./session-meta.js";
|
|
24
|
-
import { serializeConversationView } from "./views.js";
|
|
26
|
+
import { accountLinkWire, serializeConversationView } from "./views.js";
|
|
25
27
|
/**
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
* flag OR the persisted group grant, plus the raw `groupAdmin` as a
|
|
29
|
-
* definite boolean. The `/v1/users` wire keeps `admin` = the stored
|
|
30
|
-
* flag (the users-admin UI toggles that one); the two surfaces are
|
|
31
|
-
* deliberately different.
|
|
28
|
+
* Host-side callers see effective admin derived from stored groups;
|
|
29
|
+
* `groupAdmin` remains the derived compatibility field on the wire.
|
|
32
30
|
*/
|
|
33
|
-
function effectiveAdminOf(record) {
|
|
31
|
+
function effectiveAdminOf(record, adminGroups) {
|
|
34
32
|
return {
|
|
35
|
-
...record,
|
|
36
|
-
admin: record
|
|
37
|
-
groupAdmin: record.groupAdmin === true,
|
|
33
|
+
...userWire(record, adminGroups),
|
|
34
|
+
admin: effectiveAdmin(record, adminGroups),
|
|
38
35
|
};
|
|
39
36
|
}
|
|
40
37
|
/** Build the host bridge over one server context. */
|
|
@@ -136,6 +133,7 @@ export function createHostBridge(ctx) {
|
|
|
136
133
|
await configureOp(ctx, conversation, {
|
|
137
134
|
...(opts.model === undefined ? {} : { model: opts.model }),
|
|
138
135
|
...(opts.thinking === undefined ? {} : { thinking: opts.thinking }),
|
|
136
|
+
...(opts.extensions === undefined ? {} : { extensions: opts.extensions }),
|
|
139
137
|
});
|
|
140
138
|
},
|
|
141
139
|
async compact(id) {
|
|
@@ -176,11 +174,36 @@ export function createHostBridge(ctx) {
|
|
|
176
174
|
// admin-gated /v1/users routes) — with EFFECTIVE admin (gateway
|
|
177
175
|
// C8): a persisted group grant counts for host-side checks.
|
|
178
176
|
async list() {
|
|
179
|
-
return opened.identity.users.list().map(effectiveAdminOf);
|
|
177
|
+
return opened.identity.users.list().map((record) => effectiveAdminOf(record, ctx.config.auth.adminGroups));
|
|
180
178
|
},
|
|
181
179
|
async get(user) {
|
|
182
180
|
const record = opened.identity.users.get(user);
|
|
183
|
-
return record === undefined ? undefined : effectiveAdminOf(record);
|
|
181
|
+
return record === undefined ? undefined : effectiveAdminOf(record, ctx.config.auth.adminGroups);
|
|
182
|
+
},
|
|
183
|
+
},
|
|
184
|
+
accounts: {
|
|
185
|
+
// The one link table (multi-user §11.3): core code over core's
|
|
186
|
+
// store — the single write path, with the store's own validation
|
|
187
|
+
// and conflict refusal. `account_link_conflict:` errors cross the
|
|
188
|
+
// bridge as plain Errors (extensions match the message prefix).
|
|
189
|
+
async lookup(provider, subject) {
|
|
190
|
+
const link = opened.identity.accounts.lookup(provider, subject);
|
|
191
|
+
return link === undefined ? undefined : accountLinkWire(link);
|
|
192
|
+
},
|
|
193
|
+
async link(provider, subject, user, via) {
|
|
194
|
+
const link = await opened.identity.accounts.link({
|
|
195
|
+
provider,
|
|
196
|
+
subject,
|
|
197
|
+
user,
|
|
198
|
+
via,
|
|
199
|
+
});
|
|
200
|
+
return accountLinkWire(link);
|
|
201
|
+
},
|
|
202
|
+
async unlink(provider, subject) {
|
|
203
|
+
return await opened.identity.accounts.unlink(provider, subject);
|
|
204
|
+
},
|
|
205
|
+
async listForUser(user) {
|
|
206
|
+
return opened.identity.accounts.listForUser(user).map(accountLinkWire);
|
|
184
207
|
},
|
|
185
208
|
},
|
|
186
209
|
access: {
|
|
@@ -190,10 +213,7 @@ export function createHostBridge(ctx) {
|
|
|
190
213
|
// directory admin is not a superuser for accessOf); the bare
|
|
191
214
|
// `{user}` shape is the shared rule's own principal.
|
|
192
215
|
const resolved = "admin" in principal
|
|
193
|
-
?
|
|
194
|
-
user: principal.user,
|
|
195
|
-
superuser: isSuperuser(principal),
|
|
196
|
-
}
|
|
216
|
+
? accessPrincipalOf(principal, opened.identity.users)
|
|
197
217
|
: { user: principal.user };
|
|
198
218
|
return accessOfShared(resolved, resource);
|
|
199
219
|
},
|
|
@@ -241,7 +261,7 @@ export function createHostBridge(ctx) {
|
|
|
241
261
|
user: record.user,
|
|
242
262
|
...(record.name === undefined ? {} : { name: record.name }),
|
|
243
263
|
...(record.email === undefined ? {} : { email: record.email }),
|
|
244
|
-
admin:
|
|
264
|
+
admin: effectiveAdmin(record, ctx.config.auth.adminGroups),
|
|
245
265
|
service: false,
|
|
246
266
|
method: "host",
|
|
247
267
|
};
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The image proxy's on-disk LRU cache (UI-blocks U7 Task 3, spec §11.2):
|
|
3
|
+
* entries live under `<cacheDir>/images/` (pod-local and outside backups,
|
|
4
|
+
* like the other cache roots), keyed by `sha256(u)` plus the `as`
|
|
5
|
+
* variant — NEVER the signed URL, whose signature changes on every
|
|
6
|
+
* serve. Capacity is 256 MiB by default; eviction is least-recently-hit
|
|
7
|
+
* first. Failures are never cached (callers `put` successful fetches
|
|
8
|
+
* only).
|
|
9
|
+
*
|
|
10
|
+
* Layout: each entry is one file `<sha256(u)>.<variant>` beside ONE
|
|
11
|
+
* `index.json` (the metadata table: type, size, added-at, last-hit,
|
|
12
|
+
* pin). The index is written atomically on every mutation, so a re-open
|
|
13
|
+
* (the same cache dir, a new process) serves the prior entries.
|
|
14
|
+
*
|
|
15
|
+
* Eviction never deletes a file currently being STREAMED or WRITTEN
|
|
16
|
+
* (Review Focus 5): a `get` hit pins its entry until the returned
|
|
17
|
+
* stream closes (or the caller's response finishes), and a `put` holds
|
|
18
|
+
* its pin for the write's duration; the eviction pass skips pinned
|
|
19
|
+
* entries and takes the next-oldest instead.
|
|
20
|
+
*/
|
|
21
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
22
|
+
import { createReadStream } from "node:fs";
|
|
23
|
+
import { chmod, mkdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
|
|
24
|
+
import { join } from "node:path";
|
|
25
|
+
/** The images subdirectory under the cache root. */
|
|
26
|
+
export const IMAGES_CACHE_DIR = "images";
|
|
27
|
+
/** The default capacity: 256 MiB (spec §11.2). */
|
|
28
|
+
export const CACHE_CAPACITY_BYTES = 256 * 1024 * 1024;
|
|
29
|
+
/** The sha256-of-url + variant entry key (never the signed URL). */
|
|
30
|
+
function entryKey(u, variant) {
|
|
31
|
+
return `${createHash("sha256").update(u, "utf8").digest("hex")}.${variant}`;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* The `<cacheDir>/images/` LRU cache. Construct with the CACHE ROOT (the
|
|
35
|
+
* `images/` child is created lazily on the first put / at open when the
|
|
36
|
+
* index exists). `capacityBytes` is test-overridable; the default is the
|
|
37
|
+
* spec's 256 MiB.
|
|
38
|
+
*/
|
|
39
|
+
export class ImageCache {
|
|
40
|
+
dir;
|
|
41
|
+
indexPath;
|
|
42
|
+
capacityBytes;
|
|
43
|
+
entries = new Map();
|
|
44
|
+
opened = false;
|
|
45
|
+
closed = false;
|
|
46
|
+
constructor(cacheRoot, options) {
|
|
47
|
+
this.dir = join(cacheRoot, IMAGES_CACHE_DIR);
|
|
48
|
+
this.indexPath = join(this.dir, "index.json");
|
|
49
|
+
this.capacityBytes = options?.capacityBytes ?? CACHE_CAPACITY_BYTES;
|
|
50
|
+
}
|
|
51
|
+
/** Open: load the index (creating the dir lazily). Idempotent. */
|
|
52
|
+
async open() {
|
|
53
|
+
if (this.opened)
|
|
54
|
+
return;
|
|
55
|
+
this.opened = true;
|
|
56
|
+
let raw;
|
|
57
|
+
try {
|
|
58
|
+
raw = await readFile(this.indexPath, "utf8");
|
|
59
|
+
}
|
|
60
|
+
catch (cause) {
|
|
61
|
+
if (cause.code === "ENOENT")
|
|
62
|
+
return;
|
|
63
|
+
throw cause;
|
|
64
|
+
}
|
|
65
|
+
// A corrupt index is rebuilt EMPTY (the files on disk are then
|
|
66
|
+
// orphaned until their names recur; a fresh fetch rewrites them).
|
|
67
|
+
// The cache is a cache: losing it is never an error worth
|
|
68
|
+
// refusing to boot over.
|
|
69
|
+
let parsed;
|
|
70
|
+
try {
|
|
71
|
+
parsed = JSON.parse(raw);
|
|
72
|
+
}
|
|
73
|
+
catch {
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
for (const [key, entry] of Object.entries(parsed.entries)) {
|
|
77
|
+
this.entries.set(key, { ...entry, pins: 0 });
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
/** One hit, or `undefined` on a miss. `now` stamps the LRU order. */
|
|
81
|
+
async get(u, variant, now) {
|
|
82
|
+
await this.open();
|
|
83
|
+
const key = entryKey(u, variant);
|
|
84
|
+
const entry = this.entries.get(key);
|
|
85
|
+
if (entry === undefined)
|
|
86
|
+
return undefined;
|
|
87
|
+
const file = join(this.dir, key);
|
|
88
|
+
try {
|
|
89
|
+
await stat(file);
|
|
90
|
+
}
|
|
91
|
+
catch {
|
|
92
|
+
// The index names a file that isn't there (a hand-cleaned dir):
|
|
93
|
+
// drop the row and miss.
|
|
94
|
+
this.entries.delete(key);
|
|
95
|
+
await this.persist();
|
|
96
|
+
return undefined;
|
|
97
|
+
}
|
|
98
|
+
entry.lastHit = now ?? Date.now();
|
|
99
|
+
entry.pins += 1;
|
|
100
|
+
await this.persist();
|
|
101
|
+
const stream = createReadStream(file);
|
|
102
|
+
const unpin = () => {
|
|
103
|
+
entry.pins -= 1;
|
|
104
|
+
};
|
|
105
|
+
stream.once("close", unpin);
|
|
106
|
+
stream.once("error", unpin);
|
|
107
|
+
return { type: entry.type, stream };
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Store one fetched image. The write is staged and renamed (atomic:
|
|
111
|
+
* a concurrent `get` never reads a half-written file) and the entry
|
|
112
|
+
* is pinned for the write's duration — the eviction pass a same-key
|
|
113
|
+
* overwrite triggers can never remove the file mid-rename.
|
|
114
|
+
*/
|
|
115
|
+
async put(u, variant, type, body, now) {
|
|
116
|
+
await this.open();
|
|
117
|
+
if (this.closed)
|
|
118
|
+
return;
|
|
119
|
+
const key = entryKey(u, variant);
|
|
120
|
+
const at = now ?? Date.now();
|
|
121
|
+
const entry = {
|
|
122
|
+
type,
|
|
123
|
+
size: body.byteLength,
|
|
124
|
+
addedAt: at,
|
|
125
|
+
lastHit: at,
|
|
126
|
+
pins: 1,
|
|
127
|
+
};
|
|
128
|
+
this.entries.set(key, entry);
|
|
129
|
+
await mkdir(this.dir, { recursive: true });
|
|
130
|
+
const file = join(this.dir, key);
|
|
131
|
+
const staged = `${file}.${randomUUID()}.tmp`;
|
|
132
|
+
try {
|
|
133
|
+
await writeFile(staged, body, { mode: 0o644 });
|
|
134
|
+
await rename(staged, file);
|
|
135
|
+
}
|
|
136
|
+
catch (cause) {
|
|
137
|
+
await rm(staged, { force: true }).catch(() => { });
|
|
138
|
+
this.entries.delete(key);
|
|
139
|
+
throw cause;
|
|
140
|
+
}
|
|
141
|
+
finally {
|
|
142
|
+
entry.pins -= 1;
|
|
143
|
+
}
|
|
144
|
+
await this.evict();
|
|
145
|
+
await this.persist();
|
|
146
|
+
}
|
|
147
|
+
/** Evict least-recently-hit entries until total size fits capacity. */
|
|
148
|
+
async evict() {
|
|
149
|
+
let total = 0;
|
|
150
|
+
for (const entry of this.entries.values())
|
|
151
|
+
total += entry.size;
|
|
152
|
+
if (total <= this.capacityBytes)
|
|
153
|
+
return;
|
|
154
|
+
const oldestFirst = [...this.entries.entries()].sort((a, b) => a[1].lastHit - b[1].lastHit || a[1].addedAt - b[1].addedAt);
|
|
155
|
+
for (const [key, entry] of oldestFirst) {
|
|
156
|
+
if (total <= this.capacityBytes)
|
|
157
|
+
break;
|
|
158
|
+
// Never delete a file a response is streaming or a put is
|
|
159
|
+
// writing (Review Focus 5): skip it and take the next-oldest.
|
|
160
|
+
if (entry.pins > 0)
|
|
161
|
+
continue;
|
|
162
|
+
this.entries.delete(key);
|
|
163
|
+
total -= entry.size;
|
|
164
|
+
await rm(join(this.dir, key), { force: true }).catch(() => { });
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
/** Write the index (staged + renamed; the stores' discipline). */
|
|
168
|
+
async persist() {
|
|
169
|
+
if (this.closed)
|
|
170
|
+
return;
|
|
171
|
+
if (this.entries.size === 0) {
|
|
172
|
+
// An empty table is written too — a re-open then starts empty
|
|
173
|
+
// instead of resurrecting rows whose files are gone.
|
|
174
|
+
}
|
|
175
|
+
await mkdir(this.dir, { recursive: true });
|
|
176
|
+
const file = {
|
|
177
|
+
entries: Object.fromEntries([...this.entries.entries()].map(([key, entry]) => [
|
|
178
|
+
key,
|
|
179
|
+
{
|
|
180
|
+
type: entry.type,
|
|
181
|
+
size: entry.size,
|
|
182
|
+
addedAt: entry.addedAt,
|
|
183
|
+
lastHit: entry.lastHit,
|
|
184
|
+
},
|
|
185
|
+
])),
|
|
186
|
+
};
|
|
187
|
+
const staged = `${this.indexPath}.${randomUUID()}.tmp`;
|
|
188
|
+
try {
|
|
189
|
+
await writeFile(staged, `${JSON.stringify(file)}\n`, { mode: 0o644 });
|
|
190
|
+
await chmod(staged, 0o644);
|
|
191
|
+
await rename(staged, this.indexPath);
|
|
192
|
+
}
|
|
193
|
+
catch (cause) {
|
|
194
|
+
await rm(staged, { force: true }).catch(() => { });
|
|
195
|
+
throw cause;
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
/** Flush the index; further `put`/`persist` calls no-op. */
|
|
199
|
+
async close() {
|
|
200
|
+
if (this.closed)
|
|
201
|
+
return;
|
|
202
|
+
await this.persist();
|
|
203
|
+
this.closed = true;
|
|
204
|
+
}
|
|
205
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* PNG conversion for the image proxy (UI-blocks U7 Task 3, spec §11.2
|
|
3
|
+
* ruling 17): with `as=png`, the server converts the fetched (or
|
|
4
|
+
* source-form) image to PNG with `sharp` (a server dependency) and the
|
|
5
|
+
* route caches the result beside the original variant. The TUI asks for
|
|
6
|
+
* it (pi-tui draws only PNG on Kitty without a registered transcoder);
|
|
7
|
+
* the webui doesn't.
|
|
8
|
+
*/
|
|
9
|
+
import sharp from "sharp";
|
|
10
|
+
/** Convert image bytes (PNG/JPEG/GIF/WebP) to PNG bytes. */
|
|
11
|
+
export async function toPng(bytes) {
|
|
12
|
+
return await sharp(bytes).png().toBuffer();
|
|
13
|
+
}
|
|
@@ -0,0 +1,302 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Vetted-address image fetching (UI-blocks U7, spec §11.2 verbatim):
|
|
3
|
+
* the proxy's DNS-rebinding defense. Every hop is ONE `https.request`
|
|
4
|
+
* whose `lookup` option answers with the SINGLE vetted address for
|
|
5
|
+
* that hop — the connection can never re-resolve, so a resolver that
|
|
6
|
+
* answers a public address first and `127.0.0.1` second cannot cause
|
|
7
|
+
* a connection to loopback.
|
|
8
|
+
*
|
|
9
|
+
* Rules (each enforced per hop, redirects included):
|
|
10
|
+
* - https only; `http:` URLs and `http:` redirect targets are refused;
|
|
11
|
+
* - the host is resolved ONCE per hop through `deps.resolve`; every
|
|
12
|
+
* returned address is checked against the refused ranges — one bad
|
|
13
|
+
* address refuses the host (no happy-eyeballs fallback into it);
|
|
14
|
+
* - `servername` (SNI) and the `Host` header keep the ORIGINAL host
|
|
15
|
+
* name, never the resolved address;
|
|
16
|
+
* - redirects are followed BY HAND (no library follow option): at most
|
|
17
|
+
* 3 in total, each target re-resolved and re-checked the same way;
|
|
18
|
+
* - 10 s total timeout for the whole fetch — ABSOLUTE across the
|
|
19
|
+
* resolve, connect, and header phases of every hop plus the body
|
|
20
|
+
* read (a socket-inactivity timeout would never fire on a drip);
|
|
21
|
+
* - no agent reuse across hops (`agent: false`): Node's default
|
|
22
|
+
* keep-alive agent could serve a redirect hop from a pooled socket
|
|
23
|
+
* whose connection was vetted on the FIRST hop only;
|
|
24
|
+
* - 10 MiB body cap;
|
|
25
|
+
* - the body must sniff as PNG/JPEG/GIF/WebP (Content-Type is never
|
|
26
|
+
* trusted; SVG and AVIF are refused).
|
|
27
|
+
*/
|
|
28
|
+
import * as https from "node:https";
|
|
29
|
+
import { isRefusedAddress } from "./ranges.js";
|
|
30
|
+
import { isAcceptedImageType, sniffImage } from "./sniff.js";
|
|
31
|
+
/** The whole-fetch budget: 10 s, spec §11.2. */
|
|
32
|
+
export const FETCH_TIMEOUT_MS = 10_000;
|
|
33
|
+
/** The response body cap: 10 MiB, spec §11.2. */
|
|
34
|
+
export const FETCH_MAX_BYTES = 10 * 1024 * 1024;
|
|
35
|
+
/** At most 3 redirects in total, spec §11.2. */
|
|
36
|
+
export const FETCH_MAX_REDIRECTS = 3;
|
|
37
|
+
/** A refusal: the reason plus the message the caller logs. */
|
|
38
|
+
export class FetchRefusal extends Error {
|
|
39
|
+
reason;
|
|
40
|
+
constructor(reason, message) {
|
|
41
|
+
super(message);
|
|
42
|
+
this.reason = reason;
|
|
43
|
+
this.name = "FetchRefusal";
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
/** Race `p` against the whole-fetch deadline.
|
|
47
|
+
*
|
|
48
|
+
* The deadline must be absolute even in phases a socket-inactivity
|
|
49
|
+
* timeout can never cut: a resolver that hangs, or a server dripping
|
|
50
|
+
* header bytes. Rejects with `timeout` when the deadline passes
|
|
51
|
+
* before `p` settles.
|
|
52
|
+
*/
|
|
53
|
+
function withDeadline(p, deps, deadline, phase) {
|
|
54
|
+
return new Promise((resolve, reject) => {
|
|
55
|
+
const remaining = deadline - deps.now();
|
|
56
|
+
if (remaining <= 0) {
|
|
57
|
+
// Mark `p`'s (possible later) rejection handled: the
|
|
58
|
+
// deadline refusal is the one the caller sees.
|
|
59
|
+
void p.catch(() => { });
|
|
60
|
+
reject(new FetchRefusal("timeout", `deadline reached before ${phase}`));
|
|
61
|
+
return;
|
|
62
|
+
}
|
|
63
|
+
const timer = setTimeout(() => {
|
|
64
|
+
reject(new FetchRefusal("timeout", `deadline reached ${phase}`));
|
|
65
|
+
}, remaining);
|
|
66
|
+
p.then((value) => {
|
|
67
|
+
clearTimeout(timer);
|
|
68
|
+
resolve(value);
|
|
69
|
+
}, (cause) => {
|
|
70
|
+
clearTimeout(timer);
|
|
71
|
+
reject(cause);
|
|
72
|
+
});
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
/** Resolve and vet one hop's host, or refuse it. */
|
|
76
|
+
async function vettedAddress(host, deps) {
|
|
77
|
+
let addresses;
|
|
78
|
+
try {
|
|
79
|
+
addresses = await deps.resolve(host);
|
|
80
|
+
}
|
|
81
|
+
catch (cause) {
|
|
82
|
+
throw new FetchRefusal("resolve", `resolving ${host}: ${String(cause)}`);
|
|
83
|
+
}
|
|
84
|
+
if (addresses.length === 0) {
|
|
85
|
+
throw new FetchRefusal("resolve", `${host} resolved to no addresses`);
|
|
86
|
+
}
|
|
87
|
+
const vet = deps.vet ?? isRefusedAddress;
|
|
88
|
+
for (const address of addresses) {
|
|
89
|
+
if (vet(address)) {
|
|
90
|
+
throw new FetchRefusal("refused-address", `${host} resolves to refused address ${address}`);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
// The FIRST vetted address is the one the connection uses; the
|
|
94
|
+
// lookup callback never consults the resolver again.
|
|
95
|
+
return addresses[0];
|
|
96
|
+
}
|
|
97
|
+
/** One `https.request` hop: connect to `address`, keep SNI/Host.
|
|
98
|
+
*
|
|
99
|
+
* The connect+header phase is cut by an ABSOLUTE whole-fetch deadline
|
|
100
|
+
* timer (not `req.setTimeout`, which is socket-inactivity: a
|
|
101
|
+
* header-drip server resets it forever). The timer is cleared the
|
|
102
|
+
* moment headers resolve; the body read has its own deadline timer.
|
|
103
|
+
*/
|
|
104
|
+
function hop(options) {
|
|
105
|
+
const { url, address, deps, deadline } = options;
|
|
106
|
+
return new Promise((resolve, reject) => {
|
|
107
|
+
const remaining = deadline - deps.now();
|
|
108
|
+
if (remaining <= 0) {
|
|
109
|
+
reject(new FetchRefusal("timeout", "deadline reached before connecting"));
|
|
110
|
+
return;
|
|
111
|
+
}
|
|
112
|
+
const req = https.request({
|
|
113
|
+
protocol: "https:",
|
|
114
|
+
host: url.hostname,
|
|
115
|
+
port: url.port || 443,
|
|
116
|
+
servername: url.hostname,
|
|
117
|
+
path: `${url.pathname}${url.search}`,
|
|
118
|
+
method: "GET",
|
|
119
|
+
// The rebinding defense: the connection dials the ONE
|
|
120
|
+
// vetted address, never the hostname (and never a second
|
|
121
|
+
// resolution). Node calls `lookup` with `all: true`.
|
|
122
|
+
lookup(_host, opts, callback) {
|
|
123
|
+
const family = address.includes(":") ? 6 : 4;
|
|
124
|
+
if (opts && typeof opts === "object" && "all" in opts && opts.all) {
|
|
125
|
+
callback(null, [{ address, family }]);
|
|
126
|
+
}
|
|
127
|
+
else {
|
|
128
|
+
callback(null, address, family);
|
|
129
|
+
}
|
|
130
|
+
},
|
|
131
|
+
// No explicit Host header: Node derives it from the
|
|
132
|
+
// ORIGINAL hostname (and the port), never the address the
|
|
133
|
+
// lookup returned.
|
|
134
|
+
ca: deps.ca,
|
|
135
|
+
// NO agent reuse across hops: with the default keep-alive
|
|
136
|
+
// global agent (Node >= 19), a redirect hop to the same
|
|
137
|
+
// host:port could be served by the pooled FIRST-hop
|
|
138
|
+
// socket — whose lookup (and vetted address) is the first
|
|
139
|
+
// hop's, so this hop's vetting would never run.
|
|
140
|
+
agent: false,
|
|
141
|
+
}, (res) => {
|
|
142
|
+
// Headers are in: the hop's absolute timer comes down; the
|
|
143
|
+
// caller owns the stream (redirect hops drain it, the
|
|
144
|
+
// final hop reads it).
|
|
145
|
+
clearTimeout(timer);
|
|
146
|
+
resolve({ status: res.statusCode ?? 0, headers: res.headers, stream: res });
|
|
147
|
+
});
|
|
148
|
+
let timedOut = false;
|
|
149
|
+
const timer = setTimeout(() => {
|
|
150
|
+
timedOut = true;
|
|
151
|
+
req.destroy();
|
|
152
|
+
reject(new FetchRefusal("timeout", `deadline reached awaiting headers from ${url.hostname}`));
|
|
153
|
+
}, remaining);
|
|
154
|
+
// The absolute timer: cleared on headers (above) AND on a
|
|
155
|
+
// connect-phase error — an early failure must not leave a timer
|
|
156
|
+
// holding the event loop (or fake clocks) open for the full
|
|
157
|
+
// remaining budget (Task 3 pickup of Task 2's review P2).
|
|
158
|
+
req.on("error", (cause) => {
|
|
159
|
+
clearTimeout(timer);
|
|
160
|
+
if (timedOut)
|
|
161
|
+
return; // the destroy() above already rejected
|
|
162
|
+
reject(new FetchRefusal("network", `connecting to ${url.hostname}: ${cause.message}`));
|
|
163
|
+
});
|
|
164
|
+
req.end();
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
/** Read a body stream under the deadline and the byte cap. */
|
|
168
|
+
function readBody(stream, deps, deadline) {
|
|
169
|
+
return new Promise((resolve, reject) => {
|
|
170
|
+
const chunks = [];
|
|
171
|
+
let received = 0;
|
|
172
|
+
const fail = (refusal) => {
|
|
173
|
+
stream.destroy?.();
|
|
174
|
+
reject(refusal);
|
|
175
|
+
};
|
|
176
|
+
// The whole-fetch deadline: checked on every read and by a
|
|
177
|
+
// timer, so a drip that stops delivering entirely is cut too.
|
|
178
|
+
const timer = setTimeout(() => {
|
|
179
|
+
fail(new FetchRefusal("timeout", "deadline reached reading body"));
|
|
180
|
+
}, Math.max(0, deadline - deps.now()));
|
|
181
|
+
stream.on("data", (chunk) => {
|
|
182
|
+
received += chunk.length;
|
|
183
|
+
if (received > FETCH_MAX_BYTES) {
|
|
184
|
+
clearTimeout(timer);
|
|
185
|
+
fail(new FetchRefusal("too-large", `body over ${FETCH_MAX_BYTES} bytes`));
|
|
186
|
+
return;
|
|
187
|
+
}
|
|
188
|
+
chunks.push(chunk);
|
|
189
|
+
const remaining = deadline - deps.now();
|
|
190
|
+
if (remaining <= 0) {
|
|
191
|
+
clearTimeout(timer);
|
|
192
|
+
fail(new FetchRefusal("timeout", "deadline reached reading body"));
|
|
193
|
+
}
|
|
194
|
+
});
|
|
195
|
+
stream.on("end", () => {
|
|
196
|
+
clearTimeout(timer);
|
|
197
|
+
resolve(Buffer.concat(chunks));
|
|
198
|
+
});
|
|
199
|
+
stream.on("error", (cause) => {
|
|
200
|
+
clearTimeout(timer);
|
|
201
|
+
fail(new FetchRefusal("network", `reading body: ${cause.message}`));
|
|
202
|
+
});
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
/** Parse `Cache-Control: max-age=N` from response headers. */
|
|
206
|
+
function maxAgeOf(headers) {
|
|
207
|
+
const raw = headers["cache-control"];
|
|
208
|
+
if (raw === undefined)
|
|
209
|
+
return undefined;
|
|
210
|
+
for (const header of Array.isArray(raw) ? raw : [raw]) {
|
|
211
|
+
for (const directive of header.split(",")) {
|
|
212
|
+
const trimmed = directive.trim();
|
|
213
|
+
if (trimmed.startsWith("max-age=")) {
|
|
214
|
+
const parsed = Number(trimmed.slice("max-age=".length));
|
|
215
|
+
if (Number.isFinite(parsed) && parsed >= 0)
|
|
216
|
+
return parsed;
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
return undefined;
|
|
221
|
+
}
|
|
222
|
+
/**
|
|
223
|
+
* Fetch `u` as an image: resolve-and-vet every hop, follow at most 3
|
|
224
|
+
* redirects by hand, cap the body, and require the sniffed type.
|
|
225
|
+
*/
|
|
226
|
+
export async function fetchImage(u, deps) {
|
|
227
|
+
const deadline = deps.now() + FETCH_TIMEOUT_MS;
|
|
228
|
+
let current;
|
|
229
|
+
try {
|
|
230
|
+
current = new URL(u);
|
|
231
|
+
}
|
|
232
|
+
catch {
|
|
233
|
+
throw new FetchRefusal("bad-url", `not a URL: ${u}`);
|
|
234
|
+
}
|
|
235
|
+
if (current.protocol !== "https:") {
|
|
236
|
+
throw new FetchRefusal("scheme", `not https: ${current.protocol}`);
|
|
237
|
+
}
|
|
238
|
+
let redirects = 0;
|
|
239
|
+
// Each loop iteration is one hop: vet, connect, then either take
|
|
240
|
+
// the redirect (draining and looping) or read the body.
|
|
241
|
+
for (;;) {
|
|
242
|
+
// Every hop resolves through `deps.resolve` — including IP
|
|
243
|
+
// literal hosts (the real resolver, `dns.lookup`, passes literals
|
|
244
|
+
// through verbatim, so literals get range-checked like any other
|
|
245
|
+
// answer). Brackets stripped for the resolver and SNI; the Host
|
|
246
|
+
// header keeps Node's own derivation from the URL. The deadline
|
|
247
|
+
// is ABSOLUTE here too: a resolver that hangs is cut at it.
|
|
248
|
+
const bareHost = current.hostname.replace(/^\[|\]$/g, "");
|
|
249
|
+
const address = await withDeadline(vettedAddress(bareHost, deps), deps, deadline, `resolving ${bareHost}`);
|
|
250
|
+
const response = await hop({ url: current, address, deps, deadline });
|
|
251
|
+
const status = response.status;
|
|
252
|
+
if (status >= 300 && status < 400 && response.headers.location !== undefined) {
|
|
253
|
+
// Redirect hops count against the same total budget even
|
|
254
|
+
// when refused for scheme or address — the attempt was made.
|
|
255
|
+
if (redirects >= FETCH_MAX_REDIRECTS) {
|
|
256
|
+
void consume(response.stream);
|
|
257
|
+
throw new FetchRefusal("too-many-redirects", `over ${FETCH_MAX_REDIRECTS} redirects at ${current.toString()}`);
|
|
258
|
+
}
|
|
259
|
+
redirects += 1;
|
|
260
|
+
let target;
|
|
261
|
+
try {
|
|
262
|
+
target = new URL(response.headers.location, current);
|
|
263
|
+
}
|
|
264
|
+
catch {
|
|
265
|
+
void consume(response.stream);
|
|
266
|
+
throw new FetchRefusal("bad-url", `redirect target not a URL: ${u}`);
|
|
267
|
+
}
|
|
268
|
+
if (target.protocol !== "https:") {
|
|
269
|
+
void consume(response.stream);
|
|
270
|
+
throw new FetchRefusal("redirect-scheme", `redirect to ${target.protocol} refused`);
|
|
271
|
+
}
|
|
272
|
+
// Drain the redirect response so the server sees it consumed
|
|
273
|
+
// and the socket closes cleanly (with `agent: false` there is
|
|
274
|
+
// no pool for it to return to — the drain is hygiene, not reuse).
|
|
275
|
+
void consume(response.stream);
|
|
276
|
+
current = target;
|
|
277
|
+
continue;
|
|
278
|
+
}
|
|
279
|
+
if (status < 200 || status >= 300) {
|
|
280
|
+
void consume(response.stream);
|
|
281
|
+
throw new FetchRefusal("http-status", `status ${status} from ${current.toString()}`);
|
|
282
|
+
}
|
|
283
|
+
const bytes = await readBody(response.stream, deps, deadline);
|
|
284
|
+
const sniffed = sniffImage(bytes);
|
|
285
|
+
if (!isAcceptedImageType(sniffed)) {
|
|
286
|
+
throw new FetchRefusal("bad-type", sniffed === undefined
|
|
287
|
+
? `body did not sniff as an accepted image type`
|
|
288
|
+
: `body sniffed as ${sniffed}, which is refused`);
|
|
289
|
+
}
|
|
290
|
+
const maxAge = maxAgeOf(response.headers);
|
|
291
|
+
return { bytes, type: sniffed, ...(maxAge === undefined ? {} : { maxAge }) };
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
/** Best-effort drain of a refused hop's response so the server sees
|
|
295
|
+
* the response consumed and the socket closes cleanly. */
|
|
296
|
+
function consume(stream) {
|
|
297
|
+
return new Promise((resolve) => {
|
|
298
|
+
stream.on("data", () => { });
|
|
299
|
+
stream.on("end", () => resolve());
|
|
300
|
+
stream.on("error", () => resolve());
|
|
301
|
+
});
|
|
302
|
+
}
|