@fuzdev/fuz_app 0.146.0 → 0.147.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth/CLAUDE.md +1 -1
- package/dist/auth/bootstrap_account.d.ts +2 -1
- package/dist/auth/bootstrap_account.d.ts.map +1 -1
- package/dist/auth/bootstrap_account.js +17 -6
- package/dist/auth/keyring.d.ts +35 -38
- package/dist/auth/keyring.d.ts.map +1 -1
- package/dist/auth/keyring.js +43 -50
- package/dist/auth/password_argon2.d.ts.map +1 -1
- package/dist/auth/password_argon2.js +11 -8
- package/dist/auth/session_cookie.d.ts.map +1 -1
- package/dist/auth/session_cookie.js +5 -2
- package/dist/db/CLAUDE.md +18 -16
- package/dist/db/cell_queries.d.ts +3 -2
- package/dist/db/cell_queries.d.ts.map +1 -1
- package/dist/db/cell_queries.js +3 -2
- package/dist/db/fact_disk_storage.d.ts +15 -15
- package/dist/db/fact_disk_storage.d.ts.map +1 -1
- package/dist/db/fact_disk_storage.js +22 -28
- package/dist/db/fact_queries.d.ts +2 -3
- package/dist/db/fact_queries.d.ts.map +1 -1
- package/dist/db/fact_queries.js +2 -3
- package/dist/db/fact_store.d.ts +20 -18
- package/dist/db/fact_store.d.ts.map +1 -1
- package/dist/db/fact_store.js +30 -44
- package/dist/db/file_fact_url.d.ts +19 -4
- package/dist/db/file_fact_url.d.ts.map +1 -1
- package/dist/db/file_fact_url.js +19 -4
- package/dist/server/env.d.ts.map +1 -1
- package/dist/server/env.js +2 -2
- package/dist/server/serve_fact_route.d.ts +9 -5
- package/dist/server/serve_fact_route.d.ts.map +1 -1
- package/dist/server/serve_fact_route.js +15 -7
- package/dist/testing/CLAUDE.md +19 -2
- package/dist/testing/app_server.d.ts.map +1 -1
- package/dist/testing/app_server.js +2 -2
- package/dist/testing/cross_backend/bootstrap_backend.d.ts +11 -1
- package/dist/testing/cross_backend/bootstrap_backend.d.ts.map +1 -1
- package/dist/testing/cross_backend/bootstrap_backend.js +2 -1
- package/dist/testing/cross_backend/fact_serving.d.ts +5 -0
- package/dist/testing/cross_backend/fact_serving.d.ts.map +1 -1
- package/dist/testing/cross_backend/fact_serving.js +25 -1
- package/dist/testing/cross_backend/session_cookie_codec.d.ts +19 -0
- package/dist/testing/cross_backend/session_cookie_codec.d.ts.map +1 -0
- package/dist/testing/cross_backend/session_cookie_codec.js +128 -0
- package/dist/testing/cross_backend/testing_reset_actions.d.ts +7 -3
- package/dist/testing/cross_backend/testing_reset_actions.d.ts.map +1 -1
- package/dist/testing/cross_backend/testing_reset_actions.js +13 -8
- package/dist/ui/Datatable.svelte +3 -3
- package/package.json +6 -6
|
@@ -20,14 +20,15 @@
|
|
|
20
20
|
* re-hashing it (`server/serve_fact_route.ts`). This twins the Rust `fuz_fact`
|
|
21
21
|
* §fsync posture: data-sync before the rename; the parent-dir fsync stays
|
|
22
22
|
* deliberately waived (a lost dirent is regenerable under content addressing).
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
23
|
+
* The temp is renamed over the final path even when a file is already there:
|
|
24
|
+
* the temp's bytes are the ones just hashed, so the rename is a no-op for a
|
|
25
|
+
* sound file and replaces a corrupt one (a torn write, bit rot) with bytes that
|
|
26
|
+
* match their name — a re-upload heals the CAS. `.tmp/` is a sibling of
|
|
27
|
+
* `<shard>/` under the same `facts_dir` so `rename` is always same-filesystem
|
|
28
|
+
* (no EXDEV).
|
|
27
29
|
*
|
|
28
30
|
* @module
|
|
29
31
|
*/
|
|
30
|
-
import { createHash } from 'node:crypto';
|
|
31
32
|
import { join } from 'node:path';
|
|
32
33
|
import { Blake3Hasher } from '@fuzdev/blake3-wasm';
|
|
33
34
|
import { to_error_message } from '@fuzdev/fuz_util/error.ts';
|
|
@@ -42,12 +43,11 @@ export const FACT_TMP_DIRNAME = '.tmp';
|
|
|
42
43
|
/** Default age (1 hour) past which a `.tmp/*` file is considered orphaned. */
|
|
43
44
|
export const FACT_TMP_ORPHAN_MAX_AGE_MS = 60 * 60 * 1000;
|
|
44
45
|
/**
|
|
45
|
-
* Stream `source` to storage with bounded memory: hash BLAKE3
|
|
46
|
-
*
|
|
47
|
-
*
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
* never buffers in RAM.
|
|
46
|
+
* Stream `source` to storage with bounded memory: hash BLAKE3 incrementally,
|
|
47
|
+
* buffer in memory until the bytes cross `embedded_threshold`, then spill the
|
|
48
|
+
* buffer + remaining chunks through a temp file and atomically land it in the
|
|
49
|
+
* disk CAS. Peak heap is `O(chunk + embedded_threshold)`, never `O(artifact)`, so a
|
|
50
|
+
* multi-GB upload never buffers in RAM.
|
|
51
51
|
*
|
|
52
52
|
* - **Embedded vs disk.** A body `<= embedded_threshold` stays in memory and is
|
|
53
53
|
* returned as `{kind: 'embedded'}` for the PG `bytes` column. Above it (with a
|
|
@@ -66,7 +66,6 @@ export const FACT_TMP_ORPHAN_MAX_AGE_MS = 60 * 60 * 1000;
|
|
|
66
66
|
export const stream_fact_to_disk = async (deps, facts_dir, source, max_bytes, embedded_threshold) => {
|
|
67
67
|
await blake3_ready;
|
|
68
68
|
const blake3 = new Blake3Hasher();
|
|
69
|
-
const sha256 = createHash('sha256');
|
|
70
69
|
let size = 0;
|
|
71
70
|
// Buffer leading bytes until they cross the embedded threshold; small facts
|
|
72
71
|
// stay embedded (no disk), large ones never buffer past the threshold.
|
|
@@ -78,7 +77,6 @@ export const stream_fact_to_disk = async (deps, facts_dir, source, max_bytes, em
|
|
|
78
77
|
if (size > max_bytes)
|
|
79
78
|
throw new FactPayloadTooLargeError(size, max_bytes);
|
|
80
79
|
blake3.update(chunk);
|
|
81
|
-
sha256.update(chunk);
|
|
82
80
|
};
|
|
83
81
|
try {
|
|
84
82
|
// Phase 1: read + hash + buffer until the threshold is crossed or the
|
|
@@ -103,7 +101,6 @@ export const stream_fact_to_disk = async (deps, facts_dir, source, max_bytes, em
|
|
|
103
101
|
const hash = (FACT_HASH_PREFIX + to_hex(blake3.finalize()));
|
|
104
102
|
return {
|
|
105
103
|
hash,
|
|
106
|
-
sha256: sha256.digest('hex'),
|
|
107
104
|
size,
|
|
108
105
|
placement: { kind: 'embedded', bytes: concat_chunks(buffered, buffered_len) }
|
|
109
106
|
};
|
|
@@ -161,7 +158,6 @@ export const stream_fact_to_disk = async (deps, facts_dir, source, max_bytes, em
|
|
|
161
158
|
const { shard, rest } = await commit_temp_to_cas(deps, tmp_path, facts_dir, hash);
|
|
162
159
|
return {
|
|
163
160
|
hash,
|
|
164
|
-
sha256: sha256.digest('hex'),
|
|
165
161
|
size,
|
|
166
162
|
placement: { kind: 'disk', external_url: mint_file_fact_url(shard, rest) }
|
|
167
163
|
};
|
|
@@ -179,7 +175,7 @@ export const stream_fact_to_disk = async (deps, facts_dir, source, max_bytes, em
|
|
|
179
175
|
/**
|
|
180
176
|
* Write fully-buffered `bytes` for `hash` to the canonical
|
|
181
177
|
* `<facts_dir>/<shard>/<rest>` path, then publish via `commit_temp_to_cas`
|
|
182
|
-
* (fsync'd temp + atomic rename
|
|
178
|
+
* (fsync'd temp + atomic rename). The buffering twin of
|
|
183
179
|
* `stream_fact_to_disk`, used by `PgFactStore.put` for oversize sync bytes.
|
|
184
180
|
* Returns the `file:` `external_url` for the `fact` row.
|
|
185
181
|
*
|
|
@@ -271,11 +267,14 @@ export const sweep_orphan_temps = async (deps, facts_dir, options) => {
|
|
|
271
267
|
* Publish a written temp file into the CAS at `<facts_dir>/<shard>/<rest>`:
|
|
272
268
|
* `fsync` the temp's data (durability before the rename — the serve path streams
|
|
273
269
|
* the file without re-hashing, so the bytes must be stable before they become
|
|
274
|
-
* the canonical body), then
|
|
275
|
-
*
|
|
276
|
-
*
|
|
277
|
-
*
|
|
278
|
-
*
|
|
270
|
+
* the canonical body), then atomically `rename` it into place, over a file
|
|
271
|
+
* already there. The temp holds the bytes `hash` was computed from, so
|
|
272
|
+
* replacing an existing file is harmless when it is sound and heals it when it
|
|
273
|
+
* is not; a reader holding the old file open keeps reading the old inode. On
|
|
274
|
+
* any failure the temp is unlinked and an `ENOSPC` is surfaced as
|
|
275
|
+
* `FactStorageFullError`. The single commit path shared by both writers above
|
|
276
|
+
* (data-sync before rename; parent-dir fsync waived, as on the Rust `fuz_fact`
|
|
277
|
+
* disk CAS).
|
|
279
278
|
*
|
|
280
279
|
* @mutates `facts_dir` filesystem
|
|
281
280
|
*/
|
|
@@ -284,13 +283,8 @@ const commit_temp_to_cas = async (deps, tmp_path, facts_dir, hash) => {
|
|
|
284
283
|
const final_path = join(facts_dir, shard, rest);
|
|
285
284
|
try {
|
|
286
285
|
await deps.fsync(tmp_path);
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
}
|
|
290
|
-
else {
|
|
291
|
-
await deps.mkdir(join(facts_dir, shard), { recursive: true });
|
|
292
|
-
await deps.rename(tmp_path, final_path);
|
|
293
|
-
}
|
|
286
|
+
await deps.mkdir(join(facts_dir, shard), { recursive: true });
|
|
287
|
+
await deps.rename(tmp_path, final_path);
|
|
294
288
|
}
|
|
295
289
|
catch (err) {
|
|
296
290
|
await deps.remove(tmp_path).catch(() => undefined);
|
|
@@ -5,9 +5,8 @@
|
|
|
5
5
|
* idempotent (`ON CONFLICT DO NOTHING`) so the same hash can be written by
|
|
6
6
|
* two callers without the second observing an error.
|
|
7
7
|
*
|
|
8
|
-
* Higher-level lifecycle (verify-on-read,
|
|
9
|
-
*
|
|
10
|
-
* here are deliberately mechanical.
|
|
8
|
+
* Higher-level lifecycle (verify-on-read, embedded-vs-referenced selection)
|
|
9
|
+
* lives in `db/fact_store.ts`. Queries here are deliberately mechanical.
|
|
11
10
|
*
|
|
12
11
|
* @module
|
|
13
12
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fact_queries.d.ts","sourceRoot":"../src/lib/","sources":["../../src/lib/db/fact_queries.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"fact_queries.d.ts","sourceRoot":"../src/lib/","sources":["../../src/lib/db/fact_queries.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEjD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,kCAAkC,CAAC;AAGjE,0CAA0C;AAC1C,MAAM,WAAW,OAAO;IACvB,IAAI,EAAE,QAAQ,CAAC;IACf,KAAK,EAAE,UAAU,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAK5B,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;IACtB,UAAU,EAAE,IAAI,CAAC;CACjB;AAED;;;;;GAKG;AACH,eAAO,MAAM,YAAY,kFAOwB,CAAC;AAYlD,qEAAqE;AACrE,MAAM,WAAW,WAAW;IAC3B,IAAI,EAAE,QAAQ,CAAC;IACf,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAE5B,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;IACtB,UAAU,EAAE,IAAI,CAAC;CACjB;AAED,oFAAoF;AACpF,MAAM,WAAW,YAAY;IAC5B,IAAI,EAAE,QAAQ,CAAC;IACf,KAAK,EAAE,UAAU,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,IAAI,EAAE,MAAM,CAAC;CACb;AAED;;;;;;;;GAQG;AACH,eAAO,MAAM,cAAc,GAAU,MAAM,SAAS,EAAE,OAAO,YAAY,KAAG,OAAO,CAAC,OAAO,CAS1F,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,mBAAmB,GAC/B,MAAM,SAAS,EACf,aAAa,QAAQ,EACrB,eAAe,KAAK,CAAC,QAAQ,CAAC,KAC5B,OAAO,CAAC,IAAI,CAQd,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,cAAc,GAAU,MAAM,SAAS,EAAE,MAAM,QAAQ,KAAG,OAAO,CAAC,OAAO,GAAG,IAAI,CAO5F,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,mBAAmB,GAC/B,MAAM,SAAS,EACf,MAAM,QAAQ,KACZ,OAAO,CAAC,WAAW,GAAG,IAAI,CAO5B,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,cAAc,GAAU,MAAM,SAAS,EAAE,MAAM,QAAQ,KAAG,OAAO,CAAC,OAAO,CAMrF,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,mBAAmB,GAC/B,MAAM,SAAS,EACf,aAAa,QAAQ,KACnB,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAMzB,CAAC;AAEF;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,iBAAiB,GAC7B,MAAM,SAAS,EACf,MAAM,QAAQ,KACZ,OAAO,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAAG,IAAI,CAQ9D,CAAC;AAkBF;;;;;;GAMG;AACH,MAAM,WAAW,qBAAqB;IACrC,KAAK,EAAE,MAAM,CAAC;IACd,gBAAgB,EAAE,MAAM,CAAC;IACzB,MAAM,EAAE,KAAK,CAAC;QACb,IAAI,EAAE,QAAQ,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;QACb,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;KAC5B,CAAC,CAAC;CACH;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,eAAO,MAAM,uBAAuB,GACnC,MAAM,SAAS,EACf,YAAY,IAAI,GAAG,IAAI,EACvB,cAAc,MAAM,KAClB,OAAO,CAAC,qBAAqB,CA6C/B,CAAC;AAEF;;;;;;;;GAQG;AACH,eAAO,MAAM,oCAAoC,GAChD,MAAM,SAAS,EACf,YAAY,IAAI,KACd,OAAO,CAAC,KAAK,CAAC;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,CAkB9E,CAAC"}
|
package/dist/db/fact_queries.js
CHANGED
|
@@ -5,9 +5,8 @@
|
|
|
5
5
|
* idempotent (`ON CONFLICT DO NOTHING`) so the same hash can be written by
|
|
6
6
|
* two callers without the second observing an error.
|
|
7
7
|
*
|
|
8
|
-
* Higher-level lifecycle (verify-on-read,
|
|
9
|
-
*
|
|
10
|
-
* here are deliberately mechanical.
|
|
8
|
+
* Higher-level lifecycle (verify-on-read, embedded-vs-referenced selection)
|
|
9
|
+
* lives in `db/fact_store.ts`. Queries here are deliberately mechanical.
|
|
11
10
|
*
|
|
12
11
|
* @module
|
|
13
12
|
*/
|
package/dist/db/fact_store.d.ts
CHANGED
|
@@ -6,8 +6,8 @@
|
|
|
6
6
|
*
|
|
7
7
|
* - sync hash on `put`; `put_stream` hashes as it streams
|
|
8
8
|
* - idempotent insert (`ON CONFLICT DO NOTHING` in the queries layer)
|
|
9
|
-
* -
|
|
10
|
-
*
|
|
9
|
+
* - refs are exactly the caller's `options.refs` — the bytes are never
|
|
10
|
+
* scanned for hashes, whatever the `content_type`
|
|
11
11
|
* - verify-on-read for disk-backed content; embedded reads skip verify
|
|
12
12
|
* because PG storage IS the hash table
|
|
13
13
|
* - mismatched disk bytes return `null` + log warning (treat as
|
|
@@ -17,10 +17,10 @@
|
|
|
17
17
|
* land in the PG `bytes` column; larger bytes go to the disk CAS at
|
|
18
18
|
* `<facts_dir>/<shard>/<rest>` (`db/fact_disk_storage.ts`) and the row records a
|
|
19
19
|
* `file:<shard>/<rest>` `external_url`. `put` takes fully-buffered bytes;
|
|
20
|
-
* `put_stream` is the bounded-memory streaming twin (hash BLAKE3
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
20
|
+
* `put_stream` is the bounded-memory streaming twin (hash BLAKE3 as it streams,
|
|
21
|
+
* spill past the threshold, enforce `max_bytes` / `ENOSPC`). `put` needs
|
|
22
|
+
* `disk_root` + `fs` (the `runtime/*Deps`) for the over-threshold path only:
|
|
23
|
+
* without them the store has nowhere to put bytes it cannot embed, and an
|
|
24
24
|
* oversize `put` throws. `put_stream` needs `fs` for a body of any size — with
|
|
25
25
|
* none it throws before reading the stream — and `disk_root` for the
|
|
26
26
|
* over-threshold spill: with none, a body that passes the threshold throws
|
|
@@ -31,7 +31,9 @@
|
|
|
31
31
|
* — the store makes no network request. The column is free text
|
|
32
32
|
* (`query_put_fact` inserts what it is given), so `get` treats a row whose
|
|
33
33
|
* `external_url` has any other shape, and any external row on a store with no
|
|
34
|
-
* disk CAS configured, as unavailable: a warning and `null`.
|
|
34
|
+
* disk CAS configured, as unavailable: a warning and `null`. A `file:` URL
|
|
35
|
+
* naming another fact's file (`file_fact_url_addresses`) is refused the same
|
|
36
|
+
* way, logged as an error, before the file is read.
|
|
35
37
|
*
|
|
36
38
|
* ## Holders
|
|
37
39
|
*
|
|
@@ -133,24 +135,23 @@ export declare class PgFactStore implements HeldFactStore {
|
|
|
133
135
|
* are configured) at `<facts_dir>/<shard>/<rest>` with a `file:` URL. Oversize
|
|
134
136
|
* without a disk root throws: the store has nowhere to put bytes it cannot
|
|
135
137
|
* embed. Idempotent — `ON CONFLICT DO NOTHING` + content-addressed disk
|
|
136
|
-
* filenames make a re-write
|
|
137
|
-
*
|
|
138
|
+
* filenames make a re-write harmless (it renames verified bytes over the same
|
|
139
|
+
* file). Refs are exactly `options.refs` (none when omitted). `options.holder`,
|
|
140
|
+
* when set, is recorded as a holder on a fresh insert and a dedup hit alike.
|
|
138
141
|
*
|
|
139
142
|
* @mutates fact_holder - records `options.holder` (when non-null) in the transaction that writes the `fact` row
|
|
140
143
|
*/
|
|
141
144
|
put(bytes: Uint8Array, options: HeldFactPutOptions): Promise<FactHash>;
|
|
142
145
|
/**
|
|
143
|
-
* Stream bytes into the store with bounded memory, returning the
|
|
144
|
-
*
|
|
145
|
-
*
|
|
146
|
-
*
|
|
147
|
-
*
|
|
146
|
+
* Stream bytes into the store with bounded memory, returning the fact hash +
|
|
147
|
+
* size. Delegates the byte path to `stream_fact_to_disk` (hash BLAKE3 as it
|
|
148
|
+
* streams, buffer to the embedded threshold, spill to the disk CAS), then
|
|
149
|
+
* inserts the `fact` row by placement — embedded bytes go to the PG `bytes`
|
|
150
|
+
* column, disk-spilled bytes record the `file:` `external_url`.
|
|
148
151
|
* The cap is enforced mid-stream (`FactPayloadTooLargeError`); a disk-full mid-
|
|
149
152
|
* stream throws `FactStorageFullError`.
|
|
150
153
|
*
|
|
151
|
-
* Refs
|
|
152
|
-
* attempted (it would need a buffered re-read, defeating the bounded-memory
|
|
153
|
-
* contract) — streamed uploads are opaque blobs.
|
|
154
|
+
* Refs are exactly `options.refs` (none when omitted), as on `put`.
|
|
154
155
|
*
|
|
155
156
|
* Requires `fs` (and, for the over-threshold spill, `disk_root`) to be
|
|
156
157
|
* configured. The streaming twin of `put`; mirrors the Rust
|
|
@@ -167,7 +168,8 @@ export declare class PgFactStore implements HeldFactStore {
|
|
|
167
168
|
* Returns `null`, with a warning log, for an external row the store cannot
|
|
168
169
|
* serve — an `external_url` that is not a `file:<shard>/<rest>` URL, no disk
|
|
169
170
|
* CAS configured (`disk_root` + `fs`), a failed disk read, or bytes that
|
|
170
|
-
* don't match the hash
|
|
171
|
+
* don't match the hash — and, with an error log, for a URL that names
|
|
172
|
+
* another fact's file. Never makes a network request.
|
|
171
173
|
*/
|
|
172
174
|
get(hash: FactHash): Promise<Uint8Array | null>;
|
|
173
175
|
has(hash: FactHash): Promise<boolean>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fact_store.d.ts","sourceRoot":"../src/lib/","sources":["../../src/lib/db/fact_store.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"fact_store.d.ts","sourceRoot":"../src/lib/","sources":["../../src/lib/db/fact_store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkDG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEjD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAEpD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAGtD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,kCAAkC,CAAC;AACjE,OAAO,KAAK,EACX,QAAQ,EACR,cAAc,EACd,SAAS,EACT,gBAAgB,EAChB,MAAM,gCAAgC,CAAC;AAcxC,OAAO,EAIN,KAAK,mBAAmB,EACxB,MAAM,wBAAwB,CAAC;AAEhC;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,kBAAmB,SAAQ,cAAc;IACzD,MAAM,EAAE,IAAI,GAAG,IAAI,CAAC;CACpB;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,aAAc,SAAQ,IAAI,CAAC,SAAS,EAAE,KAAK,GAAG,YAAY,CAAC;IAC3E;;;OAGG;IACH,GAAG,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,kBAAkB,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC3E;;;OAGG;IACH,UAAU,EAAE,CACX,MAAM,EAAE,cAAc,CAAC,UAAU,CAAC,EAClC,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,kBAAkB,KACvB,OAAO,CAAC,gBAAgB,CAAC,CAAC;CAC/B;AAED,qDAAqD;AACrD,eAAO,MAAM,+BAA+B,QAAc,CAAC;AAE3D;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,WAAW,eAAe;IAC/B,IAAI,EAAE,SAAS,CAAC;IAChB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,EAAE,CAAC,EAAE,mBAAmB,CAAC;IACzB,GAAG,CAAC,EAAE,MAAM,CAAC;CACb;AAED;;;;GAIG;AACH,qBAAa,WAAY,YAAW,aAAa;;gBAOpC,OAAO,EAAE,eAAe;IAQpC;;;;;;;;;;;OAWG;IACG,GAAG,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,QAAQ,CAAC;IAgC5E;;;;;;;;;;;;;;;;OAgBG;IACG,UAAU,CACf,MAAM,EAAE,cAAc,CAAC,UAAU,CAAC,EAClC,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,kBAAkB,GACzB,OAAO,CAAC,gBAAgB,CAAC;IA+C5B;;;;;;;;;;OAUG;IACG,GAAG,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IAiD/C,GAAG,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC;IAIrC,QAAQ,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;IAWlD,QAAQ,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAIxD;;;;;;;;;;;;;;;;;;OAkBG;IACG,MAAM,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI,CAAC;CAG3F"}
|
package/dist/db/fact_store.js
CHANGED
|
@@ -6,8 +6,8 @@
|
|
|
6
6
|
*
|
|
7
7
|
* - sync hash on `put`; `put_stream` hashes as it streams
|
|
8
8
|
* - idempotent insert (`ON CONFLICT DO NOTHING` in the queries layer)
|
|
9
|
-
* -
|
|
10
|
-
*
|
|
9
|
+
* - refs are exactly the caller's `options.refs` — the bytes are never
|
|
10
|
+
* scanned for hashes, whatever the `content_type`
|
|
11
11
|
* - verify-on-read for disk-backed content; embedded reads skip verify
|
|
12
12
|
* because PG storage IS the hash table
|
|
13
13
|
* - mismatched disk bytes return `null` + log warning (treat as
|
|
@@ -17,10 +17,10 @@
|
|
|
17
17
|
* land in the PG `bytes` column; larger bytes go to the disk CAS at
|
|
18
18
|
* `<facts_dir>/<shard>/<rest>` (`db/fact_disk_storage.ts`) and the row records a
|
|
19
19
|
* `file:<shard>/<rest>` `external_url`. `put` takes fully-buffered bytes;
|
|
20
|
-
* `put_stream` is the bounded-memory streaming twin (hash BLAKE3
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
20
|
+
* `put_stream` is the bounded-memory streaming twin (hash BLAKE3 as it streams,
|
|
21
|
+
* spill past the threshold, enforce `max_bytes` / `ENOSPC`). `put` needs
|
|
22
|
+
* `disk_root` + `fs` (the `runtime/*Deps`) for the over-threshold path only:
|
|
23
|
+
* without them the store has nowhere to put bytes it cannot embed, and an
|
|
24
24
|
* oversize `put` throws. `put_stream` needs `fs` for a body of any size — with
|
|
25
25
|
* none it throws before reading the stream — and `disk_root` for the
|
|
26
26
|
* over-threshold spill: with none, a body that passes the threshold throws
|
|
@@ -31,7 +31,9 @@
|
|
|
31
31
|
* — the store makes no network request. The column is free text
|
|
32
32
|
* (`query_put_fact` inserts what it is given), so `get` treats a row whose
|
|
33
33
|
* `external_url` has any other shape, and any external row on a store with no
|
|
34
|
-
* disk CAS configured, as unavailable: a warning and `null`.
|
|
34
|
+
* disk CAS configured, as unavailable: a warning and `null`. A `file:` URL
|
|
35
|
+
* naming another fact's file (`file_fact_url_addresses`) is refused the same
|
|
36
|
+
* way, logged as an error, before the file is read.
|
|
35
37
|
*
|
|
36
38
|
* ## Holders
|
|
37
39
|
*
|
|
@@ -48,10 +50,10 @@
|
|
|
48
50
|
* @module
|
|
49
51
|
*/
|
|
50
52
|
import { to_error_message } from '@fuzdev/fuz_util/error.ts';
|
|
51
|
-
import { fact_hash_bytes, fact_hash_verify
|
|
53
|
+
import { fact_hash_bytes, fact_hash_verify } from '@fuzdev/fuz_util/fact_hash.ts';
|
|
52
54
|
import { query_delete_fact, query_get_fact, query_get_fact_meta, query_get_fact_refs, query_has_fact, query_put_fact, query_put_fact_refs } from './fact_queries.js';
|
|
53
55
|
import { query_put_fact_holder } from './fact_holder_queries.js';
|
|
54
|
-
import {
|
|
56
|
+
import { file_fact_url_addresses, parse_file_fact_url } from './file_fact_url.js';
|
|
55
57
|
import { read_fact_bytes_from_disk, stream_fact_to_disk, write_fact_bytes_to_disk } from './fact_disk_storage.js';
|
|
56
58
|
/** Default embedded-vs-referenced cutoff (1 MiB). */
|
|
57
59
|
export const FACT_EMBEDDED_THRESHOLD_DEFAULT = 1024 * 1024;
|
|
@@ -79,8 +81,9 @@ export class PgFactStore {
|
|
|
79
81
|
* are configured) at `<facts_dir>/<shard>/<rest>` with a `file:` URL. Oversize
|
|
80
82
|
* without a disk root throws: the store has nowhere to put bytes it cannot
|
|
81
83
|
* embed. Idempotent — `ON CONFLICT DO NOTHING` + content-addressed disk
|
|
82
|
-
* filenames make a re-write
|
|
83
|
-
*
|
|
84
|
+
* filenames make a re-write harmless (it renames verified bytes over the same
|
|
85
|
+
* file). Refs are exactly `options.refs` (none when omitted). `options.holder`,
|
|
86
|
+
* when set, is recorded as a holder on a fresh insert and a dedup hit alike.
|
|
84
87
|
*
|
|
85
88
|
* @mutates fact_holder - records `options.holder` (when non-null) in the transaction that writes the `fact` row
|
|
86
89
|
*/
|
|
@@ -105,21 +108,19 @@ export class PgFactStore {
|
|
|
105
108
|
external_url: row_external_url,
|
|
106
109
|
content_type: options.content_type ?? null,
|
|
107
110
|
size: bytes.length
|
|
108
|
-
},
|
|
111
|
+
}, options.refs ?? [], options.holder);
|
|
109
112
|
return hash;
|
|
110
113
|
}
|
|
111
114
|
/**
|
|
112
|
-
* Stream bytes into the store with bounded memory, returning the
|
|
113
|
-
*
|
|
114
|
-
*
|
|
115
|
-
*
|
|
116
|
-
*
|
|
115
|
+
* Stream bytes into the store with bounded memory, returning the fact hash +
|
|
116
|
+
* size. Delegates the byte path to `stream_fact_to_disk` (hash BLAKE3 as it
|
|
117
|
+
* streams, buffer to the embedded threshold, spill to the disk CAS), then
|
|
118
|
+
* inserts the `fact` row by placement — embedded bytes go to the PG `bytes`
|
|
119
|
+
* column, disk-spilled bytes record the `file:` `external_url`.
|
|
117
120
|
* The cap is enforced mid-stream (`FactPayloadTooLargeError`); a disk-full mid-
|
|
118
121
|
* stream throws `FactStorageFullError`.
|
|
119
122
|
*
|
|
120
|
-
* Refs
|
|
121
|
-
* attempted (it would need a buffered re-read, defeating the bounded-memory
|
|
122
|
-
* contract) — streamed uploads are opaque blobs.
|
|
123
|
+
* Refs are exactly `options.refs` (none when omitted), as on `put`.
|
|
123
124
|
*
|
|
124
125
|
* Requires `fs` (and, for the over-threshold spill, `disk_root`) to be
|
|
125
126
|
* configured. The streaming twin of `put`; mirrors the Rust
|
|
@@ -141,7 +142,7 @@ export class PgFactStore {
|
|
|
141
142
|
content_type: options.content_type ?? null,
|
|
142
143
|
size: streamed.size
|
|
143
144
|
}, options.refs ?? [], options.holder);
|
|
144
|
-
return { hash: streamed.hash,
|
|
145
|
+
return { hash: streamed.hash, size: streamed.size };
|
|
145
146
|
}
|
|
146
147
|
/**
|
|
147
148
|
* Insert the `fact` row, then — in the same transaction — its refs (fresh
|
|
@@ -167,7 +168,8 @@ export class PgFactStore {
|
|
|
167
168
|
* Returns `null`, with a warning log, for an external row the store cannot
|
|
168
169
|
* serve — an `external_url` that is not a `file:<shard>/<rest>` URL, no disk
|
|
169
170
|
* CAS configured (`disk_root` + `fs`), a failed disk read, or bytes that
|
|
170
|
-
* don't match the hash
|
|
171
|
+
* don't match the hash — and, with an error log, for a URL that names
|
|
172
|
+
* another fact's file. Never makes a network request.
|
|
171
173
|
*/
|
|
172
174
|
async get(hash) {
|
|
173
175
|
const row = await query_get_fact(this.#deps, hash);
|
|
@@ -181,10 +183,15 @@ export class PgFactStore {
|
|
|
181
183
|
}
|
|
182
184
|
// the URL is not echoed — the column is free text, so its shape and
|
|
183
185
|
// length are whatever the row's writer chose
|
|
184
|
-
|
|
186
|
+
const parsed = parse_file_fact_url(row.external_url);
|
|
187
|
+
if (!parsed) {
|
|
185
188
|
this.#log?.warn(`PgFactStore.get external_url for ${hash} is not a file:<shard>/<rest> URL; treating as not-found`);
|
|
186
189
|
return null;
|
|
187
190
|
}
|
|
191
|
+
if (!file_fact_url_addresses(parsed, hash)) {
|
|
192
|
+
this.#log?.error(`PgFactStore.get external_url for ${hash} addresses another fact's file; treating as not-found`);
|
|
193
|
+
return null;
|
|
194
|
+
}
|
|
188
195
|
if (this.#disk_root === undefined || this.#fs === undefined) {
|
|
189
196
|
this.#log?.warn(`PgFactStore.get ${hash} is disk-backed and no disk CAS is configured (disk_root + fs); treating as not-found`);
|
|
190
197
|
return null;
|
|
@@ -243,27 +250,6 @@ export class PgFactStore {
|
|
|
243
250
|
return query_delete_fact(this.#deps, hash);
|
|
244
251
|
}
|
|
245
252
|
}
|
|
246
|
-
/**
|
|
247
|
-
* Resolve refs for a `put` call: explicit `refs` win; otherwise auto-extract
|
|
248
|
-
* from JSON content; otherwise no refs.
|
|
249
|
-
*/
|
|
250
|
-
const resolve_refs = (bytes, options) => {
|
|
251
|
-
if (options.refs !== undefined)
|
|
252
|
-
return options.refs;
|
|
253
|
-
if (options.content_type !== 'application/json')
|
|
254
|
-
return [];
|
|
255
|
-
let value;
|
|
256
|
-
try {
|
|
257
|
-
value = JSON.parse(new TextDecoder().decode(bytes));
|
|
258
|
-
}
|
|
259
|
-
catch {
|
|
260
|
-
// Malformed JSON — caller mislabeled content_type. Fall back to no refs;
|
|
261
|
-
// the alternative (throwing) would surprise callers who set
|
|
262
|
-
// content_type advisorially.
|
|
263
|
-
return [];
|
|
264
|
-
}
|
|
265
|
-
return fact_hash_extract_refs(value);
|
|
266
|
-
};
|
|
267
253
|
/**
|
|
268
254
|
* Coerce whatever the driver returns for BYTEA into a `Uint8Array`.
|
|
269
255
|
*
|
|
@@ -11,9 +11,11 @@
|
|
|
11
11
|
*
|
|
12
12
|
* Centralizing the regex + the `fact_disk_path` split keeps the shape in one
|
|
13
13
|
* place: `PgFactStore`'s disk CAS (`db/fact_disk_storage.ts`), the
|
|
14
|
-
* `serve_fact_route` defense-in-depth
|
|
14
|
+
* `serve_fact_route` defense-in-depth checks (shape, then
|
|
15
|
+
* `file_fact_url_addresses`), and the disk read
|
|
15
16
|
* (`read_fact_bytes_from_disk`) all derive the layout here, so the write path
|
|
16
|
-
* and the read path can't drift. The TS twin of the Rust `fact_disk_path`
|
|
17
|
+
* and the read path can't drift. The TS twin of the Rust `fact_disk_path`
|
|
18
|
+
* (`fuz_fact`).
|
|
17
19
|
*
|
|
18
20
|
* Defense-in-depth: a `..` segment can't match (`.` isn't in `[0-9a-f]`),
|
|
19
21
|
* neither can absolute paths, query strings, or any non-hex character.
|
|
@@ -33,8 +35,6 @@ export declare const FILE_FACT_URL_PATTERN: RegExp;
|
|
|
33
35
|
*/
|
|
34
36
|
export declare const FileFactUrl: z.core.$ZodBranded<z.ZodString, "FileFactUrl", "out">;
|
|
35
37
|
export type FileFactUrl = z.infer<typeof FileFactUrl>;
|
|
36
|
-
/** Type guard. Useful when discriminating a `string | null` column. */
|
|
37
|
-
export declare const is_file_fact_url: (s: string) => s is FileFactUrl;
|
|
38
38
|
/**
|
|
39
39
|
* Split a `FactHash` into its on-disk `<shard>/<rest>` parts — the first 2
|
|
40
40
|
* hex chars of the digest (shard subdir) + the remaining 62. The single
|
|
@@ -56,6 +56,21 @@ export declare const parse_file_fact_url: (url: string) => {
|
|
|
56
56
|
shard: string;
|
|
57
57
|
rest: string;
|
|
58
58
|
} | null;
|
|
59
|
+
/**
|
|
60
|
+
* Whether a parsed `file:<shard>/<rest>` URL is the disk address of `hash` —
|
|
61
|
+
* `shard` + `rest` is its hex.
|
|
62
|
+
*
|
|
63
|
+
* The shape check (`parse_file_fact_url`) says a URL is safe to join onto the
|
|
64
|
+
* facts directory; this says it names `hash`'s own file. A reader checks both
|
|
65
|
+
* before it serves the file as `hash`'s bytes: a row whose `external_url` names
|
|
66
|
+
* another fact's file would otherwise serve those bytes under this hash, past
|
|
67
|
+
* this hash's admission. Twin of the Rust `ParsedFactUrl::addresses`
|
|
68
|
+
* (`fuz_fact`).
|
|
69
|
+
*/
|
|
70
|
+
export declare const file_fact_url_addresses: (parsed: {
|
|
71
|
+
shard: string;
|
|
72
|
+
rest: string;
|
|
73
|
+
}, hash: FactHash) => boolean;
|
|
59
74
|
/**
|
|
60
75
|
* Construct a canonical `file:<shard>/<rest>` URL. The writer side
|
|
61
76
|
* (`db/fact_disk_storage.ts`) assembles the shape from a freshly-computed
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"file_fact_url.d.ts","sourceRoot":"../src/lib/","sources":["../../src/lib/db/file_fact_url.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"file_fact_url.d.ts","sourceRoot":"../src/lib/","sources":["../../src/lib/db/file_fact_url.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,EAAoB,KAAK,QAAQ,EAAE,MAAM,kCAAkC,CAAC;AACnF,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,iEAAiE;AACjE,eAAO,MAAM,qBAAqB,QAAyC,CAAC;AAE5E;;;;GAIG;AACH,eAAO,MAAM,WAAW,uDAA+D,CAAC;AACxF,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC;AAEtD;;;;;;GAMG;AACH,eAAO,MAAM,cAAc,GAAI,MAAM,QAAQ,KAAG;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAG5E,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,mBAAmB,GAC/B,KAAK,MAAM,KACT;IAAE,GAAG,EAAE,WAAW,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAItD,CAAC;AAEF;;;;;;;;;;GAUG;AACH,eAAO,MAAM,uBAAuB,GACnC,QAAQ;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,EACvC,MAAM,QAAQ,KACZ,OAGF,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,kBAAkB,GAAI,OAAO,MAAM,EAAE,MAAM,MAAM,KAAG,WAC1B,CAAC"}
|
package/dist/db/file_fact_url.js
CHANGED
|
@@ -11,9 +11,11 @@
|
|
|
11
11
|
*
|
|
12
12
|
* Centralizing the regex + the `fact_disk_path` split keeps the shape in one
|
|
13
13
|
* place: `PgFactStore`'s disk CAS (`db/fact_disk_storage.ts`), the
|
|
14
|
-
* `serve_fact_route` defense-in-depth
|
|
14
|
+
* `serve_fact_route` defense-in-depth checks (shape, then
|
|
15
|
+
* `file_fact_url_addresses`), and the disk read
|
|
15
16
|
* (`read_fact_bytes_from_disk`) all derive the layout here, so the write path
|
|
16
|
-
* and the read path can't drift. The TS twin of the Rust `fact_disk_path`
|
|
17
|
+
* and the read path can't drift. The TS twin of the Rust `fact_disk_path`
|
|
18
|
+
* (`fuz_fact`).
|
|
17
19
|
*
|
|
18
20
|
* Defense-in-depth: a `..` segment can't match (`.` isn't in `[0-9a-f]`),
|
|
19
21
|
* neither can absolute paths, query strings, or any non-hex character.
|
|
@@ -32,8 +34,6 @@ export const FILE_FACT_URL_PATTERN = /^file:([0-9a-f]{2})\/([0-9a-f]{62})$/;
|
|
|
32
34
|
* string literals don't satisfy the brand.
|
|
33
35
|
*/
|
|
34
36
|
export const FileFactUrl = z.string().regex(FILE_FACT_URL_PATTERN).brand('FileFactUrl');
|
|
35
|
-
/** Type guard. Useful when discriminating a `string | null` column. */
|
|
36
|
-
export const is_file_fact_url = (s) => FILE_FACT_URL_PATTERN.test(s);
|
|
37
37
|
/**
|
|
38
38
|
* Split a `FactHash` into its on-disk `<shard>/<rest>` parts — the first 2
|
|
39
39
|
* hex chars of the digest (shard subdir) + the remaining 62. The single
|
|
@@ -56,6 +56,21 @@ export const parse_file_fact_url = (url) => {
|
|
|
56
56
|
return null;
|
|
57
57
|
return { url: url, shard: m[1], rest: m[2] };
|
|
58
58
|
};
|
|
59
|
+
/**
|
|
60
|
+
* Whether a parsed `file:<shard>/<rest>` URL is the disk address of `hash` —
|
|
61
|
+
* `shard` + `rest` is its hex.
|
|
62
|
+
*
|
|
63
|
+
* The shape check (`parse_file_fact_url`) says a URL is safe to join onto the
|
|
64
|
+
* facts directory; this says it names `hash`'s own file. A reader checks both
|
|
65
|
+
* before it serves the file as `hash`'s bytes: a row whose `external_url` names
|
|
66
|
+
* another fact's file would otherwise serve those bytes under this hash, past
|
|
67
|
+
* this hash's admission. Twin of the Rust `ParsedFactUrl::addresses`
|
|
68
|
+
* (`fuz_fact`).
|
|
69
|
+
*/
|
|
70
|
+
export const file_fact_url_addresses = (parsed, hash) => {
|
|
71
|
+
const { shard, rest } = fact_disk_path(hash);
|
|
72
|
+
return parsed.shard === shard && parsed.rest === rest;
|
|
73
|
+
};
|
|
59
74
|
/**
|
|
60
75
|
* Construct a canonical `file:<shard>/<rest>` URL. The writer side
|
|
61
76
|
* (`db/fact_disk_storage.ts`) assembles the shape from a freshly-computed
|
package/dist/server/env.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"env.d.ts","sourceRoot":"../src/lib/","sources":["../../src/lib/server/env.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,
|
|
1
|
+
{"version":3,"file":"env.d.ts","sourceRoot":"../src/lib/","sources":["../../src/lib/server/env.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAkB,KAAK,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAGlE;;;;;;;GAOG;AACH,eAAO,MAAM,aAAa;;;;;;;;;;;;;;;;;;;kBAqDxB,CAAC;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,aAAa,CAAC,CAAC;AAE1D;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC,EAAE,EAAE,IAAI,CAAC;IACT,OAAO,EAAE,OAAO,CAAC;IACjB,eAAe,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC/B,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC,EAAE,EAAE,KAAK,CAAC;IACV,KAAK,EAAE,wBAAwB,GAAG,qBAAqB,CAAC;IACxD,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CACtB;AAED,MAAM,MAAM,sBAAsB,GAAG,gBAAgB,GAAG,qBAAqB,CAAC;AAE9E;;;;;;;;;GASG;AACH,eAAO,MAAM,mBAAmB,GAAI,KAAK,aAAa,KAAG,sBAmBxD,CAAC"}
|
package/dist/server/env.js
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
* @module
|
|
10
10
|
*/
|
|
11
11
|
import { z } from 'zod';
|
|
12
|
-
import {
|
|
12
|
+
import { create_keyring } from '../auth/keyring.js';
|
|
13
13
|
import { format_allowed_origins_error, parse_allowed_origins } from '../http/origin.js';
|
|
14
14
|
/**
|
|
15
15
|
* Base Zod schema for server environment variables.
|
|
@@ -83,7 +83,7 @@ export const BaseServerEnv = z.strictObject({
|
|
|
83
83
|
* @returns `{ok: true, keyring, allowed_origins, bootstrap_token_path}` or `{ok: false, field, errors}`
|
|
84
84
|
*/
|
|
85
85
|
export const validate_server_env = (env) => {
|
|
86
|
-
const keyring_result =
|
|
86
|
+
const keyring_result = create_keyring(env.SECRET_FUZ_COOKIE_KEYS);
|
|
87
87
|
if (!keyring_result.ok) {
|
|
88
88
|
return { ok: false, field: 'SECRET_FUZ_COOKIE_KEYS', errors: keyring_result.errors };
|
|
89
89
|
}
|
|
@@ -113,13 +113,17 @@
|
|
|
113
113
|
*
|
|
114
114
|
* ## Defense-in-depth
|
|
115
115
|
*
|
|
116
|
-
* `external_url` is re-validated
|
|
117
|
-
*
|
|
118
|
-
*
|
|
116
|
+
* `external_url` is re-validated before it addresses the filesystem or an
|
|
117
|
+
* `X-Accel-Redirect`: it must have the `file:<shard>/<rest>` shape, and it must
|
|
118
|
+
* be the requested hash's own address (`file_fact_url_addresses`). `PgFactStore`
|
|
119
|
+
* writes only such URLs, but the column is free text and `query_put_fact`
|
|
119
120
|
* inserts what it is given, so a row written past the store — or a
|
|
120
121
|
* row-injection bug upstream — would otherwise hand nginx an
|
|
121
|
-
* attacker-controlled path
|
|
122
|
-
*
|
|
122
|
+
* attacker-controlled path, or serve another fact's bytes under this hash,
|
|
123
|
+
* behind this hash's admission. The serve path never re-hashes what it
|
|
124
|
+
* streams, so this binding is what ties the bytes to the name. Either refusal
|
|
125
|
+
* is a logged error and the masked 404, the same unavailable that
|
|
126
|
+
* `PgFactStore.get` reports as `null`. Twin of the Rust `fuz_fact_serving`.
|
|
123
127
|
*
|
|
124
128
|
* @module
|
|
125
129
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"serve_fact_route.d.ts","sourceRoot":"../src/lib/","sources":["../../src/lib/server/serve_fact_route.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"serve_fact_route.d.ts","sourceRoot":"../src/lib/","sources":["../../src/lib/server/serve_fact_route.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgIG;AAMH,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAetD,OAAO,EAAuC,KAAK,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAE5F,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAO/C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAoCjD,MAAM,WAAW,+BAA+B;IAC/C;;;;;;OAMG;IACH,IAAI,EAAE,OAAO,CAAC;IACd,kFAAkF;IAClF,SAAS,EAAE,MAAM,CAAC;IAClB;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,YAAY,CAAC;IACvB,GAAG,EAAE,MAAM,CAAC;CACZ;AA6HD;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,iCAAiC,GAC7C,SAAS,+BAA+B,KACtC,SAuDF,CAAC;AAEF;;;;;;;;;;;;;;;;;GAiBG;AACH,eAAO,MAAM,4BAA4B,GACxC,SAAS,+BAA+B,KACtC,SA2CF,CAAC"}
|
|
@@ -113,13 +113,17 @@
|
|
|
113
113
|
*
|
|
114
114
|
* ## Defense-in-depth
|
|
115
115
|
*
|
|
116
|
-
* `external_url` is re-validated
|
|
117
|
-
*
|
|
118
|
-
*
|
|
116
|
+
* `external_url` is re-validated before it addresses the filesystem or an
|
|
117
|
+
* `X-Accel-Redirect`: it must have the `file:<shard>/<rest>` shape, and it must
|
|
118
|
+
* be the requested hash's own address (`file_fact_url_addresses`). `PgFactStore`
|
|
119
|
+
* writes only such URLs, but the column is free text and `query_put_fact`
|
|
119
120
|
* inserts what it is given, so a row written past the store — or a
|
|
120
121
|
* row-injection bug upstream — would otherwise hand nginx an
|
|
121
|
-
* attacker-controlled path
|
|
122
|
-
*
|
|
122
|
+
* attacker-controlled path, or serve another fact's bytes under this hash,
|
|
123
|
+
* behind this hash's admission. The serve path never re-hashes what it
|
|
124
|
+
* streams, so this binding is what ties the bytes to the name. Either refusal
|
|
125
|
+
* is a logged error and the masked 404, the same unavailable that
|
|
126
|
+
* `PgFactStore.get` reports as `null`. Twin of the Rust `fuz_fact_serving`.
|
|
123
127
|
*
|
|
124
128
|
* @module
|
|
125
129
|
*/
|
|
@@ -139,7 +143,7 @@ import { query_cell_get } from '../db/cell_queries.js';
|
|
|
139
143
|
import { query_cell_grant_list_for_cell } from '../db/cell_grant_queries.js';
|
|
140
144
|
import { can_view_cell } from '../auth/cell_authorize.js';
|
|
141
145
|
import { cell_steward_admits } from '../auth/cell_steward.js';
|
|
142
|
-
import { parse_file_fact_url } from '../db/file_fact_url.js';
|
|
146
|
+
import { file_fact_url_addresses, parse_file_fact_url } from '../db/file_fact_url.js';
|
|
143
147
|
import { FACT_CACHE_CONTROL, FACT_SECURITY_HEADERS } from './fact_security_headers.js';
|
|
144
148
|
/**
|
|
145
149
|
* Path-param schema for the bare-hash route. Matching the canonical
|
|
@@ -208,12 +212,16 @@ const serve_fact_bytes = async (c, route, hash, config) => {
|
|
|
208
212
|
});
|
|
209
213
|
}
|
|
210
214
|
// External — defense-in-depth re-validate the URL before trusting it
|
|
211
|
-
// to address the filesystem
|
|
215
|
+
// to address the filesystem: its shape, then that it is this hash's file
|
|
212
216
|
const parsed = parse_file_fact_url(meta.external_url);
|
|
213
217
|
if (!parsed) {
|
|
214
218
|
log.error(`serve_fact: external_url for ${hash} is not a file:<shard>/<rest> URL; answering 404`);
|
|
215
219
|
return c.body(null, 404);
|
|
216
220
|
}
|
|
221
|
+
if (!file_fact_url_addresses(parsed, hash)) {
|
|
222
|
+
log.error(`serve_fact: external_url for ${hash} addresses another fact's file; answering 404`);
|
|
223
|
+
return c.body(null, 404);
|
|
224
|
+
}
|
|
217
225
|
const { shard, rest } = parsed;
|
|
218
226
|
if (x_accel !== undefined) {
|
|
219
227
|
// Production: hand off to nginx via the validated `internal;` facts location.
|