@fuzdev/fuz_app 0.119.0 → 0.121.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -61,9 +61,9 @@ export interface AppDeps {
61
61
  /**
62
62
  * Optional content-addressed byte store. Present only on backends that
63
63
  * serve binary content (facts) — minimal consumers leave it unset. The
64
- * consumer constructs a `PgFactStore` (`db/fact_store.ts`) wired to a
65
- * `file_fact_fetcher` (`server/file_fact_fetcher.ts`) at its own backend
66
- * assembly and assigns it here; `create_app_backend` stays facts-agnostic.
64
+ * consumer constructs a `PgFactStore` (`db/fact_store.ts`) over its facts
65
+ * directory (`disk_root` + `fs`) at its own backend assembly and assigns it
66
+ * here; `create_app_backend` stays facts-agnostic.
67
67
  */
68
68
  fact_store?: FactStore;
69
69
  }
package/dist/db/CLAUDE.md CHANGED
@@ -156,11 +156,16 @@ DO NOTHING`), `_put_fact_refs`, `_get_fact` / `_get_fact_meta` / `_has_fact`
156
156
  / `_select_for_delete` (a fact is orphan when no active `cell.refs` names it).
157
157
  - **`fact_store.ts`** — `PgFactStore implements FactStore` (the interface lives
158
158
  in `@fuzdev/fuz_util/fact_store.ts`): size-routed writes (embedded ≤
159
- `embedded_threshold` / disk CAS above it / `put_ref` for an externally-managed
160
- URL), JSON ref auto-extract, idempotent put, verify-on-read for external
161
- content via an injected `FactExternalFetcher`. With `disk_root` + `fs` (the
159
+ `embedded_threshold` / disk CAS above it), JSON ref auto-extract, idempotent
160
+ put, verify-on-read for disk-backed content. With `disk_root` + `fs` (the
162
161
  `runtime/*Deps`) configured, oversize `put` and the streaming `put_stream`
163
- write to the `<shard>/<rest>` disk CAS and the default fetcher reads from it.
162
+ write to the `<shard>/<rest>` disk CAS and `get` reads from it; without them
163
+ an oversize write throws (`put_stream` needs `fs` for a body of any size).
164
+ The disk CAS is the only place `get` reads external bytes from; the store
165
+ makes no network request. A row whose `external_url` is not a
166
+ `file:<shard>/<rest>` URL (the store never writes one), and any external row
167
+ on a store with no disk CAS, reads as `null` with a warning; the serve route
168
+ 404s the former.
164
169
  - **`file_fact_url.ts`** — the canonical `file:<shard>/<rest>` URL shape
165
170
  (`FileFactUrl` brand, `mint_file_fact_url` / `parse_file_fact_url` /
166
171
  `FILE_FACT_URL_PATTERN`) plus `fact_disk_path(hash) → {shard, rest}`, the
@@ -168,15 +173,15 @@ DO NOTHING`), `_put_fact_refs`, `_get_fact` / `_get_fact_meta` / `_has_fact`
168
173
  - **`fact_disk_storage.ts`** — the filesystem CAS over `runtime/{FsStream,FsWrite,FsRemove,FsRead}Deps`
169
174
  (not raw `node:fs`): `stream_fact_to_disk` (bounded-memory blake3+sha256 single
170
175
  pass, buffer→spill, fsync-then-atomic-rename, dedup-drop if the CAS path already
171
- exists), `write_fact_bytes_to_disk` (buffering twin), `create_disk_fact_fetcher`,
176
+ exists), `write_fact_bytes_to_disk` (buffering twin), `read_fact_bytes_from_disk`,
172
177
  and `sweep_orphan_temps` (reaps stale `.tmp` spills by mtime). The temp is
173
178
  `fsync`ed before the rename publishes it (twins the Rust `fuz_fact` §fsync
174
179
  posture: data-sync before rename, parent-dir fsync waived) — the serve path
175
180
  streams the file without re-hashing, so write-time durability is the guard.
176
181
  - **`fact_store_errors.ts`** — `PayloadTooLargeError` / `StorageFullError` (+
177
182
  `is_enospc_error`) thrown by `put_stream`, for a consumer route's 413 / 507.
178
- - The read-side fetcher + write/serve plumbing also live under `server/`
179
- (`file_fact_fetcher.ts`, `serve_fact_route.ts`).
183
+ - The HTTP serving plumbing lives under `server/` (`serve_fact_route.ts`,
184
+ `x_accel.ts`).
180
185
 
181
186
  ### Migration namespace order
182
187
 
@@ -1,5 +1,5 @@
1
1
  /**
2
- * Filesystem CAS for externally-stored fact bytes — the disk half of
2
+ * Filesystem CAS for fact bytes stored outside Postgres — the disk half of
3
3
  * `PgFactStore`, threaded over the injectable `runtime/*Deps` rather than raw
4
4
  * `node:fs`, so it runs unchanged under Node, Deno, and a mock runtime.
5
5
  *
@@ -31,7 +31,6 @@ import { type FactHash } from '@fuzdev/fuz_util/hash_schemas.ts';
31
31
  import type { Logger } from '@fuzdev/fuz_util/log.ts';
32
32
  import type { FsReadDeps, FsWriteDeps, FsStreamDeps, FsRemoveDeps } from '../runtime/deps.js';
33
33
  import { type FileFactUrl } from './file_fact_url.js';
34
- import type { FactExternalFetcher } from './fact_store.js';
35
34
  /** Subdirectory under `facts_dir` for in-flight atomic temp files. */
36
35
  export declare const FACT_TMP_DIRNAME = ".tmp";
37
36
  /** Default age (1 hour) past which a `.tmp/*` file is considered orphaned. */
@@ -41,7 +40,7 @@ export declare const FACT_TMP_ORPHAN_MAX_AGE_MS: number;
41
40
  * full `RuntimeDeps` (Node or Deno) satisfies this; each function below picks
42
41
  * the narrow subset it actually uses.
43
42
  */
44
- export type FactDiskStorageDeps = Pick<FsReadDeps, 'stat' | 'readdir' | 'read_file'> & Pick<FsWriteDeps, 'mkdir' | 'rename' | 'write_file' | 'fsync'> & Pick<FsStreamDeps, 'write_file_stream' | 'read_file_stream'> & Pick<FsRemoveDeps, 'remove'>;
43
+ export type FactDiskStorageDeps = Pick<FsReadDeps, 'stat' | 'readdir' | 'read_file'> & Pick<FsWriteDeps, 'mkdir' | 'rename' | 'write_file' | 'fsync'> & Pick<FsStreamDeps, 'write_file_stream'> & Pick<FsRemoveDeps, 'remove'>;
45
44
  /**
46
45
  * Where a streamed body landed — `embedded` carries the in-memory bytes (under
47
46
  * the embedded threshold, bound for the PG `fact.bytes` column); `disk` means
@@ -100,16 +99,19 @@ export declare const stream_fact_to_disk: (deps: Pick<FactDiskStorageDeps, "mkdi
100
99
  */
101
100
  export declare const write_fact_bytes_to_disk: (deps: Pick<FactDiskStorageDeps, "mkdir" | "rename" | "remove" | "write_file" | "fsync" | "stat">, facts_dir: string, hash: FactHash, bytes: Uint8Array) => Promise<FileFactUrl>;
102
101
  /**
103
- * `FactExternalFetcher` reading from the `<facts_dir>/<shard>/<rest>` layout the
104
- * writers above produce, over the injected `*Deps`. Does NOT verify hash content
105
- * — `PgFactStore.get` calls `fact_hash_verify(hash, bytes)` after the fetch and
106
- * returns `null` on mismatch.
102
+ * Read a disk-backed fact's bytes from the `<facts_dir>/<shard>/<rest>` layout
103
+ * the writers above produce, addressed by the row's `file:<shard>/<rest>`
104
+ * `external_url`. The only read of external bytes `PgFactStore.get` performs.
105
+ * Does NOT verify hash content — `PgFactStore.get` calls
106
+ * `fact_hash_verify(hash, bytes)` on the result and returns `null` on mismatch.
107
107
  *
108
108
  * Defense at the read seam is the `FILE_FACT_URL_PATTERN` regex (via
109
109
  * `parse_file_fact_url`) — `..` segments, foreign schemes, and non-hex chars
110
110
  * fail before any disk access.
111
+ *
112
+ * @throws Error when `url` is not a `file:<shard>/<rest>` URL, or when the read fails
111
113
  */
112
- export declare const create_disk_fact_fetcher: (deps: Pick<FactDiskStorageDeps, "read_file" | "read_file_stream">, facts_dir: string) => FactExternalFetcher;
114
+ export declare const read_fact_bytes_from_disk: (deps: Pick<FactDiskStorageDeps, "read_file">, facts_dir: string, url: string) => Promise<Uint8Array>;
113
115
  /**
114
116
  * Reap stale temp files left under `<facts_dir>/.tmp/` by a hard crash (SIGKILL
115
117
  * / OOM / host crash) mid-write — the `finally` cleanup in the writers above
@@ -1 +1 @@
1
- {"version":3,"file":"fact_disk_storage.d.ts","sourceRoot":"../src/lib/","sources":["../../src/lib/db/fact_disk_storage.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AASH,OAAO,EAAoB,KAAK,QAAQ,EAAE,MAAM,kCAAkC,CAAC;AACnF,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAEtD,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAE9F,OAAO,EAIN,KAAK,WAAW,EAChB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAG3D,sEAAsE;AACtE,eAAO,MAAM,gBAAgB,SAAS,CAAC;AAEvC,8EAA8E;AAC9E,eAAO,MAAM,0BAA0B,QAAiB,CAAC;AAEzD;;;;GAIG;AACH,MAAM,MAAM,mBAAmB,GAAG,IAAI,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,GAAG,WAAW,CAAC,GACnF,IAAI,CAAC,WAAW,EAAE,OAAO,GAAG,QAAQ,GAAG,YAAY,GAAG,OAAO,CAAC,GAC9D,IAAI,CAAC,YAAY,EAAE,mBAAmB,GAAG,kBAAkB,CAAC,GAC5D,IAAI,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;AAE9B;;;;;GAKG;AACH,MAAM,MAAM,eAAe,GAC1B;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,KAAK,EAAE,UAAU,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,WAAW,CAAA;CAAE,CAAC;AAEvF;;;;GAIG;AACH,MAAM,WAAW,sBAAsB;IACtC,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,eAAe,CAAC;CAC3B;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,eAAO,MAAM,mBAAmB,GAC/B,MAAM,IAAI,CACT,mBAAmB,EACnB,OAAO,GAAG,QAAQ,GAAG,QAAQ,GAAG,mBAAmB,GAAG,OAAO,GAAG,MAAM,CACtE,EACD,WAAW,MAAM,GAAG,SAAS,EAC7B,QAAQ,cAAc,CAAC,UAAU,CAAC,EAClC,WAAW,MAAM,EACjB,oBAAoB,MAAM,KACxB,OAAO,CAAC,sBAAsB,CA8GhC,CAAC;AAEF;;;;;;;;GAQG;AACH,eAAO,MAAM,wBAAwB,GACpC,MAAM,IAAI,CAAC,mBAAmB,EAAE,OAAO,GAAG,QAAQ,GAAG,QAAQ,GAAG,YAAY,GAAG,OAAO,GAAG,MAAM,CAAC,EAChG,WAAW,MAAM,EACjB,MAAM,QAAQ,EACd,OAAO,UAAU,KACf,OAAO,CAAC,WAAW,CAgBrB,CAAC;AAEF;;;;;;;;;GASG;AACH,eAAO,MAAM,wBAAwB,GACpC,MAAM,IAAI,CAAC,mBAAmB,EAAE,WAAW,GAAG,kBAAkB,CAAC,EACjE,WAAW,MAAM,KACf,mBAWF,CAAC;AAEF;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,kBAAkB,GAC9B,MAAM,IAAI,CAAC,mBAAmB,EAAE,SAAS,GAAG,MAAM,GAAG,QAAQ,CAAC,EAC9D,WAAW,MAAM,EACjB,UAAU;IAAE,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,KAC3D,OAAO,CAAC,MAAM,CA0BhB,CAAC"}
1
+ {"version":3,"file":"fact_disk_storage.d.ts","sourceRoot":"../src/lib/","sources":["../../src/lib/db/fact_disk_storage.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AASH,OAAO,EAAoB,KAAK,QAAQ,EAAE,MAAM,kCAAkC,CAAC;AACnF,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAEtD,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAE9F,OAAO,EAIN,KAAK,WAAW,EAChB,MAAM,oBAAoB,CAAC;AAG5B,sEAAsE;AACtE,eAAO,MAAM,gBAAgB,SAAS,CAAC;AAEvC,8EAA8E;AAC9E,eAAO,MAAM,0BAA0B,QAAiB,CAAC;AAEzD;;;;GAIG;AACH,MAAM,MAAM,mBAAmB,GAAG,IAAI,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,GAAG,WAAW,CAAC,GACnF,IAAI,CAAC,WAAW,EAAE,OAAO,GAAG,QAAQ,GAAG,YAAY,GAAG,OAAO,CAAC,GAC9D,IAAI,CAAC,YAAY,EAAE,mBAAmB,CAAC,GACvC,IAAI,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;AAE9B;;;;;GAKG;AACH,MAAM,MAAM,eAAe,GAC1B;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,KAAK,EAAE,UAAU,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,WAAW,CAAA;CAAE,CAAC;AAEvF;;;;GAIG;AACH,MAAM,WAAW,sBAAsB;IACtC,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,eAAe,CAAC;CAC3B;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,eAAO,MAAM,mBAAmB,GAC/B,MAAM,IAAI,CACT,mBAAmB,EACnB,OAAO,GAAG,QAAQ,GAAG,QAAQ,GAAG,mBAAmB,GAAG,OAAO,GAAG,MAAM,CACtE,EACD,WAAW,MAAM,GAAG,SAAS,EAC7B,QAAQ,cAAc,CAAC,UAAU,CAAC,EAClC,WAAW,MAAM,EACjB,oBAAoB,MAAM,KACxB,OAAO,CAAC,sBAAsB,CA8GhC,CAAC;AAEF;;;;;;;;GAQG;AACH,eAAO,MAAM,wBAAwB,GACpC,MAAM,IAAI,CAAC,mBAAmB,EAAE,OAAO,GAAG,QAAQ,GAAG,QAAQ,GAAG,YAAY,GAAG,OAAO,GAAG,MAAM,CAAC,EAChG,WAAW,MAAM,EACjB,MAAM,QAAQ,EACd,OAAO,UAAU,KACf,OAAO,CAAC,WAAW,CAgBrB,CAAC;AAEF;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,yBAAyB,GACrC,MAAM,IAAI,CAAC,mBAAmB,EAAE,WAAW,CAAC,EAC5C,WAAW,MAAM,EACjB,KAAK,MAAM,KACT,OAAO,CAAC,UAAU,CAIpB,CAAC;AAEF;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,kBAAkB,GAC9B,MAAM,IAAI,CAAC,mBAAmB,EAAE,SAAS,GAAG,MAAM,GAAG,QAAQ,CAAC,EAC9D,WAAW,MAAM,EACjB,UAAU;IAAE,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,KAC3D,OAAO,CAAC,MAAM,CA0BhB,CAAC"}
@@ -1,5 +1,5 @@
1
1
  /**
2
- * Filesystem CAS for externally-stored fact bytes — the disk half of
2
+ * Filesystem CAS for fact bytes stored outside Postgres — the disk half of
3
3
  * `PgFactStore`, threaded over the injectable `runtime/*Deps` rather than raw
4
4
  * `node:fs`, so it runs unchanged under Node, Deno, and a mock runtime.
5
5
  *
@@ -204,27 +204,23 @@ export const write_fact_bytes_to_disk = async (deps, facts_dir, hash, bytes) =>
204
204
  return mint_file_fact_url(shard, rest);
205
205
  };
206
206
  /**
207
- * `FactExternalFetcher` reading from the `<facts_dir>/<shard>/<rest>` layout the
208
- * writers above produce, over the injected `*Deps`. Does NOT verify hash content
209
- * — `PgFactStore.get` calls `fact_hash_verify(hash, bytes)` after the fetch and
210
- * returns `null` on mismatch.
207
+ * Read a disk-backed fact's bytes from the `<facts_dir>/<shard>/<rest>` layout
208
+ * the writers above produce, addressed by the row's `file:<shard>/<rest>`
209
+ * `external_url`. The only read of external bytes `PgFactStore.get` performs.
210
+ * Does NOT verify hash content — `PgFactStore.get` calls
211
+ * `fact_hash_verify(hash, bytes)` on the result and returns `null` on mismatch.
211
212
  *
212
213
  * Defense at the read seam is the `FILE_FACT_URL_PATTERN` regex (via
213
214
  * `parse_file_fact_url`) — `..` segments, foreign schemes, and non-hex chars
214
215
  * fail before any disk access.
216
+ *
217
+ * @throws Error when `url` is not a `file:<shard>/<rest>` URL, or when the read fails
215
218
  */
216
- export const create_disk_fact_fetcher = (deps, facts_dir) => {
217
- const resolve_path = (url) => {
218
- const parsed = parse_file_fact_url(url);
219
- if (!parsed)
220
- throw new Error(`invalid file fact url: ${url}`);
221
- return join(facts_dir, parsed.shard, parsed.rest);
222
- };
223
- return {
224
- fetch_bytes: (url) => deps.read_file(resolve_path(url)),
225
- // `async` funnels a synchronous `resolve_path` throw into a rejection.
226
- fetch_stream: async (url) => deps.read_file_stream(resolve_path(url))
227
- };
219
+ export const read_fact_bytes_from_disk = async (deps, facts_dir, url) => {
220
+ const parsed = parse_file_fact_url(url);
221
+ if (!parsed)
222
+ throw new Error('invalid file fact url');
223
+ return deps.read_file(join(facts_dir, parsed.shard, parsed.rest));
228
224
  };
229
225
  /**
230
226
  * Reap stale temp files left under `<facts_dir>/.tmp/` by a hard crash (SIGKILL
@@ -4,14 +4,13 @@
4
4
  * Wraps the raw queries in `db/fact_queries.ts` with the lifecycle the
5
5
  * `FactStore` interface promises:
6
6
  *
7
- * - sync hash on `put`, stream hash on `put_ref` (counting bytes against
8
- * the caller-supplied `size`)
7
+ * - sync hash on `put`; `put_stream` hashes as it streams
9
8
  * - idempotent insert (`ON CONFLICT DO NOTHING` in the queries layer)
10
9
  * - JSON ref auto-extraction when `content_type` signals JSON and the
11
10
  * caller didn't pass an explicit `refs` array
12
- * - verify-on-read for external content; embedded reads skip verify
11
+ * - verify-on-read for disk-backed content; embedded reads skip verify
13
12
  * because PG storage IS the hash table
14
- * - mismatched external bytes return `null` + log warning (treat as
13
+ * - mismatched disk bytes return `null` + log warning (treat as
15
14
  * unavailable; GC / repair is a separate concern)
16
15
  *
17
16
  * Embedded vs disk split: writes route by size. Bytes `<= embedded_threshold`
@@ -19,10 +18,20 @@
19
18
  * `<facts_dir>/<shard>/<rest>` (`db/fact_disk_storage.ts`) and the row records a
20
19
  * `file:<shard>/<rest>` `external_url`. `put` takes fully-buffered bytes;
21
20
  * `put_stream` is the bounded-memory streaming twin (hash BLAKE3 + SHA-256 in
22
- * one pass, spill past the threshold, enforce `max_bytes` / `ENOSPC`). Both need
23
- * `disk_root` + `fs` (the `runtime/*Deps`) configured for the over-threshold
24
- * path; without them, an oversize `put` throws and the caller must `put_ref`
25
- * against an externally-managed URL (federation / stub-fetcher tests).
21
+ * one pass, spill past the threshold, enforce `max_bytes` / `ENOSPC`). `put`
22
+ * needs `disk_root` + `fs` (the `runtime/*Deps`) for the over-threshold path
23
+ * only: without them the store has nowhere to put bytes it cannot embed, and an
24
+ * oversize `put` throws. `put_stream` needs `fs` for a body of any size — with
25
+ * none it throws before reading the stream — and `disk_root` for the
26
+ * over-threshold spill: with none, a body that passes the threshold throws
27
+ * `PayloadTooLargeError`.
28
+ *
29
+ * Every `external_url` the store writes is a `file:<shard>/<rest>` URL into its
30
+ * own disk CAS, and that CAS is the only place `get` reads external bytes from
31
+ * — the store makes no network request. The column is free text
32
+ * (`query_put_fact` inserts what it is given), so `get` treats a row whose
33
+ * `external_url` has any other shape, and any external row on a store with no
34
+ * disk CAS configured, as unavailable: a warning and `null`.
26
35
  *
27
36
  * @module
28
37
  */
@@ -33,13 +42,6 @@ import type { FactMeta, FactPutOptions, FactStore, PutStreamOutcome } from '@fuz
33
42
  import { type FactDiskStorageDeps } from './fact_disk_storage.js';
34
43
  /** Default embedded-vs-referenced cutoff (1 MiB). */
35
44
  export declare const FACT_EMBEDDED_THRESHOLD_DEFAULT: number;
36
- /** Fetcher abstraction so tests can stub external URL retrieval. */
37
- export interface FactExternalFetcher {
38
- fetch_stream: (url: string) => Promise<ReadableStream<Uint8Array>>;
39
- fetch_bytes: (url: string) => Promise<Uint8Array>;
40
- }
41
- /** Default fetcher backed by `globalThis.fetch`. */
42
- export declare const create_default_fetcher: () => FactExternalFetcher;
43
45
  /**
44
46
  * Construction-time deps for `PgFactStore`.
45
47
  *
@@ -52,18 +54,19 @@ export declare const create_default_fetcher: () => FactExternalFetcher;
52
54
  *
53
55
  * `disk_root` is the facts directory backing the `<shard>/<rest>` disk CAS;
54
56
  * `fs` supplies the filesystem capabilities (a `RuntimeDeps` satisfies it).
55
- * When both are set, oversize `put` + `put_stream` write to disk and the
56
- * default `fetcher` reads from it. When unset, oversize `put`/`put_stream`
57
- * spill throws and reads fall back to the `globalThis.fetch`-backed default
58
- * fetcher (or an injected stub). `log` is optional — the only call site is the
59
- * verify-mismatch warning path.
57
+ * When both are set, oversize `put` + `put_stream` write to disk and `get`
58
+ * reads disk-backed facts from it. When either is unset the store writes
59
+ * embedded rows only and `get` returns `null` for an external row: an oversize
60
+ * `put` throws; `put_stream` throws for a body of any size when `fs` is unset,
61
+ * and with `fs` but no `disk_root` throws `PayloadTooLargeError` for a body
62
+ * that passes the threshold. `log` is optional — its call sites are the
63
+ * warnings on the paths where `get` returns `null` for an external row.
60
64
  */
61
65
  export interface PgFactStoreDeps {
62
66
  deps: QueryDeps;
63
67
  embedded_threshold?: number;
64
68
  disk_root?: string;
65
69
  fs?: FactDiskStorageDeps;
66
- fetcher?: FactExternalFetcher;
67
70
  log?: Logger;
68
71
  }
69
72
  /**
@@ -77,8 +80,8 @@ export declare class PgFactStore implements FactStore {
77
80
  * Store fully-buffered bytes, routing by size: `<= embedded_threshold` into
78
81
  * the PG `bytes` column; larger into the disk CAS (when `disk_root` + `fs`
79
82
  * are configured) at `<facts_dir>/<shard>/<rest>` with a `file:` URL. Oversize
80
- * without a disk root throws so the caller routes it through `put_ref`
81
- * explicitly. Idempotent — `ON CONFLICT DO NOTHING` + content-addressed disk
83
+ * without a disk root throws: the store has nowhere to put bytes it cannot
84
+ * embed. Idempotent — `ON CONFLICT DO NOTHING` + content-addressed disk
82
85
  * filenames make a re-write a no-op.
83
86
  */
84
87
  put(bytes: Uint8Array, options?: FactPutOptions): Promise<FactHash>;
@@ -101,16 +104,14 @@ export declare class PgFactStore implements FactStore {
101
104
  */
102
105
  put_stream(stream: ReadableStream<Uint8Array>, max_bytes: number, options?: FactPutOptions): Promise<PutStreamOutcome>;
103
106
  /**
104
- * Stream-hash external content and record `(hash, external_url, size)`.
105
- * Throws when the streamed byte count disagrees with the caller's
106
- * declared `size` — a size mismatch usually means the upload was
107
- * truncated or the URL points at the wrong content.
108
- */
109
- put_ref(url: string, size: number, options?: FactPutOptions): Promise<FactHash>;
110
- /**
111
- * Retrieve bytes. Embedded reads return PG bytes directly; external
112
- * reads fetch + verify and return `null` (with a warning log) when
113
- * the bytes don't match the stored hash.
107
+ * Retrieve bytes. Embedded reads return PG bytes directly. An external
108
+ * read comes from the disk CAS and nowhere else: the bytes at the row's
109
+ * `file:<shard>/<rest>` URL are read and verified against the hash.
110
+ *
111
+ * Returns `null`, with a warning log, for an external row the store cannot
112
+ * serve — an `external_url` that is not a `file:<shard>/<rest>` URL, no disk
113
+ * CAS configured (`disk_root` + `fs`), a failed disk read, or bytes that
114
+ * don't match the hash. Never makes a network request.
114
115
  */
115
116
  get(hash: FactHash): Promise<Uint8Array | null>;
116
117
  has(hash: FactHash): Promise<boolean>;
@@ -126,11 +127,11 @@ export declare class PgFactStore implements FactStore {
126
127
  * NOT verify the fact is unreferenced — that policy lives one layer
127
128
  * up (the orphan-fact admin surface in the consumer; a future GC walker).
128
129
  *
129
- * External-URL unlink is the caller's responsibility — the store
130
- * doesn't know how to resolve `file:` / `s3:` / etc. URLs to a
131
- * deletable handle. Caller iterates the returned `external_url`
132
- * (when non-null) and dispatches to the appropriate cleanup
133
- * routine. Mirrors the read-side `FactExternalFetcher` split.
130
+ * The disk file of a disk-backed fact is NOT unlinked — that is the
131
+ * caller's responsibility, using the returned `external_url` (when
132
+ * non-null): a `file:<shard>/<rest>` URL names
133
+ * `<disk_root>/<shard>/<rest>`. The value is the row's free text — parse
134
+ * it with `parse_file_fact_url` before joining a path.
134
135
  *
135
136
  * @returns `{size, external_url}` for the deleted row, or `null` if
136
137
  * no row matched the hash.
@@ -1 +1 @@
1
- {"version":3,"file":"fact_store.d.ts","sourceRoot":"../src/lib/","sources":["../../src/lib/db/fact_store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEjD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAQtD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,kCAAkC,CAAC;AACjE,OAAO,KAAK,EACX,QAAQ,EACR,cAAc,EACd,SAAS,EACT,gBAAgB,EAChB,MAAM,gCAAgC,CAAC;AAWxC,OAAO,EAIN,KAAK,mBAAmB,EACxB,MAAM,wBAAwB,CAAC;AAEhC,qDAAqD;AACrD,eAAO,MAAM,+BAA+B,QAAc,CAAC;AAE3D,oEAAoE;AACpE,MAAM,WAAW,mBAAmB;IACnC,YAAY,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC,CAAC;IACnE,WAAW,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,UAAU,CAAC,CAAC;CAClD;AAED,oDAAoD;AACpD,eAAO,MAAM,sBAAsB,QAAO,mBAkBxC,CAAC;AAEH;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,WAAW,eAAe;IAC/B,IAAI,EAAE,SAAS,CAAC;IAChB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,EAAE,CAAC,EAAE,mBAAmB,CAAC;IACzB,OAAO,CAAC,EAAE,mBAAmB,CAAC;IAC9B,GAAG,CAAC,EAAE,MAAM,CAAC;CACb;AAED;;;GAGG;AACH,qBAAa,WAAY,YAAW,SAAS;;gBAQhC,OAAO,EAAE,eAAe;IAapC;;;;;;;OAOG;IACG,GAAG,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,QAAQ,CAAC;IA+BzE;;;;;;;;;;;;;;;;OAgBG;IACG,UAAU,CACf,MAAM,EAAE,cAAc,CAAC,UAAU,CAAC,EAClC,SAAS,EAAE,MAAM,EACjB,OAAO,CAAC,EAAE,cAAc,GACtB,OAAO,CAAC,gBAAgB,CAAC;IA6B5B;;;;;OAKG;IACG,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,QAAQ,CAAC;IAqBrF;;;;OAIG;IACG,GAAG,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IA4B/C,GAAG,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC;IAIrC,QAAQ,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;IAWlD,QAAQ,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAIxD;;;;;;;;;;;;;;;;;;OAkBG;IACG,MAAM,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI,CAAC;CAG3F"}
1
+ {"version":3,"file":"fact_store.d.ts","sourceRoot":"../src/lib/","sources":["../../src/lib/db/fact_store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEjD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAOtD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,kCAAkC,CAAC;AACjE,OAAO,KAAK,EACX,QAAQ,EACR,cAAc,EACd,SAAS,EACT,gBAAgB,EAChB,MAAM,gCAAgC,CAAC;AAYxC,OAAO,EAIN,KAAK,mBAAmB,EACxB,MAAM,wBAAwB,CAAC;AAEhC,qDAAqD;AACrD,eAAO,MAAM,+BAA+B,QAAc,CAAC;AAE3D;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,WAAW,eAAe;IAC/B,IAAI,EAAE,SAAS,CAAC;IAChB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,EAAE,CAAC,EAAE,mBAAmB,CAAC;IACzB,GAAG,CAAC,EAAE,MAAM,CAAC;CACb;AAED;;;GAGG;AACH,qBAAa,WAAY,YAAW,SAAS;;gBAOhC,OAAO,EAAE,eAAe;IAQpC;;;;;;;OAOG;IACG,GAAG,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,QAAQ,CAAC;IA+BzE;;;;;;;;;;;;;;;;OAgBG;IACG,UAAU,CACf,MAAM,EAAE,cAAc,CAAC,UAAU,CAAC,EAClC,SAAS,EAAE,MAAM,EACjB,OAAO,CAAC,EAAE,cAAc,GACtB,OAAO,CAAC,gBAAgB,CAAC;IA6B5B;;;;;;;;;OASG;IACG,GAAG,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IA0C/C,GAAG,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC;IAIrC,QAAQ,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;IAWlD,QAAQ,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAIxD;;;;;;;;;;;;;;;;;;OAkBG;IACG,MAAM,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI,CAAC;CAG3F"}
@@ -4,14 +4,13 @@
4
4
  * Wraps the raw queries in `db/fact_queries.ts` with the lifecycle the
5
5
  * `FactStore` interface promises:
6
6
  *
7
- * - sync hash on `put`, stream hash on `put_ref` (counting bytes against
8
- * the caller-supplied `size`)
7
+ * - sync hash on `put`; `put_stream` hashes as it streams
9
8
  * - idempotent insert (`ON CONFLICT DO NOTHING` in the queries layer)
10
9
  * - JSON ref auto-extraction when `content_type` signals JSON and the
11
10
  * caller didn't pass an explicit `refs` array
12
- * - verify-on-read for external content; embedded reads skip verify
11
+ * - verify-on-read for disk-backed content; embedded reads skip verify
13
12
  * because PG storage IS the hash table
14
- * - mismatched external bytes return `null` + log warning (treat as
13
+ * - mismatched disk bytes return `null` + log warning (treat as
15
14
  * unavailable; GC / repair is a separate concern)
16
15
  *
17
16
  * Embedded vs disk split: writes route by size. Bytes `<= embedded_threshold`
@@ -19,39 +18,30 @@
19
18
  * `<facts_dir>/<shard>/<rest>` (`db/fact_disk_storage.ts`) and the row records a
20
19
  * `file:<shard>/<rest>` `external_url`. `put` takes fully-buffered bytes;
21
20
  * `put_stream` is the bounded-memory streaming twin (hash BLAKE3 + SHA-256 in
22
- * one pass, spill past the threshold, enforce `max_bytes` / `ENOSPC`). Both need
23
- * `disk_root` + `fs` (the `runtime/*Deps`) configured for the over-threshold
24
- * path; without them, an oversize `put` throws and the caller must `put_ref`
25
- * against an externally-managed URL (federation / stub-fetcher tests).
21
+ * one pass, spill past the threshold, enforce `max_bytes` / `ENOSPC`). `put`
22
+ * needs `disk_root` + `fs` (the `runtime/*Deps`) for the over-threshold path
23
+ * only: without them the store has nowhere to put bytes it cannot embed, and an
24
+ * oversize `put` throws. `put_stream` needs `fs` for a body of any size — with
25
+ * none it throws before reading the stream — and `disk_root` for the
26
+ * over-threshold spill: with none, a body that passes the threshold throws
27
+ * `PayloadTooLargeError`.
28
+ *
29
+ * Every `external_url` the store writes is a `file:<shard>/<rest>` URL into its
30
+ * own disk CAS, and that CAS is the only place `get` reads external bytes from
31
+ * — the store makes no network request. The column is free text
32
+ * (`query_put_fact` inserts what it is given), so `get` treats a row whose
33
+ * `external_url` has any other shape, and any external row on a store with no
34
+ * disk CAS configured, as unavailable: a warning and `null`.
26
35
  *
27
36
  * @module
28
37
  */
29
38
  import { to_error_message } from '@fuzdev/fuz_util/error.ts';
30
- import { fact_hash_bytes, fact_hash_stream, fact_hash_verify, fact_hash_extract_refs } from '@fuzdev/fuz_util/fact_hash.ts';
39
+ import { fact_hash_bytes, fact_hash_verify, fact_hash_extract_refs } from '@fuzdev/fuz_util/fact_hash.ts';
31
40
  import { query_delete_fact, query_get_fact, query_get_fact_meta, query_get_fact_refs, query_has_fact, query_put_fact, query_put_fact_refs } from './fact_queries.js';
32
- import { create_disk_fact_fetcher, stream_fact_to_disk, write_fact_bytes_to_disk } from './fact_disk_storage.js';
41
+ import { is_file_fact_url } from './file_fact_url.js';
42
+ import { read_fact_bytes_from_disk, stream_fact_to_disk, write_fact_bytes_to_disk } from './fact_disk_storage.js';
33
43
  /** Default embedded-vs-referenced cutoff (1 MiB). */
34
44
  export const FACT_EMBEDDED_THRESHOLD_DEFAULT = 1024 * 1024;
35
- /** Default fetcher backed by `globalThis.fetch`. */
36
- export const create_default_fetcher = () => ({
37
- fetch_stream: async (url) => {
38
- const response = await fetch(url);
39
- if (!response.ok) {
40
- throw new Error(`fact fetch failed: ${response.status} ${url}`);
41
- }
42
- if (!response.body) {
43
- throw new Error(`fact fetch returned no body: ${url}`);
44
- }
45
- return response.body;
46
- },
47
- fetch_bytes: async (url) => {
48
- const response = await fetch(url);
49
- if (!response.ok) {
50
- throw new Error(`fact fetch failed: ${response.status} ${url}`);
51
- }
52
- return new Uint8Array(await response.arrayBuffer());
53
- }
54
- });
55
45
  /**
56
46
  * PG-backed `FactStore`. Delegates to `db/fact_queries.ts` for I/O and adds
57
47
  * the lifecycle layer described in the module doc.
@@ -61,26 +51,20 @@ export class PgFactStore {
61
51
  #embedded_threshold;
62
52
  #disk_root;
63
53
  #fs;
64
- #fetcher;
65
54
  #log;
66
55
  constructor(options) {
67
56
  this.#deps = options.deps;
68
57
  this.#embedded_threshold = options.embedded_threshold ?? FACT_EMBEDDED_THRESHOLD_DEFAULT;
69
58
  this.#disk_root = options.disk_root;
70
59
  this.#fs = options.fs;
71
- this.#fetcher =
72
- options.fetcher ??
73
- (options.disk_root !== undefined && options.fs !== undefined
74
- ? create_disk_fact_fetcher(options.fs, options.disk_root)
75
- : create_default_fetcher());
76
60
  this.#log = options.log;
77
61
  }
78
62
  /**
79
63
  * Store fully-buffered bytes, routing by size: `<= embedded_threshold` into
80
64
  * the PG `bytes` column; larger into the disk CAS (when `disk_root` + `fs`
81
65
  * are configured) at `<facts_dir>/<shard>/<rest>` with a `file:` URL. Oversize
82
- * without a disk root throws so the caller routes it through `put_ref`
83
- * explicitly. Idempotent — `ON CONFLICT DO NOTHING` + content-addressed disk
66
+ * without a disk root throws: the store has nowhere to put bytes it cannot
67
+ * embed. Idempotent — `ON CONFLICT DO NOTHING` + content-addressed disk
84
68
  * filenames make a re-write a no-op.
85
69
  */
86
70
  async put(bytes, options) {
@@ -89,7 +73,7 @@ export class PgFactStore {
89
73
  let row_external_url;
90
74
  if (bytes.length > this.#embedded_threshold) {
91
75
  if (this.#disk_root === undefined || this.#fs === undefined) {
92
- throw new Error(`fact bytes exceed embedded threshold (${bytes.length} > ${this.#embedded_threshold}); configure disk_root or use put_ref for external storage`);
76
+ throw new Error(`fact bytes exceed embedded threshold (${bytes.length} > ${this.#embedded_threshold}) and no disk CAS is configured; set disk_root and fs to store them on disk`);
93
77
  }
94
78
  row_bytes = null;
95
79
  row_external_url = await write_fact_bytes_to_disk(this.#fs, this.#disk_root, hash, bytes);
@@ -147,33 +131,14 @@ export class PgFactStore {
147
131
  return { hash: streamed.hash, sha256: streamed.sha256, size: streamed.size };
148
132
  }
149
133
  /**
150
- * Stream-hash external content and record `(hash, external_url, size)`.
151
- * Throws when the streamed byte count disagrees with the caller's
152
- * declared `size` — a size mismatch usually means the upload was
153
- * truncated or the URL points at the wrong content.
154
- */
155
- async put_ref(url, size, options) {
156
- const stream = await this.#fetcher.fetch_stream(url);
157
- const { hash, byte_count } = await hash_counted_stream(stream);
158
- if (byte_count !== size) {
159
- throw new Error(`fact size mismatch for ${url}: caller declared ${size}, streamed ${byte_count}`);
160
- }
161
- const inserted = await query_put_fact(this.#deps, {
162
- hash,
163
- bytes: null,
164
- external_url: url,
165
- content_type: options?.content_type ?? null,
166
- size
167
- });
168
- if (inserted && options?.refs && options.refs.length > 0) {
169
- await query_put_fact_refs(this.#deps, hash, options.refs);
170
- }
171
- return hash;
172
- }
173
- /**
174
- * Retrieve bytes. Embedded reads return PG bytes directly; external
175
- * reads fetch + verify and return `null` (with a warning log) when
176
- * the bytes don't match the stored hash.
134
+ * Retrieve bytes. Embedded reads return PG bytes directly. An external
135
+ * read comes from the disk CAS and nowhere else: the bytes at the row's
136
+ * `file:<shard>/<rest>` URL are read and verified against the hash.
137
+ *
138
+ * Returns `null`, with a warning log, for an external row the store cannot
139
+ * serve — an `external_url` that is not a `file:<shard>/<rest>` URL, no disk
140
+ * CAS configured (`disk_root` + `fs`), a failed disk read, or bytes that
141
+ * don't match the hash. Never makes a network request.
177
142
  */
178
143
  async get(hash) {
179
144
  const row = await query_get_fact(this.#deps, hash);
@@ -185,12 +150,22 @@ export class PgFactStore {
185
150
  if (row.external_url === null) {
186
151
  return null;
187
152
  }
153
+ // the URL is not echoed — the column is free text, so its shape and
154
+ // length are whatever the row's writer chose
155
+ if (!is_file_fact_url(row.external_url)) {
156
+ this.#log?.warn(`PgFactStore.get external_url for ${hash} is not a file:<shard>/<rest> URL; treating as not-found`);
157
+ return null;
158
+ }
159
+ if (this.#disk_root === undefined || this.#fs === undefined) {
160
+ this.#log?.warn(`PgFactStore.get ${hash} is disk-backed and no disk CAS is configured (disk_root + fs); treating as not-found`);
161
+ return null;
162
+ }
188
163
  let bytes;
189
164
  try {
190
- bytes = await this.#fetcher.fetch_bytes(row.external_url);
165
+ bytes = await read_fact_bytes_from_disk(this.#fs, this.#disk_root, row.external_url);
191
166
  }
192
167
  catch (err) {
193
- this.#log?.warn(`PgFactStore.get fetch failed for ${hash} at ${row.external_url}:`, to_error_message(err));
168
+ this.#log?.warn(`PgFactStore.get disk read failed for ${hash} at ${row.external_url}:`, to_error_message(err));
194
169
  return null;
195
170
  }
196
171
  if (!fact_hash_verify(hash, bytes)) {
@@ -226,11 +201,11 @@ export class PgFactStore {
226
201
  * NOT verify the fact is unreferenced — that policy lives one layer
227
202
  * up (the orphan-fact admin surface in the consumer; a future GC walker).
228
203
  *
229
- * External-URL unlink is the caller's responsibility — the store
230
- * doesn't know how to resolve `file:` / `s3:` / etc. URLs to a
231
- * deletable handle. Caller iterates the returned `external_url`
232
- * (when non-null) and dispatches to the appropriate cleanup
233
- * routine. Mirrors the read-side `FactExternalFetcher` split.
204
+ * The disk file of a disk-backed fact is NOT unlinked — that is the
205
+ * caller's responsibility, using the returned `external_url` (when
206
+ * non-null): a `file:<shard>/<rest>` URL names
207
+ * `<disk_root>/<shard>/<rest>`. The value is the row's free text — parse
208
+ * it with `parse_file_fact_url` before joining a path.
234
209
  *
235
210
  * @returns `{size, external_url}` for the deleted row, or `null` if
236
211
  * no row matched the hash.
@@ -260,19 +235,6 @@ const resolve_refs = (bytes, options) => {
260
235
  }
261
236
  return fact_hash_extract_refs(value);
262
237
  };
263
- /** Hash a stream while counting bytes. Lets `put_ref` verify size in one pass. */
264
- const hash_counted_stream = async (stream) => {
265
- let byte_count = 0;
266
- const counting = new TransformStream({
267
- transform(chunk, controller) {
268
- byte_count += chunk.length;
269
- controller.enqueue(chunk);
270
- }
271
- });
272
- const piped = stream.pipeThrough(counting);
273
- const hash = await fact_hash_stream(piped);
274
- return { hash, byte_count };
275
- };
276
238
  /**
277
239
  * Coerce whatever the driver returns for BYTEA into a `Uint8Array`.
278
240
  *
@@ -1,9 +1,9 @@
1
1
  /**
2
2
  * Canonical filesystem-fact URL shape + on-disk layout.
3
3
  *
4
- * `external_url` on the generic `fact` row is `string | null` because the
5
- * `FactStore` interface stays federation-friendly (future
6
- * `https://...` / `s3://...` shapes). Filesystem-minted URLs are exactly
4
+ * `external_url` on the `fact` row is free text (`string | null`), but the
5
+ * only shape `PgFactStore` writes, reads, or serves is the filesystem-minted
6
+ * one — any other reads as unavailable. Those URLs are exactly
7
7
  * `file:<shard>/<rest>` where `<shard>` is the first 2 hex chars of the
8
8
  * blake3 digest and `<rest>` the remaining 62 — files land at
9
9
  * `<facts_dir>/<shard>/<rest>` after the writer atomically temp+renames
@@ -11,9 +11,9 @@
11
11
  *
12
12
  * Centralizing the regex + the `fact_disk_path` split keeps the shape in one
13
13
  * place: `PgFactStore`'s disk CAS (`db/fact_disk_storage.ts`), the
14
- * `serve_fact_route` defense-in-depth check, and the `file_fact_fetcher`
15
- * resolver all derive the layout here, so the write path and the read path
16
- * can't drift. The TS twin of the Rust `fact_disk_path` (`fuz_fact`).
14
+ * `serve_fact_route` defense-in-depth check, and the disk read
15
+ * (`read_fact_bytes_from_disk`) all derive the layout here, so the write path
16
+ * and the read path can't drift. The TS twin of the Rust `fact_disk_path` (`fuz_fact`).
17
17
  *
18
18
  * Defense-in-depth: a `..` segment can't match (`.` isn't in `[0-9a-f]`),
19
19
  * neither can absolute paths, query strings, or any non-hex character.
@@ -1,9 +1,9 @@
1
1
  /**
2
2
  * Canonical filesystem-fact URL shape + on-disk layout.
3
3
  *
4
- * `external_url` on the generic `fact` row is `string | null` because the
5
- * `FactStore` interface stays federation-friendly (future
6
- * `https://...` / `s3://...` shapes). Filesystem-minted URLs are exactly
4
+ * `external_url` on the `fact` row is free text (`string | null`), but the
5
+ * only shape `PgFactStore` writes, reads, or serves is the filesystem-minted
6
+ * one — any other reads as unavailable. Those URLs are exactly
7
7
  * `file:<shard>/<rest>` where `<shard>` is the first 2 hex chars of the
8
8
  * blake3 digest and `<rest>` the remaining 62 — files land at
9
9
  * `<facts_dir>/<shard>/<rest>` after the writer atomically temp+renames
@@ -11,9 +11,9 @@
11
11
  *
12
12
  * Centralizing the regex + the `fact_disk_path` split keeps the shape in one
13
13
  * place: `PgFactStore`'s disk CAS (`db/fact_disk_storage.ts`), the
14
- * `serve_fact_route` defense-in-depth check, and the `file_fact_fetcher`
15
- * resolver all derive the layout here, so the write path and the read path
16
- * can't drift. The TS twin of the Rust `fact_disk_path` (`fuz_fact`).
14
+ * `serve_fact_route` defense-in-depth check, and the disk read
15
+ * (`read_fact_bytes_from_disk`) all derive the layout here, so the write path
16
+ * and the read path can't drift. The TS twin of the Rust `fact_disk_path` (`fuz_fact`).
17
17
  *
18
18
  * Defense-in-depth: a `..` segment can't match (`.` isn't in `[0-9a-f]`),
19
19
  * neither can absolute paths, query strings, or any non-hex character.
@@ -13,9 +13,9 @@
13
13
  * Embedded facts stream from the `fact.bytes` PG column; external facts
14
14
  * (filesystem-backed `file:<shard>/<rest>` URLs) either return an
15
15
  * `X-Accel-Redirect` header pointing into nginx's internal facts location
16
- * (production) or stream from disk via the filesystem `FactExternalFetcher`
17
- * (dev / tests). The runtime mode is selected by the optional `x_accel`
18
- * factory option — a validated `XAccelConfig` (built via
16
+ * (production) or stream from disk (dev / tests). The runtime mode is
17
+ * selected by the optional `x_accel` factory option — a validated
18
+ * `XAccelConfig` (built via
19
19
  * `create_x_accel_config` in `server/x_accel.ts`) set in prod, unset in dev.
20
20
  * The handle can only be obtained by proving the facts nginx location is
21
21
  * `internal;`, so X-Accel serving can't be enabled against a public location.
@@ -80,10 +80,13 @@
80
80
  *
81
81
  * ## Defense-in-depth
82
82
  *
83
- * The `external_url` regex is re-validated before issuing
84
- * `X-Accel-Redirect` even though `PgFactStore.put_ref` only writes
85
- * `file:<shard>/<rest>`-shaped URLs. A future row-injection bug
86
- * upstream would otherwise hand nginx an attacker-controlled path.
83
+ * `external_url` is re-validated against the `file:<shard>/<rest>` shape
84
+ * before it addresses the filesystem or an `X-Accel-Redirect`. `PgFactStore`
85
+ * writes only that shape, but the column is free text and `query_put_fact`
86
+ * inserts what it is given, so a row written past the store — or a
87
+ * row-injection bug upstream — would otherwise hand nginx an
88
+ * attacker-controlled path. Any other shape is a logged 404, the same
89
+ * unavailable that `PgFactStore.get` reports for it as `null`.
87
90
  *
88
91
  * @module
89
92
  */
@@ -1 +1 @@
1
- {"version":3,"file":"serve_fact_route.d.ts","sourceRoot":"../src/lib/","sources":["../../src/lib/server/serve_fact_route.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwFG;AAMH,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAetD,OAAO,EAAuC,KAAK,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAE5F,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAM/C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAkEjD,MAAM,WAAW,+BAA+B;IAC/C;;;;;;OAMG;IACH,IAAI,EAAE,OAAO,CAAC;IACd,kFAAkF;IAClF,SAAS,EAAE,MAAM,CAAC;IAClB;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,YAAY,CAAC;IACvB,GAAG,EAAE,MAAM,CAAC;CACZ;AAgHD;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,iCAAiC,GAC7C,SAAS,+BAA+B,KACtC,SAgDF,CAAC;AAEF;;;;;;;;;;;;;;;;;GAiBG;AACH,eAAO,MAAM,4BAA4B,GACxC,SAAS,+BAA+B,KACtC,SA2CF,CAAC"}
1
+ {"version":3,"file":"serve_fact_route.d.ts","sourceRoot":"../src/lib/","sources":["../../src/lib/server/serve_fact_route.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2FG;AAMH,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAetD,OAAO,EAAuC,KAAK,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAE5F,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAM/C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAkEjD,MAAM,WAAW,+BAA+B;IAC/C;;;;;;OAMG;IACH,IAAI,EAAE,OAAO,CAAC;IACd,kFAAkF;IAClF,SAAS,EAAE,MAAM,CAAC;IAClB;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,YAAY,CAAC;IACvB,GAAG,EAAE,MAAM,CAAC;CACZ;AAkHD;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,iCAAiC,GAC7C,SAAS,+BAA+B,KACtC,SAgDF,CAAC;AAEF;;;;;;;;;;;;;;;;;GAiBG;AACH,eAAO,MAAM,4BAA4B,GACxC,SAAS,+BAA+B,KACtC,SA2CF,CAAC"}
@@ -13,9 +13,9 @@
13
13
  * Embedded facts stream from the `fact.bytes` PG column; external facts
14
14
  * (filesystem-backed `file:<shard>/<rest>` URLs) either return an
15
15
  * `X-Accel-Redirect` header pointing into nginx's internal facts location
16
- * (production) or stream from disk via the filesystem `FactExternalFetcher`
17
- * (dev / tests). The runtime mode is selected by the optional `x_accel`
18
- * factory option — a validated `XAccelConfig` (built via
16
+ * (production) or stream from disk (dev / tests). The runtime mode is
17
+ * selected by the optional `x_accel` factory option — a validated
18
+ * `XAccelConfig` (built via
19
19
  * `create_x_accel_config` in `server/x_accel.ts`) set in prod, unset in dev.
20
20
  * The handle can only be obtained by proving the facts nginx location is
21
21
  * `internal;`, so X-Accel serving can't be enabled against a public location.
@@ -80,10 +80,13 @@
80
80
  *
81
81
  * ## Defense-in-depth
82
82
  *
83
- * The `external_url` regex is re-validated before issuing
84
- * `X-Accel-Redirect` even though `PgFactStore.put_ref` only writes
85
- * `file:<shard>/<rest>`-shaped URLs. A future row-injection bug
86
- * upstream would otherwise hand nginx an attacker-controlled path.
83
+ * `external_url` is re-validated against the `file:<shard>/<rest>` shape
84
+ * before it addresses the filesystem or an `X-Accel-Redirect`. `PgFactStore`
85
+ * writes only that shape, but the column is free text and `query_put_fact`
86
+ * inserts what it is given, so a row written past the store — or a
87
+ * row-injection bug upstream — would otherwise hand nginx an
88
+ * attacker-controlled path. Any other shape is a logged 404, the same
89
+ * unavailable that `PgFactStore.get` reports for it as `null`.
87
90
  *
88
91
  * @module
89
92
  */
@@ -202,7 +205,7 @@ const serve_fact_bytes = async (c, route, hash, config) => {
202
205
  // to address the filesystem.
203
206
  const parsed = parse_file_fact_url(meta.external_url);
204
207
  if (!parsed) {
205
- log.error(`serve_fact: rejecting malformed external_url for ${hash}: ${meta.external_url}`);
208
+ log.error(`serve_fact: external_url for ${hash} is not a file:<shard>/<rest> URL; answering 404`);
206
209
  return c.body(null, 404);
207
210
  }
208
211
  const { shard, rest } = parsed;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@fuzdev/fuz_app",
3
- "version": "0.119.0",
3
+ "version": "0.121.0",
4
4
  "description": "fullstack app library",
5
5
  "glyph": "🗝",
6
6
  "logo": "logo.svg",
@@ -33,7 +33,7 @@
33
33
  "peerDependencies": {
34
34
  "@electric-sql/pglite": ">=0.4",
35
35
  "@fuzdev/blake3-wasm": ">=0.2.0",
36
- "@fuzdev/fuz_util": ">=0.68.0",
36
+ "@fuzdev/fuz_util": ">=0.70.0",
37
37
  "@hono/node-server": ">=1",
38
38
  "@hono/node-ws": ">=1",
39
39
  "@node-rs/argon2": ">=2",
@@ -77,7 +77,7 @@
77
77
  "@fuzdev/fuz_code": "^0.48.0",
78
78
  "@fuzdev/fuz_css": "^0.64.0",
79
79
  "@fuzdev/fuz_ui": "^0.208.0",
80
- "@fuzdev/fuz_util": "^0.69.0",
80
+ "@fuzdev/fuz_util": "^0.70.0",
81
81
  "@fuzdev/gro": "^0.213.0",
82
82
  "@fuzdev/mdz": "^0.3.0",
83
83
  "@hono/node-server": "^1.19.14",
@@ -1,42 +0,0 @@
1
- /**
2
- * Filesystem-backed `FactExternalFetcher`.
3
- *
4
- * Resolves `file:<shard>/<rest>` external URLs against a configured
5
- * facts directory. The URL shape is the canonical relative-path scheme:
6
- * 2 hex chars for the shard subdir + 62 hex chars for the rest of the
7
- * blake3 hash (64 hex chars total). Files live at
8
- * `<facts_dir>/<shard>/<rest>` after the writer atomically temp+renames
9
- * them in.
10
- *
11
- * **Pre-filter regex**: `^file:[0-9a-f]{2}/[0-9a-f]{62}$`. A `..`
12
- * segment can't match (`.` isn't in `[0-9a-f]`); neither can absolute
13
- * paths, query strings, or any non-hex character. Defense-in-depth in
14
- * front of `path.join` — the fetcher never trusts the URL came from a
15
- * fact row, even though it always does in practice.
16
- *
17
- * The fetcher does NOT verify hash content — `PgFactStore.get` calls
18
- * `fact_hash_verify(hash, bytes)` after the fetch and returns null on
19
- * mismatch.
20
- *
21
- * Runtime: uses `node:fs/promises` + `node:fs` `createReadStream` so
22
- * the same code works under Deno (via node compat) and vitest.
23
- *
24
- * @module
25
- */
26
- import type { FactExternalFetcher } from '../db/fact_store.js';
27
- /** Construction options. */
28
- export interface FileFactFetcherOptions {
29
- /**
30
- * Absolute path to the facts directory. Files resolve to
31
- * `<facts_dir>/<shard>/<rest>`.
32
- */
33
- facts_dir: string;
34
- }
35
- /**
36
- * Build a `FactExternalFetcher` that resolves `file:` URLs against the
37
- * filesystem. Throws on a malformed URL before touching the disk so
38
- * `PgFactStore.get` logs the warning + returns null without an I/O
39
- * round-trip on bad data.
40
- */
41
- export declare const create_file_fact_fetcher: (options: FileFactFetcherOptions) => FactExternalFetcher;
42
- //# sourceMappingURL=file_fact_fetcher.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"file_fact_fetcher.d.ts","sourceRoot":"../src/lib/","sources":["../../src/lib/server/file_fact_fetcher.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAOH,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAG/D,4BAA4B;AAC5B,MAAM,WAAW,sBAAsB;IACtC;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;CAClB;AAED;;;;;GAKG;AACH,eAAO,MAAM,wBAAwB,GAAI,SAAS,sBAAsB,KAAG,mBA0B1E,CAAC"}
@@ -1,60 +0,0 @@
1
- /**
2
- * Filesystem-backed `FactExternalFetcher`.
3
- *
4
- * Resolves `file:<shard>/<rest>` external URLs against a configured
5
- * facts directory. The URL shape is the canonical relative-path scheme:
6
- * 2 hex chars for the shard subdir + 62 hex chars for the rest of the
7
- * blake3 hash (64 hex chars total). Files live at
8
- * `<facts_dir>/<shard>/<rest>` after the writer atomically temp+renames
9
- * them in.
10
- *
11
- * **Pre-filter regex**: `^file:[0-9a-f]{2}/[0-9a-f]{62}$`. A `..`
12
- * segment can't match (`.` isn't in `[0-9a-f]`); neither can absolute
13
- * paths, query strings, or any non-hex character. Defense-in-depth in
14
- * front of `path.join` — the fetcher never trusts the URL came from a
15
- * fact row, even though it always does in practice.
16
- *
17
- * The fetcher does NOT verify hash content — `PgFactStore.get` calls
18
- * `fact_hash_verify(hash, bytes)` after the fetch and returns null on
19
- * mismatch.
20
- *
21
- * Runtime: uses `node:fs/promises` + `node:fs` `createReadStream` so
22
- * the same code works under Deno (via node compat) and vitest.
23
- *
24
- * @module
25
- */
26
- import { readFile } from 'node:fs/promises';
27
- import { createReadStream } from 'node:fs';
28
- import { Readable } from 'node:stream';
29
- import { join } from 'node:path';
30
- import { parse_file_fact_url } from '../db/file_fact_url.js';
31
- /**
32
- * Build a `FactExternalFetcher` that resolves `file:` URLs against the
33
- * filesystem. Throws on a malformed URL before touching the disk so
34
- * `PgFactStore.get` logs the warning + returns null without an I/O
35
- * round-trip on bad data.
36
- */
37
- export const create_file_fact_fetcher = (options) => {
38
- const { facts_dir } = options;
39
- const resolve_path = (url) => {
40
- const parsed = parse_file_fact_url(url);
41
- if (!parsed) {
42
- throw new Error(`invalid file fact url: ${url}`);
43
- }
44
- return join(facts_dir, parsed.shard, parsed.rest);
45
- };
46
- return {
47
- fetch_bytes: async (url) => {
48
- const path = resolve_path(url);
49
- const buf = await readFile(path);
50
- return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength);
51
- },
52
- // `Promise.resolve().then(...)` (rather than `async`) funnels any
53
- // `resolve_path` throw into a rejection without an unused `await`.
54
- fetch_stream: (url) => Promise.resolve().then(() => {
55
- const path = resolve_path(url);
56
- const node_stream = createReadStream(path);
57
- return Readable.toWeb(node_stream);
58
- })
59
- };
60
- };