@fuzdev/fuz_app 0.119.0 → 0.121.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth/deps.d.ts +3 -3
- package/dist/db/CLAUDE.md +12 -7
- package/dist/db/fact_disk_storage.d.ts +10 -8
- package/dist/db/fact_disk_storage.d.ts.map +1 -1
- package/dist/db/fact_disk_storage.js +13 -17
- package/dist/db/fact_store.d.ts +39 -38
- package/dist/db/fact_store.d.ts.map +1 -1
- package/dist/db/fact_store.js +48 -86
- package/dist/db/file_fact_url.d.ts +6 -6
- package/dist/db/file_fact_url.js +6 -6
- package/dist/server/serve_fact_route.d.ts +10 -7
- package/dist/server/serve_fact_route.d.ts.map +1 -1
- package/dist/server/serve_fact_route.js +11 -8
- package/package.json +3 -3
- package/dist/server/file_fact_fetcher.d.ts +0 -42
- package/dist/server/file_fact_fetcher.d.ts.map +0 -1
- package/dist/server/file_fact_fetcher.js +0 -60
package/dist/auth/deps.d.ts
CHANGED
|
@@ -61,9 +61,9 @@ export interface AppDeps {
|
|
|
61
61
|
/**
|
|
62
62
|
* Optional content-addressed byte store. Present only on backends that
|
|
63
63
|
* serve binary content (facts) — minimal consumers leave it unset. The
|
|
64
|
-
* consumer constructs a `PgFactStore` (`db/fact_store.ts`)
|
|
65
|
-
* `
|
|
66
|
-
*
|
|
64
|
+
* consumer constructs a `PgFactStore` (`db/fact_store.ts`) over its facts
|
|
65
|
+
* directory (`disk_root` + `fs`) at its own backend assembly and assigns it
|
|
66
|
+
* here; `create_app_backend` stays facts-agnostic.
|
|
67
67
|
*/
|
|
68
68
|
fact_store?: FactStore;
|
|
69
69
|
}
|
package/dist/db/CLAUDE.md
CHANGED
|
@@ -156,11 +156,16 @@ DO NOTHING`), `_put_fact_refs`, `_get_fact` / `_get_fact_meta` / `_has_fact`
|
|
|
156
156
|
/ `_select_for_delete` (a fact is orphan when no active `cell.refs` names it).
|
|
157
157
|
- **`fact_store.ts`** — `PgFactStore implements FactStore` (the interface lives
|
|
158
158
|
in `@fuzdev/fuz_util/fact_store.ts`): size-routed writes (embedded ≤
|
|
159
|
-
`embedded_threshold` / disk CAS above it
|
|
160
|
-
|
|
161
|
-
content via an injected `FactExternalFetcher`. With `disk_root` + `fs` (the
|
|
159
|
+
`embedded_threshold` / disk CAS above it), JSON ref auto-extract, idempotent
|
|
160
|
+
put, verify-on-read for disk-backed content. With `disk_root` + `fs` (the
|
|
162
161
|
`runtime/*Deps`) configured, oversize `put` and the streaming `put_stream`
|
|
163
|
-
write to the `<shard>/<rest>` disk CAS and
|
|
162
|
+
write to the `<shard>/<rest>` disk CAS and `get` reads from it; without them
|
|
163
|
+
an oversize write throws (`put_stream` needs `fs` for a body of any size).
|
|
164
|
+
The disk CAS is the only place `get` reads external bytes from; the store
|
|
165
|
+
makes no network request. A row whose `external_url` is not a
|
|
166
|
+
`file:<shard>/<rest>` URL (the store never writes one), and any external row
|
|
167
|
+
on a store with no disk CAS, reads as `null` with a warning; the serve route
|
|
168
|
+
404s the former.
|
|
164
169
|
- **`file_fact_url.ts`** — the canonical `file:<shard>/<rest>` URL shape
|
|
165
170
|
(`FileFactUrl` brand, `mint_file_fact_url` / `parse_file_fact_url` /
|
|
166
171
|
`FILE_FACT_URL_PATTERN`) plus `fact_disk_path(hash) → {shard, rest}`, the
|
|
@@ -168,15 +173,15 @@ DO NOTHING`), `_put_fact_refs`, `_get_fact` / `_get_fact_meta` / `_has_fact`
|
|
|
168
173
|
- **`fact_disk_storage.ts`** — the filesystem CAS over `runtime/{FsStream,FsWrite,FsRemove,FsRead}Deps`
|
|
169
174
|
(not raw `node:fs`): `stream_fact_to_disk` (bounded-memory blake3+sha256 single
|
|
170
175
|
pass, buffer→spill, fsync-then-atomic-rename, dedup-drop if the CAS path already
|
|
171
|
-
exists), `write_fact_bytes_to_disk` (buffering twin), `
|
|
176
|
+
exists), `write_fact_bytes_to_disk` (buffering twin), `read_fact_bytes_from_disk`,
|
|
172
177
|
and `sweep_orphan_temps` (reaps stale `.tmp` spills by mtime). The temp is
|
|
173
178
|
`fsync`ed before the rename publishes it (twins the Rust `fuz_fact` §fsync
|
|
174
179
|
posture: data-sync before rename, parent-dir fsync waived) — the serve path
|
|
175
180
|
streams the file without re-hashing, so write-time durability is the guard.
|
|
176
181
|
- **`fact_store_errors.ts`** — `PayloadTooLargeError` / `StorageFullError` (+
|
|
177
182
|
`is_enospc_error`) thrown by `put_stream`, for a consumer route's 413 / 507.
|
|
178
|
-
- The
|
|
179
|
-
|
|
183
|
+
- The HTTP serving plumbing lives under `server/` (`serve_fact_route.ts`,
|
|
184
|
+
`x_accel.ts`).
|
|
180
185
|
|
|
181
186
|
### Migration namespace order
|
|
182
187
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Filesystem CAS for
|
|
2
|
+
* Filesystem CAS for fact bytes stored outside Postgres — the disk half of
|
|
3
3
|
* `PgFactStore`, threaded over the injectable `runtime/*Deps` rather than raw
|
|
4
4
|
* `node:fs`, so it runs unchanged under Node, Deno, and a mock runtime.
|
|
5
5
|
*
|
|
@@ -31,7 +31,6 @@ import { type FactHash } from '@fuzdev/fuz_util/hash_schemas.ts';
|
|
|
31
31
|
import type { Logger } from '@fuzdev/fuz_util/log.ts';
|
|
32
32
|
import type { FsReadDeps, FsWriteDeps, FsStreamDeps, FsRemoveDeps } from '../runtime/deps.js';
|
|
33
33
|
import { type FileFactUrl } from './file_fact_url.js';
|
|
34
|
-
import type { FactExternalFetcher } from './fact_store.js';
|
|
35
34
|
/** Subdirectory under `facts_dir` for in-flight atomic temp files. */
|
|
36
35
|
export declare const FACT_TMP_DIRNAME = ".tmp";
|
|
37
36
|
/** Default age (1 hour) past which a `.tmp/*` file is considered orphaned. */
|
|
@@ -41,7 +40,7 @@ export declare const FACT_TMP_ORPHAN_MAX_AGE_MS: number;
|
|
|
41
40
|
* full `RuntimeDeps` (Node or Deno) satisfies this; each function below picks
|
|
42
41
|
* the narrow subset it actually uses.
|
|
43
42
|
*/
|
|
44
|
-
export type FactDiskStorageDeps = Pick<FsReadDeps, 'stat' | 'readdir' | 'read_file'> & Pick<FsWriteDeps, 'mkdir' | 'rename' | 'write_file' | 'fsync'> & Pick<FsStreamDeps, 'write_file_stream'
|
|
43
|
+
export type FactDiskStorageDeps = Pick<FsReadDeps, 'stat' | 'readdir' | 'read_file'> & Pick<FsWriteDeps, 'mkdir' | 'rename' | 'write_file' | 'fsync'> & Pick<FsStreamDeps, 'write_file_stream'> & Pick<FsRemoveDeps, 'remove'>;
|
|
45
44
|
/**
|
|
46
45
|
* Where a streamed body landed — `embedded` carries the in-memory bytes (under
|
|
47
46
|
* the embedded threshold, bound for the PG `fact.bytes` column); `disk` means
|
|
@@ -100,16 +99,19 @@ export declare const stream_fact_to_disk: (deps: Pick<FactDiskStorageDeps, "mkdi
|
|
|
100
99
|
*/
|
|
101
100
|
export declare const write_fact_bytes_to_disk: (deps: Pick<FactDiskStorageDeps, "mkdir" | "rename" | "remove" | "write_file" | "fsync" | "stat">, facts_dir: string, hash: FactHash, bytes: Uint8Array) => Promise<FileFactUrl>;
|
|
102
101
|
/**
|
|
103
|
-
*
|
|
104
|
-
* writers above produce,
|
|
105
|
-
*
|
|
106
|
-
*
|
|
102
|
+
* Read a disk-backed fact's bytes from the `<facts_dir>/<shard>/<rest>` layout
|
|
103
|
+
* the writers above produce, addressed by the row's `file:<shard>/<rest>`
|
|
104
|
+
* `external_url`. The only read of external bytes `PgFactStore.get` performs.
|
|
105
|
+
* Does NOT verify hash content — `PgFactStore.get` calls
|
|
106
|
+
* `fact_hash_verify(hash, bytes)` on the result and returns `null` on mismatch.
|
|
107
107
|
*
|
|
108
108
|
* Defense at the read seam is the `FILE_FACT_URL_PATTERN` regex (via
|
|
109
109
|
* `parse_file_fact_url`) — `..` segments, foreign schemes, and non-hex chars
|
|
110
110
|
* fail before any disk access.
|
|
111
|
+
*
|
|
112
|
+
* @throws Error when `url` is not a `file:<shard>/<rest>` URL, or when the read fails
|
|
111
113
|
*/
|
|
112
|
-
export declare const
|
|
114
|
+
export declare const read_fact_bytes_from_disk: (deps: Pick<FactDiskStorageDeps, "read_file">, facts_dir: string, url: string) => Promise<Uint8Array>;
|
|
113
115
|
/**
|
|
114
116
|
* Reap stale temp files left under `<facts_dir>/.tmp/` by a hard crash (SIGKILL
|
|
115
117
|
* / OOM / host crash) mid-write — the `finally` cleanup in the writers above
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fact_disk_storage.d.ts","sourceRoot":"../src/lib/","sources":["../../src/lib/db/fact_disk_storage.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AASH,OAAO,EAAoB,KAAK,QAAQ,EAAE,MAAM,kCAAkC,CAAC;AACnF,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAEtD,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAE9F,OAAO,EAIN,KAAK,WAAW,EAChB,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"fact_disk_storage.d.ts","sourceRoot":"../src/lib/","sources":["../../src/lib/db/fact_disk_storage.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AASH,OAAO,EAAoB,KAAK,QAAQ,EAAE,MAAM,kCAAkC,CAAC;AACnF,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAEtD,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAE9F,OAAO,EAIN,KAAK,WAAW,EAChB,MAAM,oBAAoB,CAAC;AAG5B,sEAAsE;AACtE,eAAO,MAAM,gBAAgB,SAAS,CAAC;AAEvC,8EAA8E;AAC9E,eAAO,MAAM,0BAA0B,QAAiB,CAAC;AAEzD;;;;GAIG;AACH,MAAM,MAAM,mBAAmB,GAAG,IAAI,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,GAAG,WAAW,CAAC,GACnF,IAAI,CAAC,WAAW,EAAE,OAAO,GAAG,QAAQ,GAAG,YAAY,GAAG,OAAO,CAAC,GAC9D,IAAI,CAAC,YAAY,EAAE,mBAAmB,CAAC,GACvC,IAAI,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;AAE9B;;;;;GAKG;AACH,MAAM,MAAM,eAAe,GAC1B;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,KAAK,EAAE,UAAU,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,WAAW,CAAA;CAAE,CAAC;AAEvF;;;;GAIG;AACH,MAAM,WAAW,sBAAsB;IACtC,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,eAAe,CAAC;CAC3B;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,eAAO,MAAM,mBAAmB,GAC/B,MAAM,IAAI,CACT,mBAAmB,EACnB,OAAO,GAAG,QAAQ,GAAG,QAAQ,GAAG,mBAAmB,GAAG,OAAO,GAAG,MAAM,CACtE,EACD,WAAW,MAAM,GAAG,SAAS,EAC7B,QAAQ,cAAc,CAAC,UAAU,CAAC,EAClC,WAAW,MAAM,EACjB,oBAAoB,MAAM,KACxB,OAAO,CAAC,sBAAsB,CA8GhC,CAAC;AAEF;;;;;;;;GAQG;AACH,eAAO,MAAM,wBAAwB,GACpC,MAAM,IAAI,CAAC,mBAAmB,EAAE,OAAO,GAAG,QAAQ,GAAG,QAAQ,GAAG,YAAY,GAAG,OAAO,GAAG,MAAM,CAAC,EAChG,WAAW,MAAM,EACjB,MAAM,QAAQ,EACd,OAAO,UAAU,KACf,OAAO,CAAC,WAAW,CAgBrB,CAAC;AAEF;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,yBAAyB,GACrC,MAAM,IAAI,CAAC,mBAAmB,EAAE,WAAW,CAAC,EAC5C,WAAW,MAAM,EACjB,KAAK,MAAM,KACT,OAAO,CAAC,UAAU,CAIpB,CAAC;AAEF;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,kBAAkB,GAC9B,MAAM,IAAI,CAAC,mBAAmB,EAAE,SAAS,GAAG,MAAM,GAAG,QAAQ,CAAC,EAC9D,WAAW,MAAM,EACjB,UAAU;IAAE,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,KAC3D,OAAO,CAAC,MAAM,CA0BhB,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Filesystem CAS for
|
|
2
|
+
* Filesystem CAS for fact bytes stored outside Postgres — the disk half of
|
|
3
3
|
* `PgFactStore`, threaded over the injectable `runtime/*Deps` rather than raw
|
|
4
4
|
* `node:fs`, so it runs unchanged under Node, Deno, and a mock runtime.
|
|
5
5
|
*
|
|
@@ -204,27 +204,23 @@ export const write_fact_bytes_to_disk = async (deps, facts_dir, hash, bytes) =>
|
|
|
204
204
|
return mint_file_fact_url(shard, rest);
|
|
205
205
|
};
|
|
206
206
|
/**
|
|
207
|
-
*
|
|
208
|
-
* writers above produce,
|
|
209
|
-
*
|
|
210
|
-
*
|
|
207
|
+
* Read a disk-backed fact's bytes from the `<facts_dir>/<shard>/<rest>` layout
|
|
208
|
+
* the writers above produce, addressed by the row's `file:<shard>/<rest>`
|
|
209
|
+
* `external_url`. The only read of external bytes `PgFactStore.get` performs.
|
|
210
|
+
* Does NOT verify hash content — `PgFactStore.get` calls
|
|
211
|
+
* `fact_hash_verify(hash, bytes)` on the result and returns `null` on mismatch.
|
|
211
212
|
*
|
|
212
213
|
* Defense at the read seam is the `FILE_FACT_URL_PATTERN` regex (via
|
|
213
214
|
* `parse_file_fact_url`) — `..` segments, foreign schemes, and non-hex chars
|
|
214
215
|
* fail before any disk access.
|
|
216
|
+
*
|
|
217
|
+
* @throws Error when `url` is not a `file:<shard>/<rest>` URL, or when the read fails
|
|
215
218
|
*/
|
|
216
|
-
export const
|
|
217
|
-
const
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
return join(facts_dir, parsed.shard, parsed.rest);
|
|
222
|
-
};
|
|
223
|
-
return {
|
|
224
|
-
fetch_bytes: (url) => deps.read_file(resolve_path(url)),
|
|
225
|
-
// `async` funnels a synchronous `resolve_path` throw into a rejection.
|
|
226
|
-
fetch_stream: async (url) => deps.read_file_stream(resolve_path(url))
|
|
227
|
-
};
|
|
219
|
+
export const read_fact_bytes_from_disk = async (deps, facts_dir, url) => {
|
|
220
|
+
const parsed = parse_file_fact_url(url);
|
|
221
|
+
if (!parsed)
|
|
222
|
+
throw new Error('invalid file fact url');
|
|
223
|
+
return deps.read_file(join(facts_dir, parsed.shard, parsed.rest));
|
|
228
224
|
};
|
|
229
225
|
/**
|
|
230
226
|
* Reap stale temp files left under `<facts_dir>/.tmp/` by a hard crash (SIGKILL
|
package/dist/db/fact_store.d.ts
CHANGED
|
@@ -4,14 +4,13 @@
|
|
|
4
4
|
* Wraps the raw queries in `db/fact_queries.ts` with the lifecycle the
|
|
5
5
|
* `FactStore` interface promises:
|
|
6
6
|
*
|
|
7
|
-
* - sync hash on `put
|
|
8
|
-
* the caller-supplied `size`)
|
|
7
|
+
* - sync hash on `put`; `put_stream` hashes as it streams
|
|
9
8
|
* - idempotent insert (`ON CONFLICT DO NOTHING` in the queries layer)
|
|
10
9
|
* - JSON ref auto-extraction when `content_type` signals JSON and the
|
|
11
10
|
* caller didn't pass an explicit `refs` array
|
|
12
|
-
* - verify-on-read for
|
|
11
|
+
* - verify-on-read for disk-backed content; embedded reads skip verify
|
|
13
12
|
* because PG storage IS the hash table
|
|
14
|
-
* - mismatched
|
|
13
|
+
* - mismatched disk bytes return `null` + log warning (treat as
|
|
15
14
|
* unavailable; GC / repair is a separate concern)
|
|
16
15
|
*
|
|
17
16
|
* Embedded vs disk split: writes route by size. Bytes `<= embedded_threshold`
|
|
@@ -19,10 +18,20 @@
|
|
|
19
18
|
* `<facts_dir>/<shard>/<rest>` (`db/fact_disk_storage.ts`) and the row records a
|
|
20
19
|
* `file:<shard>/<rest>` `external_url`. `put` takes fully-buffered bytes;
|
|
21
20
|
* `put_stream` is the bounded-memory streaming twin (hash BLAKE3 + SHA-256 in
|
|
22
|
-
* one pass, spill past the threshold, enforce `max_bytes` / `ENOSPC`).
|
|
23
|
-
* `disk_root` + `fs` (the `runtime/*Deps`)
|
|
24
|
-
*
|
|
25
|
-
*
|
|
21
|
+
* one pass, spill past the threshold, enforce `max_bytes` / `ENOSPC`). `put`
|
|
22
|
+
* needs `disk_root` + `fs` (the `runtime/*Deps`) for the over-threshold path
|
|
23
|
+
* only: without them the store has nowhere to put bytes it cannot embed, and an
|
|
24
|
+
* oversize `put` throws. `put_stream` needs `fs` for a body of any size — with
|
|
25
|
+
* none it throws before reading the stream — and `disk_root` for the
|
|
26
|
+
* over-threshold spill: with none, a body that passes the threshold throws
|
|
27
|
+
* `PayloadTooLargeError`.
|
|
28
|
+
*
|
|
29
|
+
* Every `external_url` the store writes is a `file:<shard>/<rest>` URL into its
|
|
30
|
+
* own disk CAS, and that CAS is the only place `get` reads external bytes from
|
|
31
|
+
* — the store makes no network request. The column is free text
|
|
32
|
+
* (`query_put_fact` inserts what it is given), so `get` treats a row whose
|
|
33
|
+
* `external_url` has any other shape, and any external row on a store with no
|
|
34
|
+
* disk CAS configured, as unavailable: a warning and `null`.
|
|
26
35
|
*
|
|
27
36
|
* @module
|
|
28
37
|
*/
|
|
@@ -33,13 +42,6 @@ import type { FactMeta, FactPutOptions, FactStore, PutStreamOutcome } from '@fuz
|
|
|
33
42
|
import { type FactDiskStorageDeps } from './fact_disk_storage.js';
|
|
34
43
|
/** Default embedded-vs-referenced cutoff (1 MiB). */
|
|
35
44
|
export declare const FACT_EMBEDDED_THRESHOLD_DEFAULT: number;
|
|
36
|
-
/** Fetcher abstraction so tests can stub external URL retrieval. */
|
|
37
|
-
export interface FactExternalFetcher {
|
|
38
|
-
fetch_stream: (url: string) => Promise<ReadableStream<Uint8Array>>;
|
|
39
|
-
fetch_bytes: (url: string) => Promise<Uint8Array>;
|
|
40
|
-
}
|
|
41
|
-
/** Default fetcher backed by `globalThis.fetch`. */
|
|
42
|
-
export declare const create_default_fetcher: () => FactExternalFetcher;
|
|
43
45
|
/**
|
|
44
46
|
* Construction-time deps for `PgFactStore`.
|
|
45
47
|
*
|
|
@@ -52,18 +54,19 @@ export declare const create_default_fetcher: () => FactExternalFetcher;
|
|
|
52
54
|
*
|
|
53
55
|
* `disk_root` is the facts directory backing the `<shard>/<rest>` disk CAS;
|
|
54
56
|
* `fs` supplies the filesystem capabilities (a `RuntimeDeps` satisfies it).
|
|
55
|
-
* When both are set, oversize `put` + `put_stream` write to disk and
|
|
56
|
-
*
|
|
57
|
-
*
|
|
58
|
-
*
|
|
59
|
-
*
|
|
57
|
+
* When both are set, oversize `put` + `put_stream` write to disk and `get`
|
|
58
|
+
* reads disk-backed facts from it. When either is unset the store writes
|
|
59
|
+
* embedded rows only and `get` returns `null` for an external row: an oversize
|
|
60
|
+
* `put` throws; `put_stream` throws for a body of any size when `fs` is unset,
|
|
61
|
+
* and with `fs` but no `disk_root` throws `PayloadTooLargeError` for a body
|
|
62
|
+
* that passes the threshold. `log` is optional — its call sites are the
|
|
63
|
+
* warnings on the paths where `get` returns `null` for an external row.
|
|
60
64
|
*/
|
|
61
65
|
export interface PgFactStoreDeps {
|
|
62
66
|
deps: QueryDeps;
|
|
63
67
|
embedded_threshold?: number;
|
|
64
68
|
disk_root?: string;
|
|
65
69
|
fs?: FactDiskStorageDeps;
|
|
66
|
-
fetcher?: FactExternalFetcher;
|
|
67
70
|
log?: Logger;
|
|
68
71
|
}
|
|
69
72
|
/**
|
|
@@ -77,8 +80,8 @@ export declare class PgFactStore implements FactStore {
|
|
|
77
80
|
* Store fully-buffered bytes, routing by size: `<= embedded_threshold` into
|
|
78
81
|
* the PG `bytes` column; larger into the disk CAS (when `disk_root` + `fs`
|
|
79
82
|
* are configured) at `<facts_dir>/<shard>/<rest>` with a `file:` URL. Oversize
|
|
80
|
-
* without a disk root throws
|
|
81
|
-
*
|
|
83
|
+
* without a disk root throws: the store has nowhere to put bytes it cannot
|
|
84
|
+
* embed. Idempotent — `ON CONFLICT DO NOTHING` + content-addressed disk
|
|
82
85
|
* filenames make a re-write a no-op.
|
|
83
86
|
*/
|
|
84
87
|
put(bytes: Uint8Array, options?: FactPutOptions): Promise<FactHash>;
|
|
@@ -101,16 +104,14 @@ export declare class PgFactStore implements FactStore {
|
|
|
101
104
|
*/
|
|
102
105
|
put_stream(stream: ReadableStream<Uint8Array>, max_bytes: number, options?: FactPutOptions): Promise<PutStreamOutcome>;
|
|
103
106
|
/**
|
|
104
|
-
*
|
|
105
|
-
*
|
|
106
|
-
*
|
|
107
|
-
*
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
*
|
|
112
|
-
* reads fetch + verify and return `null` (with a warning log) when
|
|
113
|
-
* the bytes don't match the stored hash.
|
|
107
|
+
* Retrieve bytes. Embedded reads return PG bytes directly. An external
|
|
108
|
+
* read comes from the disk CAS and nowhere else: the bytes at the row's
|
|
109
|
+
* `file:<shard>/<rest>` URL are read and verified against the hash.
|
|
110
|
+
*
|
|
111
|
+
* Returns `null`, with a warning log, for an external row the store cannot
|
|
112
|
+
* serve — an `external_url` that is not a `file:<shard>/<rest>` URL, no disk
|
|
113
|
+
* CAS configured (`disk_root` + `fs`), a failed disk read, or bytes that
|
|
114
|
+
* don't match the hash. Never makes a network request.
|
|
114
115
|
*/
|
|
115
116
|
get(hash: FactHash): Promise<Uint8Array | null>;
|
|
116
117
|
has(hash: FactHash): Promise<boolean>;
|
|
@@ -126,11 +127,11 @@ export declare class PgFactStore implements FactStore {
|
|
|
126
127
|
* NOT verify the fact is unreferenced — that policy lives one layer
|
|
127
128
|
* up (the orphan-fact admin surface in the consumer; a future GC walker).
|
|
128
129
|
*
|
|
129
|
-
*
|
|
130
|
-
*
|
|
131
|
-
*
|
|
132
|
-
*
|
|
133
|
-
*
|
|
130
|
+
* The disk file of a disk-backed fact is NOT unlinked — that is the
|
|
131
|
+
* caller's responsibility, using the returned `external_url` (when
|
|
132
|
+
* non-null): a `file:<shard>/<rest>` URL names
|
|
133
|
+
* `<disk_root>/<shard>/<rest>`. The value is the row's free text — parse
|
|
134
|
+
* it with `parse_file_fact_url` before joining a path.
|
|
134
135
|
*
|
|
135
136
|
* @returns `{size, external_url}` for the deleted row, or `null` if
|
|
136
137
|
* no row matched the hash.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fact_store.d.ts","sourceRoot":"../src/lib/","sources":["../../src/lib/db/fact_store.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"fact_store.d.ts","sourceRoot":"../src/lib/","sources":["../../src/lib/db/fact_store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEjD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAOtD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,kCAAkC,CAAC;AACjE,OAAO,KAAK,EACX,QAAQ,EACR,cAAc,EACd,SAAS,EACT,gBAAgB,EAChB,MAAM,gCAAgC,CAAC;AAYxC,OAAO,EAIN,KAAK,mBAAmB,EACxB,MAAM,wBAAwB,CAAC;AAEhC,qDAAqD;AACrD,eAAO,MAAM,+BAA+B,QAAc,CAAC;AAE3D;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,WAAW,eAAe;IAC/B,IAAI,EAAE,SAAS,CAAC;IAChB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,EAAE,CAAC,EAAE,mBAAmB,CAAC;IACzB,GAAG,CAAC,EAAE,MAAM,CAAC;CACb;AAED;;;GAGG;AACH,qBAAa,WAAY,YAAW,SAAS;;gBAOhC,OAAO,EAAE,eAAe;IAQpC;;;;;;;OAOG;IACG,GAAG,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,QAAQ,CAAC;IA+BzE;;;;;;;;;;;;;;;;OAgBG;IACG,UAAU,CACf,MAAM,EAAE,cAAc,CAAC,UAAU,CAAC,EAClC,SAAS,EAAE,MAAM,EACjB,OAAO,CAAC,EAAE,cAAc,GACtB,OAAO,CAAC,gBAAgB,CAAC;IA6B5B;;;;;;;;;OASG;IACG,GAAG,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IA0C/C,GAAG,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC;IAIrC,QAAQ,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;IAWlD,QAAQ,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAIxD;;;;;;;;;;;;;;;;;;OAkBG;IACG,MAAM,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI,CAAC;CAG3F"}
|
package/dist/db/fact_store.js
CHANGED
|
@@ -4,14 +4,13 @@
|
|
|
4
4
|
* Wraps the raw queries in `db/fact_queries.ts` with the lifecycle the
|
|
5
5
|
* `FactStore` interface promises:
|
|
6
6
|
*
|
|
7
|
-
* - sync hash on `put
|
|
8
|
-
* the caller-supplied `size`)
|
|
7
|
+
* - sync hash on `put`; `put_stream` hashes as it streams
|
|
9
8
|
* - idempotent insert (`ON CONFLICT DO NOTHING` in the queries layer)
|
|
10
9
|
* - JSON ref auto-extraction when `content_type` signals JSON and the
|
|
11
10
|
* caller didn't pass an explicit `refs` array
|
|
12
|
-
* - verify-on-read for
|
|
11
|
+
* - verify-on-read for disk-backed content; embedded reads skip verify
|
|
13
12
|
* because PG storage IS the hash table
|
|
14
|
-
* - mismatched
|
|
13
|
+
* - mismatched disk bytes return `null` + log warning (treat as
|
|
15
14
|
* unavailable; GC / repair is a separate concern)
|
|
16
15
|
*
|
|
17
16
|
* Embedded vs disk split: writes route by size. Bytes `<= embedded_threshold`
|
|
@@ -19,39 +18,30 @@
|
|
|
19
18
|
* `<facts_dir>/<shard>/<rest>` (`db/fact_disk_storage.ts`) and the row records a
|
|
20
19
|
* `file:<shard>/<rest>` `external_url`. `put` takes fully-buffered bytes;
|
|
21
20
|
* `put_stream` is the bounded-memory streaming twin (hash BLAKE3 + SHA-256 in
|
|
22
|
-
* one pass, spill past the threshold, enforce `max_bytes` / `ENOSPC`).
|
|
23
|
-
* `disk_root` + `fs` (the `runtime/*Deps`)
|
|
24
|
-
*
|
|
25
|
-
*
|
|
21
|
+
* one pass, spill past the threshold, enforce `max_bytes` / `ENOSPC`). `put`
|
|
22
|
+
* needs `disk_root` + `fs` (the `runtime/*Deps`) for the over-threshold path
|
|
23
|
+
* only: without them the store has nowhere to put bytes it cannot embed, and an
|
|
24
|
+
* oversize `put` throws. `put_stream` needs `fs` for a body of any size — with
|
|
25
|
+
* none it throws before reading the stream — and `disk_root` for the
|
|
26
|
+
* over-threshold spill: with none, a body that passes the threshold throws
|
|
27
|
+
* `PayloadTooLargeError`.
|
|
28
|
+
*
|
|
29
|
+
* Every `external_url` the store writes is a `file:<shard>/<rest>` URL into its
|
|
30
|
+
* own disk CAS, and that CAS is the only place `get` reads external bytes from
|
|
31
|
+
* — the store makes no network request. The column is free text
|
|
32
|
+
* (`query_put_fact` inserts what it is given), so `get` treats a row whose
|
|
33
|
+
* `external_url` has any other shape, and any external row on a store with no
|
|
34
|
+
* disk CAS configured, as unavailable: a warning and `null`.
|
|
26
35
|
*
|
|
27
36
|
* @module
|
|
28
37
|
*/
|
|
29
38
|
import { to_error_message } from '@fuzdev/fuz_util/error.ts';
|
|
30
|
-
import { fact_hash_bytes,
|
|
39
|
+
import { fact_hash_bytes, fact_hash_verify, fact_hash_extract_refs } from '@fuzdev/fuz_util/fact_hash.ts';
|
|
31
40
|
import { query_delete_fact, query_get_fact, query_get_fact_meta, query_get_fact_refs, query_has_fact, query_put_fact, query_put_fact_refs } from './fact_queries.js';
|
|
32
|
-
import {
|
|
41
|
+
import { is_file_fact_url } from './file_fact_url.js';
|
|
42
|
+
import { read_fact_bytes_from_disk, stream_fact_to_disk, write_fact_bytes_to_disk } from './fact_disk_storage.js';
|
|
33
43
|
/** Default embedded-vs-referenced cutoff (1 MiB). */
|
|
34
44
|
export const FACT_EMBEDDED_THRESHOLD_DEFAULT = 1024 * 1024;
|
|
35
|
-
/** Default fetcher backed by `globalThis.fetch`. */
|
|
36
|
-
export const create_default_fetcher = () => ({
|
|
37
|
-
fetch_stream: async (url) => {
|
|
38
|
-
const response = await fetch(url);
|
|
39
|
-
if (!response.ok) {
|
|
40
|
-
throw new Error(`fact fetch failed: ${response.status} ${url}`);
|
|
41
|
-
}
|
|
42
|
-
if (!response.body) {
|
|
43
|
-
throw new Error(`fact fetch returned no body: ${url}`);
|
|
44
|
-
}
|
|
45
|
-
return response.body;
|
|
46
|
-
},
|
|
47
|
-
fetch_bytes: async (url) => {
|
|
48
|
-
const response = await fetch(url);
|
|
49
|
-
if (!response.ok) {
|
|
50
|
-
throw new Error(`fact fetch failed: ${response.status} ${url}`);
|
|
51
|
-
}
|
|
52
|
-
return new Uint8Array(await response.arrayBuffer());
|
|
53
|
-
}
|
|
54
|
-
});
|
|
55
45
|
/**
|
|
56
46
|
* PG-backed `FactStore`. Delegates to `db/fact_queries.ts` for I/O and adds
|
|
57
47
|
* the lifecycle layer described in the module doc.
|
|
@@ -61,26 +51,20 @@ export class PgFactStore {
|
|
|
61
51
|
#embedded_threshold;
|
|
62
52
|
#disk_root;
|
|
63
53
|
#fs;
|
|
64
|
-
#fetcher;
|
|
65
54
|
#log;
|
|
66
55
|
constructor(options) {
|
|
67
56
|
this.#deps = options.deps;
|
|
68
57
|
this.#embedded_threshold = options.embedded_threshold ?? FACT_EMBEDDED_THRESHOLD_DEFAULT;
|
|
69
58
|
this.#disk_root = options.disk_root;
|
|
70
59
|
this.#fs = options.fs;
|
|
71
|
-
this.#fetcher =
|
|
72
|
-
options.fetcher ??
|
|
73
|
-
(options.disk_root !== undefined && options.fs !== undefined
|
|
74
|
-
? create_disk_fact_fetcher(options.fs, options.disk_root)
|
|
75
|
-
: create_default_fetcher());
|
|
76
60
|
this.#log = options.log;
|
|
77
61
|
}
|
|
78
62
|
/**
|
|
79
63
|
* Store fully-buffered bytes, routing by size: `<= embedded_threshold` into
|
|
80
64
|
* the PG `bytes` column; larger into the disk CAS (when `disk_root` + `fs`
|
|
81
65
|
* are configured) at `<facts_dir>/<shard>/<rest>` with a `file:` URL. Oversize
|
|
82
|
-
* without a disk root throws
|
|
83
|
-
*
|
|
66
|
+
* without a disk root throws: the store has nowhere to put bytes it cannot
|
|
67
|
+
* embed. Idempotent — `ON CONFLICT DO NOTHING` + content-addressed disk
|
|
84
68
|
* filenames make a re-write a no-op.
|
|
85
69
|
*/
|
|
86
70
|
async put(bytes, options) {
|
|
@@ -89,7 +73,7 @@ export class PgFactStore {
|
|
|
89
73
|
let row_external_url;
|
|
90
74
|
if (bytes.length > this.#embedded_threshold) {
|
|
91
75
|
if (this.#disk_root === undefined || this.#fs === undefined) {
|
|
92
|
-
throw new Error(`fact bytes exceed embedded threshold (${bytes.length} > ${this.#embedded_threshold});
|
|
76
|
+
throw new Error(`fact bytes exceed embedded threshold (${bytes.length} > ${this.#embedded_threshold}) and no disk CAS is configured; set disk_root and fs to store them on disk`);
|
|
93
77
|
}
|
|
94
78
|
row_bytes = null;
|
|
95
79
|
row_external_url = await write_fact_bytes_to_disk(this.#fs, this.#disk_root, hash, bytes);
|
|
@@ -147,33 +131,14 @@ export class PgFactStore {
|
|
|
147
131
|
return { hash: streamed.hash, sha256: streamed.sha256, size: streamed.size };
|
|
148
132
|
}
|
|
149
133
|
/**
|
|
150
|
-
*
|
|
151
|
-
*
|
|
152
|
-
*
|
|
153
|
-
*
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
if (byte_count !== size) {
|
|
159
|
-
throw new Error(`fact size mismatch for ${url}: caller declared ${size}, streamed ${byte_count}`);
|
|
160
|
-
}
|
|
161
|
-
const inserted = await query_put_fact(this.#deps, {
|
|
162
|
-
hash,
|
|
163
|
-
bytes: null,
|
|
164
|
-
external_url: url,
|
|
165
|
-
content_type: options?.content_type ?? null,
|
|
166
|
-
size
|
|
167
|
-
});
|
|
168
|
-
if (inserted && options?.refs && options.refs.length > 0) {
|
|
169
|
-
await query_put_fact_refs(this.#deps, hash, options.refs);
|
|
170
|
-
}
|
|
171
|
-
return hash;
|
|
172
|
-
}
|
|
173
|
-
/**
|
|
174
|
-
* Retrieve bytes. Embedded reads return PG bytes directly; external
|
|
175
|
-
* reads fetch + verify and return `null` (with a warning log) when
|
|
176
|
-
* the bytes don't match the stored hash.
|
|
134
|
+
* Retrieve bytes. Embedded reads return PG bytes directly. An external
|
|
135
|
+
* read comes from the disk CAS and nowhere else: the bytes at the row's
|
|
136
|
+
* `file:<shard>/<rest>` URL are read and verified against the hash.
|
|
137
|
+
*
|
|
138
|
+
* Returns `null`, with a warning log, for an external row the store cannot
|
|
139
|
+
* serve — an `external_url` that is not a `file:<shard>/<rest>` URL, no disk
|
|
140
|
+
* CAS configured (`disk_root` + `fs`), a failed disk read, or bytes that
|
|
141
|
+
* don't match the hash. Never makes a network request.
|
|
177
142
|
*/
|
|
178
143
|
async get(hash) {
|
|
179
144
|
const row = await query_get_fact(this.#deps, hash);
|
|
@@ -185,12 +150,22 @@ export class PgFactStore {
|
|
|
185
150
|
if (row.external_url === null) {
|
|
186
151
|
return null;
|
|
187
152
|
}
|
|
153
|
+
// the URL is not echoed — the column is free text, so its shape and
|
|
154
|
+
// length are whatever the row's writer chose
|
|
155
|
+
if (!is_file_fact_url(row.external_url)) {
|
|
156
|
+
this.#log?.warn(`PgFactStore.get external_url for ${hash} is not a file:<shard>/<rest> URL; treating as not-found`);
|
|
157
|
+
return null;
|
|
158
|
+
}
|
|
159
|
+
if (this.#disk_root === undefined || this.#fs === undefined) {
|
|
160
|
+
this.#log?.warn(`PgFactStore.get ${hash} is disk-backed and no disk CAS is configured (disk_root + fs); treating as not-found`);
|
|
161
|
+
return null;
|
|
162
|
+
}
|
|
188
163
|
let bytes;
|
|
189
164
|
try {
|
|
190
|
-
bytes = await this.#
|
|
165
|
+
bytes = await read_fact_bytes_from_disk(this.#fs, this.#disk_root, row.external_url);
|
|
191
166
|
}
|
|
192
167
|
catch (err) {
|
|
193
|
-
this.#log?.warn(`PgFactStore.get
|
|
168
|
+
this.#log?.warn(`PgFactStore.get disk read failed for ${hash} at ${row.external_url}:`, to_error_message(err));
|
|
194
169
|
return null;
|
|
195
170
|
}
|
|
196
171
|
if (!fact_hash_verify(hash, bytes)) {
|
|
@@ -226,11 +201,11 @@ export class PgFactStore {
|
|
|
226
201
|
* NOT verify the fact is unreferenced — that policy lives one layer
|
|
227
202
|
* up (the orphan-fact admin surface in the consumer; a future GC walker).
|
|
228
203
|
*
|
|
229
|
-
*
|
|
230
|
-
*
|
|
231
|
-
*
|
|
232
|
-
*
|
|
233
|
-
*
|
|
204
|
+
* The disk file of a disk-backed fact is NOT unlinked — that is the
|
|
205
|
+
* caller's responsibility, using the returned `external_url` (when
|
|
206
|
+
* non-null): a `file:<shard>/<rest>` URL names
|
|
207
|
+
* `<disk_root>/<shard>/<rest>`. The value is the row's free text — parse
|
|
208
|
+
* it with `parse_file_fact_url` before joining a path.
|
|
234
209
|
*
|
|
235
210
|
* @returns `{size, external_url}` for the deleted row, or `null` if
|
|
236
211
|
* no row matched the hash.
|
|
@@ -260,19 +235,6 @@ const resolve_refs = (bytes, options) => {
|
|
|
260
235
|
}
|
|
261
236
|
return fact_hash_extract_refs(value);
|
|
262
237
|
};
|
|
263
|
-
/** Hash a stream while counting bytes. Lets `put_ref` verify size in one pass. */
|
|
264
|
-
const hash_counted_stream = async (stream) => {
|
|
265
|
-
let byte_count = 0;
|
|
266
|
-
const counting = new TransformStream({
|
|
267
|
-
transform(chunk, controller) {
|
|
268
|
-
byte_count += chunk.length;
|
|
269
|
-
controller.enqueue(chunk);
|
|
270
|
-
}
|
|
271
|
-
});
|
|
272
|
-
const piped = stream.pipeThrough(counting);
|
|
273
|
-
const hash = await fact_hash_stream(piped);
|
|
274
|
-
return { hash, byte_count };
|
|
275
|
-
};
|
|
276
238
|
/**
|
|
277
239
|
* Coerce whatever the driver returns for BYTEA into a `Uint8Array`.
|
|
278
240
|
*
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Canonical filesystem-fact URL shape + on-disk layout.
|
|
3
3
|
*
|
|
4
|
-
* `external_url` on the
|
|
5
|
-
* `
|
|
6
|
-
*
|
|
4
|
+
* `external_url` on the `fact` row is free text (`string | null`), but the
|
|
5
|
+
* only shape `PgFactStore` writes, reads, or serves is the filesystem-minted
|
|
6
|
+
* one — any other reads as unavailable. Those URLs are exactly
|
|
7
7
|
* `file:<shard>/<rest>` where `<shard>` is the first 2 hex chars of the
|
|
8
8
|
* blake3 digest and `<rest>` the remaining 62 — files land at
|
|
9
9
|
* `<facts_dir>/<shard>/<rest>` after the writer atomically temp+renames
|
|
@@ -11,9 +11,9 @@
|
|
|
11
11
|
*
|
|
12
12
|
* Centralizing the regex + the `fact_disk_path` split keeps the shape in one
|
|
13
13
|
* place: `PgFactStore`'s disk CAS (`db/fact_disk_storage.ts`), the
|
|
14
|
-
* `serve_fact_route` defense-in-depth check, and the
|
|
15
|
-
*
|
|
16
|
-
* can't drift. The TS twin of the Rust `fact_disk_path` (`fuz_fact`).
|
|
14
|
+
* `serve_fact_route` defense-in-depth check, and the disk read
|
|
15
|
+
* (`read_fact_bytes_from_disk`) all derive the layout here, so the write path
|
|
16
|
+
* and the read path can't drift. The TS twin of the Rust `fact_disk_path` (`fuz_fact`).
|
|
17
17
|
*
|
|
18
18
|
* Defense-in-depth: a `..` segment can't match (`.` isn't in `[0-9a-f]`),
|
|
19
19
|
* neither can absolute paths, query strings, or any non-hex character.
|
package/dist/db/file_fact_url.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Canonical filesystem-fact URL shape + on-disk layout.
|
|
3
3
|
*
|
|
4
|
-
* `external_url` on the
|
|
5
|
-
* `
|
|
6
|
-
*
|
|
4
|
+
* `external_url` on the `fact` row is free text (`string | null`), but the
|
|
5
|
+
* only shape `PgFactStore` writes, reads, or serves is the filesystem-minted
|
|
6
|
+
* one — any other reads as unavailable. Those URLs are exactly
|
|
7
7
|
* `file:<shard>/<rest>` where `<shard>` is the first 2 hex chars of the
|
|
8
8
|
* blake3 digest and `<rest>` the remaining 62 — files land at
|
|
9
9
|
* `<facts_dir>/<shard>/<rest>` after the writer atomically temp+renames
|
|
@@ -11,9 +11,9 @@
|
|
|
11
11
|
*
|
|
12
12
|
* Centralizing the regex + the `fact_disk_path` split keeps the shape in one
|
|
13
13
|
* place: `PgFactStore`'s disk CAS (`db/fact_disk_storage.ts`), the
|
|
14
|
-
* `serve_fact_route` defense-in-depth check, and the
|
|
15
|
-
*
|
|
16
|
-
* can't drift. The TS twin of the Rust `fact_disk_path` (`fuz_fact`).
|
|
14
|
+
* `serve_fact_route` defense-in-depth check, and the disk read
|
|
15
|
+
* (`read_fact_bytes_from_disk`) all derive the layout here, so the write path
|
|
16
|
+
* and the read path can't drift. The TS twin of the Rust `fact_disk_path` (`fuz_fact`).
|
|
17
17
|
*
|
|
18
18
|
* Defense-in-depth: a `..` segment can't match (`.` isn't in `[0-9a-f]`),
|
|
19
19
|
* neither can absolute paths, query strings, or any non-hex character.
|
|
@@ -13,9 +13,9 @@
|
|
|
13
13
|
* Embedded facts stream from the `fact.bytes` PG column; external facts
|
|
14
14
|
* (filesystem-backed `file:<shard>/<rest>` URLs) either return an
|
|
15
15
|
* `X-Accel-Redirect` header pointing into nginx's internal facts location
|
|
16
|
-
* (production) or stream from disk
|
|
17
|
-
*
|
|
18
|
-
*
|
|
16
|
+
* (production) or stream from disk (dev / tests). The runtime mode is
|
|
17
|
+
* selected by the optional `x_accel` factory option — a validated
|
|
18
|
+
* `XAccelConfig` (built via
|
|
19
19
|
* `create_x_accel_config` in `server/x_accel.ts`) set in prod, unset in dev.
|
|
20
20
|
* The handle can only be obtained by proving the facts nginx location is
|
|
21
21
|
* `internal;`, so X-Accel serving can't be enabled against a public location.
|
|
@@ -80,10 +80,13 @@
|
|
|
80
80
|
*
|
|
81
81
|
* ## Defense-in-depth
|
|
82
82
|
*
|
|
83
|
-
*
|
|
84
|
-
* `X-Accel-Redirect
|
|
85
|
-
*
|
|
86
|
-
*
|
|
83
|
+
* `external_url` is re-validated against the `file:<shard>/<rest>` shape
|
|
84
|
+
* before it addresses the filesystem or an `X-Accel-Redirect`. `PgFactStore`
|
|
85
|
+
* writes only that shape, but the column is free text and `query_put_fact`
|
|
86
|
+
* inserts what it is given, so a row written past the store — or a
|
|
87
|
+
* row-injection bug upstream — would otherwise hand nginx an
|
|
88
|
+
* attacker-controlled path. Any other shape is a logged 404, the same
|
|
89
|
+
* unavailable that `PgFactStore.get` reports for it as `null`.
|
|
87
90
|
*
|
|
88
91
|
* @module
|
|
89
92
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"serve_fact_route.d.ts","sourceRoot":"../src/lib/","sources":["../../src/lib/server/serve_fact_route.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"serve_fact_route.d.ts","sourceRoot":"../src/lib/","sources":["../../src/lib/server/serve_fact_route.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2FG;AAMH,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAetD,OAAO,EAAuC,KAAK,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAE5F,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAM/C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAkEjD,MAAM,WAAW,+BAA+B;IAC/C;;;;;;OAMG;IACH,IAAI,EAAE,OAAO,CAAC;IACd,kFAAkF;IAClF,SAAS,EAAE,MAAM,CAAC;IAClB;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,YAAY,CAAC;IACvB,GAAG,EAAE,MAAM,CAAC;CACZ;AAkHD;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,iCAAiC,GAC7C,SAAS,+BAA+B,KACtC,SAgDF,CAAC;AAEF;;;;;;;;;;;;;;;;;GAiBG;AACH,eAAO,MAAM,4BAA4B,GACxC,SAAS,+BAA+B,KACtC,SA2CF,CAAC"}
|
|
@@ -13,9 +13,9 @@
|
|
|
13
13
|
* Embedded facts stream from the `fact.bytes` PG column; external facts
|
|
14
14
|
* (filesystem-backed `file:<shard>/<rest>` URLs) either return an
|
|
15
15
|
* `X-Accel-Redirect` header pointing into nginx's internal facts location
|
|
16
|
-
* (production) or stream from disk
|
|
17
|
-
*
|
|
18
|
-
*
|
|
16
|
+
* (production) or stream from disk (dev / tests). The runtime mode is
|
|
17
|
+
* selected by the optional `x_accel` factory option — a validated
|
|
18
|
+
* `XAccelConfig` (built via
|
|
19
19
|
* `create_x_accel_config` in `server/x_accel.ts`) set in prod, unset in dev.
|
|
20
20
|
* The handle can only be obtained by proving the facts nginx location is
|
|
21
21
|
* `internal;`, so X-Accel serving can't be enabled against a public location.
|
|
@@ -80,10 +80,13 @@
|
|
|
80
80
|
*
|
|
81
81
|
* ## Defense-in-depth
|
|
82
82
|
*
|
|
83
|
-
*
|
|
84
|
-
* `X-Accel-Redirect
|
|
85
|
-
*
|
|
86
|
-
*
|
|
83
|
+
* `external_url` is re-validated against the `file:<shard>/<rest>` shape
|
|
84
|
+
* before it addresses the filesystem or an `X-Accel-Redirect`. `PgFactStore`
|
|
85
|
+
* writes only that shape, but the column is free text and `query_put_fact`
|
|
86
|
+
* inserts what it is given, so a row written past the store — or a
|
|
87
|
+
* row-injection bug upstream — would otherwise hand nginx an
|
|
88
|
+
* attacker-controlled path. Any other shape is a logged 404, the same
|
|
89
|
+
* unavailable that `PgFactStore.get` reports for it as `null`.
|
|
87
90
|
*
|
|
88
91
|
* @module
|
|
89
92
|
*/
|
|
@@ -202,7 +205,7 @@ const serve_fact_bytes = async (c, route, hash, config) => {
|
|
|
202
205
|
// to address the filesystem.
|
|
203
206
|
const parsed = parse_file_fact_url(meta.external_url);
|
|
204
207
|
if (!parsed) {
|
|
205
|
-
log.error(`serve_fact:
|
|
208
|
+
log.error(`serve_fact: external_url for ${hash} is not a file:<shard>/<rest> URL; answering 404`);
|
|
206
209
|
return c.body(null, 404);
|
|
207
210
|
}
|
|
208
211
|
const { shard, rest } = parsed;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fuzdev/fuz_app",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.121.0",
|
|
4
4
|
"description": "fullstack app library",
|
|
5
5
|
"glyph": "🗝",
|
|
6
6
|
"logo": "logo.svg",
|
|
@@ -33,7 +33,7 @@
|
|
|
33
33
|
"peerDependencies": {
|
|
34
34
|
"@electric-sql/pglite": ">=0.4",
|
|
35
35
|
"@fuzdev/blake3-wasm": ">=0.2.0",
|
|
36
|
-
"@fuzdev/fuz_util": ">=0.
|
|
36
|
+
"@fuzdev/fuz_util": ">=0.70.0",
|
|
37
37
|
"@hono/node-server": ">=1",
|
|
38
38
|
"@hono/node-ws": ">=1",
|
|
39
39
|
"@node-rs/argon2": ">=2",
|
|
@@ -77,7 +77,7 @@
|
|
|
77
77
|
"@fuzdev/fuz_code": "^0.48.0",
|
|
78
78
|
"@fuzdev/fuz_css": "^0.64.0",
|
|
79
79
|
"@fuzdev/fuz_ui": "^0.208.0",
|
|
80
|
-
"@fuzdev/fuz_util": "^0.
|
|
80
|
+
"@fuzdev/fuz_util": "^0.70.0",
|
|
81
81
|
"@fuzdev/gro": "^0.213.0",
|
|
82
82
|
"@fuzdev/mdz": "^0.3.0",
|
|
83
83
|
"@hono/node-server": "^1.19.14",
|
|
@@ -1,42 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Filesystem-backed `FactExternalFetcher`.
|
|
3
|
-
*
|
|
4
|
-
* Resolves `file:<shard>/<rest>` external URLs against a configured
|
|
5
|
-
* facts directory. The URL shape is the canonical relative-path scheme:
|
|
6
|
-
* 2 hex chars for the shard subdir + 62 hex chars for the rest of the
|
|
7
|
-
* blake3 hash (64 hex chars total). Files live at
|
|
8
|
-
* `<facts_dir>/<shard>/<rest>` after the writer atomically temp+renames
|
|
9
|
-
* them in.
|
|
10
|
-
*
|
|
11
|
-
* **Pre-filter regex**: `^file:[0-9a-f]{2}/[0-9a-f]{62}$`. A `..`
|
|
12
|
-
* segment can't match (`.` isn't in `[0-9a-f]`); neither can absolute
|
|
13
|
-
* paths, query strings, or any non-hex character. Defense-in-depth in
|
|
14
|
-
* front of `path.join` — the fetcher never trusts the URL came from a
|
|
15
|
-
* fact row, even though it always does in practice.
|
|
16
|
-
*
|
|
17
|
-
* The fetcher does NOT verify hash content — `PgFactStore.get` calls
|
|
18
|
-
* `fact_hash_verify(hash, bytes)` after the fetch and returns null on
|
|
19
|
-
* mismatch.
|
|
20
|
-
*
|
|
21
|
-
* Runtime: uses `node:fs/promises` + `node:fs` `createReadStream` so
|
|
22
|
-
* the same code works under Deno (via node compat) and vitest.
|
|
23
|
-
*
|
|
24
|
-
* @module
|
|
25
|
-
*/
|
|
26
|
-
import type { FactExternalFetcher } from '../db/fact_store.js';
|
|
27
|
-
/** Construction options. */
|
|
28
|
-
export interface FileFactFetcherOptions {
|
|
29
|
-
/**
|
|
30
|
-
* Absolute path to the facts directory. Files resolve to
|
|
31
|
-
* `<facts_dir>/<shard>/<rest>`.
|
|
32
|
-
*/
|
|
33
|
-
facts_dir: string;
|
|
34
|
-
}
|
|
35
|
-
/**
|
|
36
|
-
* Build a `FactExternalFetcher` that resolves `file:` URLs against the
|
|
37
|
-
* filesystem. Throws on a malformed URL before touching the disk so
|
|
38
|
-
* `PgFactStore.get` logs the warning + returns null without an I/O
|
|
39
|
-
* round-trip on bad data.
|
|
40
|
-
*/
|
|
41
|
-
export declare const create_file_fact_fetcher: (options: FileFactFetcherOptions) => FactExternalFetcher;
|
|
42
|
-
//# sourceMappingURL=file_fact_fetcher.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"file_fact_fetcher.d.ts","sourceRoot":"../src/lib/","sources":["../../src/lib/server/file_fact_fetcher.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAOH,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAG/D,4BAA4B;AAC5B,MAAM,WAAW,sBAAsB;IACtC;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;CAClB;AAED;;;;;GAKG;AACH,eAAO,MAAM,wBAAwB,GAAI,SAAS,sBAAsB,KAAG,mBA0B1E,CAAC"}
|
|
@@ -1,60 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Filesystem-backed `FactExternalFetcher`.
|
|
3
|
-
*
|
|
4
|
-
* Resolves `file:<shard>/<rest>` external URLs against a configured
|
|
5
|
-
* facts directory. The URL shape is the canonical relative-path scheme:
|
|
6
|
-
* 2 hex chars for the shard subdir + 62 hex chars for the rest of the
|
|
7
|
-
* blake3 hash (64 hex chars total). Files live at
|
|
8
|
-
* `<facts_dir>/<shard>/<rest>` after the writer atomically temp+renames
|
|
9
|
-
* them in.
|
|
10
|
-
*
|
|
11
|
-
* **Pre-filter regex**: `^file:[0-9a-f]{2}/[0-9a-f]{62}$`. A `..`
|
|
12
|
-
* segment can't match (`.` isn't in `[0-9a-f]`); neither can absolute
|
|
13
|
-
* paths, query strings, or any non-hex character. Defense-in-depth in
|
|
14
|
-
* front of `path.join` — the fetcher never trusts the URL came from a
|
|
15
|
-
* fact row, even though it always does in practice.
|
|
16
|
-
*
|
|
17
|
-
* The fetcher does NOT verify hash content — `PgFactStore.get` calls
|
|
18
|
-
* `fact_hash_verify(hash, bytes)` after the fetch and returns null on
|
|
19
|
-
* mismatch.
|
|
20
|
-
*
|
|
21
|
-
* Runtime: uses `node:fs/promises` + `node:fs` `createReadStream` so
|
|
22
|
-
* the same code works under Deno (via node compat) and vitest.
|
|
23
|
-
*
|
|
24
|
-
* @module
|
|
25
|
-
*/
|
|
26
|
-
import { readFile } from 'node:fs/promises';
|
|
27
|
-
import { createReadStream } from 'node:fs';
|
|
28
|
-
import { Readable } from 'node:stream';
|
|
29
|
-
import { join } from 'node:path';
|
|
30
|
-
import { parse_file_fact_url } from '../db/file_fact_url.js';
|
|
31
|
-
/**
|
|
32
|
-
* Build a `FactExternalFetcher` that resolves `file:` URLs against the
|
|
33
|
-
* filesystem. Throws on a malformed URL before touching the disk so
|
|
34
|
-
* `PgFactStore.get` logs the warning + returns null without an I/O
|
|
35
|
-
* round-trip on bad data.
|
|
36
|
-
*/
|
|
37
|
-
export const create_file_fact_fetcher = (options) => {
|
|
38
|
-
const { facts_dir } = options;
|
|
39
|
-
const resolve_path = (url) => {
|
|
40
|
-
const parsed = parse_file_fact_url(url);
|
|
41
|
-
if (!parsed) {
|
|
42
|
-
throw new Error(`invalid file fact url: ${url}`);
|
|
43
|
-
}
|
|
44
|
-
return join(facts_dir, parsed.shard, parsed.rest);
|
|
45
|
-
};
|
|
46
|
-
return {
|
|
47
|
-
fetch_bytes: async (url) => {
|
|
48
|
-
const path = resolve_path(url);
|
|
49
|
-
const buf = await readFile(path);
|
|
50
|
-
return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength);
|
|
51
|
-
},
|
|
52
|
-
// `Promise.resolve().then(...)` (rather than `async`) funnels any
|
|
53
|
-
// `resolve_path` throw into a rejection without an unused `await`.
|
|
54
|
-
fetch_stream: (url) => Promise.resolve().then(() => {
|
|
55
|
-
const path = resolve_path(url);
|
|
56
|
-
const node_stream = createReadStream(path);
|
|
57
|
-
return Readable.toWeb(node_stream);
|
|
58
|
-
})
|
|
59
|
-
};
|
|
60
|
-
};
|