@factoidal/core 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. package/CHANGELOG.md +79 -0
  2. package/NOTICE +31 -0
  3. package/README.md +35 -1
  4. package/bin/factoidal.mjs +250 -14
  5. package/bin/pack-host.mjs +341 -0
  6. package/bin/pack-worker.mjs +31 -0
  7. package/bin/pack.mjs +174 -0
  8. package/bin/store.mjs +32 -0
  9. package/l4-assets/l4factoidal.js +61 -1
  10. package/l4-assets/l4factoidal.mjs +1 -1
  11. package/l4-assets/l4factoidal.wasm +0 -0
  12. package/l4-assets/version.json +4 -4
  13. package/package.json +10 -2
  14. package/sample-store/CURRENT +1 -0
  15. package/sample-store/gen-1/manifest.sbm2 +0 -0
  16. package/sample-store/gen-1/manifest.tsv +14 -0
  17. package/sample-store/gen-1/predicate-0.ibk3 +0 -0
  18. package/sample-store/gen-1/predicate-0.ibk3.merkle +1 -0
  19. package/sample-store/gen-1/predicate-0.ibk3.oli2 +0 -0
  20. package/sample-store/gen-1/predicate-0.ibk3.oli2.merkle +1 -0
  21. package/sample-store/gen-1/predicate-0.ibk3.sri2 +0 -0
  22. package/sample-store/gen-1/predicate-0.ibk3.sri2.merkle +1 -0
  23. package/sample-store/gen-1/predicate-0.ibk3.tli1 +0 -0
  24. package/sample-store/gen-1/predicate-0.ibk3.tli1.merkle +0 -0
  25. package/sample-store/gen-1/predicate-1.ibk3 +0 -0
  26. package/sample-store/gen-1/predicate-1.ibk3.merkle +0 -0
  27. package/sample-store/gen-1/predicate-1.ibk3.oli2 +0 -0
  28. package/sample-store/gen-1/predicate-1.ibk3.oli2.merkle +0 -0
  29. package/sample-store/gen-1/predicate-1.ibk3.sri2 +0 -0
  30. package/sample-store/gen-1/predicate-1.ibk3.sri2.merkle +1 -0
  31. package/sample-store/gen-1/predicate-1.ibk3.tli1 +0 -0
  32. package/sample-store/gen-1/predicate-1.ibk3.tli1.merkle +1 -0
  33. package/sample-store/gen-1/predicate-10.ibk3 +0 -0
  34. package/sample-store/gen-1/predicate-10.ibk3.merkle +1 -0
  35. package/sample-store/gen-1/predicate-10.ibk3.oli2 +0 -0
  36. package/sample-store/gen-1/predicate-10.ibk3.oli2.merkle +0 -0
  37. package/sample-store/gen-1/predicate-10.ibk3.sri2 +0 -0
  38. package/sample-store/gen-1/predicate-10.ibk3.sri2.merkle +1 -0
  39. package/sample-store/gen-1/predicate-10.ibk3.tli1 +0 -0
  40. package/sample-store/gen-1/predicate-10.ibk3.tli1.merkle +1 -0
  41. package/sample-store/gen-1/predicate-11.ibk3 +0 -0
  42. package/sample-store/gen-1/predicate-11.ibk3.merkle +0 -0
  43. package/sample-store/gen-1/predicate-11.ibk3.oli2 +0 -0
  44. package/sample-store/gen-1/predicate-11.ibk3.oli2.merkle +1 -0
  45. package/sample-store/gen-1/predicate-11.ibk3.sri2 +0 -0
  46. package/sample-store/gen-1/predicate-11.ibk3.sri2.merkle +1 -0
  47. package/sample-store/gen-1/predicate-11.ibk3.tli1 +0 -0
  48. package/sample-store/gen-1/predicate-11.ibk3.tli1.merkle +1 -0
  49. package/sample-store/gen-1/predicate-12.ibk3 +0 -0
  50. package/sample-store/gen-1/predicate-12.ibk3.merkle +1 -0
  51. package/sample-store/gen-1/predicate-12.ibk3.oli2 +0 -0
  52. package/sample-store/gen-1/predicate-12.ibk3.oli2.merkle +1 -0
  53. package/sample-store/gen-1/predicate-12.ibk3.sri2 +0 -0
  54. package/sample-store/gen-1/predicate-12.ibk3.sri2.merkle +1 -0
  55. package/sample-store/gen-1/predicate-12.ibk3.tli1 +0 -0
  56. package/sample-store/gen-1/predicate-12.ibk3.tli1.merkle +1 -0
  57. package/sample-store/gen-1/predicate-2.ibk3 +0 -0
  58. package/sample-store/gen-1/predicate-2.ibk3.merkle +1 -0
  59. package/sample-store/gen-1/predicate-2.ibk3.oli2 +0 -0
  60. package/sample-store/gen-1/predicate-2.ibk3.oli2.merkle +1 -0
  61. package/sample-store/gen-1/predicate-2.ibk3.sri2 +0 -0
  62. package/sample-store/gen-1/predicate-2.ibk3.sri2.merkle +0 -0
  63. package/sample-store/gen-1/predicate-2.ibk3.tli1 +0 -0
  64. package/sample-store/gen-1/predicate-2.ibk3.tli1.merkle +0 -0
  65. package/sample-store/gen-1/predicate-3.ibk3 +0 -0
  66. package/sample-store/gen-1/predicate-3.ibk3.merkle +1 -0
  67. package/sample-store/gen-1/predicate-3.ibk3.oli2 +0 -0
  68. package/sample-store/gen-1/predicate-3.ibk3.oli2.merkle +1 -0
  69. package/sample-store/gen-1/predicate-3.ibk3.sri2 +0 -0
  70. package/sample-store/gen-1/predicate-3.ibk3.sri2.merkle +1 -0
  71. package/sample-store/gen-1/predicate-3.ibk3.tli1 +0 -0
  72. package/sample-store/gen-1/predicate-3.ibk3.tli1.merkle +1 -0
  73. package/sample-store/gen-1/predicate-4.ibk3 +0 -0
  74. package/sample-store/gen-1/predicate-4.ibk3.merkle +0 -0
  75. package/sample-store/gen-1/predicate-4.ibk3.oli2 +0 -0
  76. package/sample-store/gen-1/predicate-4.ibk3.oli2.merkle +1 -0
  77. package/sample-store/gen-1/predicate-4.ibk3.sri2 +0 -0
  78. package/sample-store/gen-1/predicate-4.ibk3.sri2.merkle +1 -0
  79. package/sample-store/gen-1/predicate-4.ibk3.tli1 +0 -0
  80. package/sample-store/gen-1/predicate-4.ibk3.tli1.merkle +2 -0
  81. package/sample-store/gen-1/predicate-5.ibk3 +0 -0
  82. package/sample-store/gen-1/predicate-5.ibk3.merkle +1 -0
  83. package/sample-store/gen-1/predicate-5.ibk3.oli2 +0 -0
  84. package/sample-store/gen-1/predicate-5.ibk3.oli2.merkle +1 -0
  85. package/sample-store/gen-1/predicate-5.ibk3.sri2 +0 -0
  86. package/sample-store/gen-1/predicate-5.ibk3.sri2.merkle +1 -0
  87. package/sample-store/gen-1/predicate-5.ibk3.tli1 +0 -0
  88. package/sample-store/gen-1/predicate-5.ibk3.tli1.merkle +1 -0
  89. package/sample-store/gen-1/predicate-6.ibk3 +0 -0
  90. package/sample-store/gen-1/predicate-6.ibk3.merkle +1 -0
  91. package/sample-store/gen-1/predicate-6.ibk3.oli2 +0 -0
  92. package/sample-store/gen-1/predicate-6.ibk3.oli2.merkle +1 -0
  93. package/sample-store/gen-1/predicate-6.ibk3.sri2 +0 -0
  94. package/sample-store/gen-1/predicate-6.ibk3.sri2.merkle +1 -0
  95. package/sample-store/gen-1/predicate-6.ibk3.tli1 +0 -0
  96. package/sample-store/gen-1/predicate-6.ibk3.tli1.merkle +0 -0
  97. package/sample-store/gen-1/predicate-7.ibk3 +0 -0
  98. package/sample-store/gen-1/predicate-7.ibk3.merkle +0 -0
  99. package/sample-store/gen-1/predicate-7.ibk3.oli2 +0 -0
  100. package/sample-store/gen-1/predicate-7.ibk3.oli2.merkle +2 -0
  101. package/sample-store/gen-1/predicate-7.ibk3.sri2 +0 -0
  102. package/sample-store/gen-1/predicate-7.ibk3.sri2.merkle +1 -0
  103. package/sample-store/gen-1/predicate-7.ibk3.tli1 +0 -0
  104. package/sample-store/gen-1/predicate-7.ibk3.tli1.merkle +1 -0
  105. package/sample-store/gen-1/predicate-8.ibk3 +0 -0
  106. package/sample-store/gen-1/predicate-8.ibk3.merkle +1 -0
  107. package/sample-store/gen-1/predicate-8.ibk3.oli2 +0 -0
  108. package/sample-store/gen-1/predicate-8.ibk3.oli2.merkle +1 -0
  109. package/sample-store/gen-1/predicate-8.ibk3.sri2 +0 -0
  110. package/sample-store/gen-1/predicate-8.ibk3.sri2.merkle +1 -0
  111. package/sample-store/gen-1/predicate-8.ibk3.tli1 +0 -0
  112. package/sample-store/gen-1/predicate-8.ibk3.tli1.merkle +1 -0
  113. package/sample-store/gen-1/predicate-9.ibk3 +0 -0
  114. package/sample-store/gen-1/predicate-9.ibk3.merkle +1 -0
  115. package/sample-store/gen-1/predicate-9.ibk3.oli2 +0 -0
  116. package/sample-store/gen-1/predicate-9.ibk3.oli2.merkle +1 -0
  117. package/sample-store/gen-1/predicate-9.ibk3.sri2 +0 -0
  118. package/sample-store/gen-1/predicate-9.ibk3.sri2.merkle +3 -0
  119. package/sample-store/gen-1/predicate-9.ibk3.tli1 +0 -0
  120. package/sample-store/gen-1/predicate-9.ibk3.tli1.merkle +2 -0
  121. package/sample-store.d.ts +16 -0
  122. package/sample-store.mjs +37 -0
  123. package/store-host/deno.mjs +63 -0
  124. package/store-host/index.mjs +39 -0
  125. package/store-host/node.mjs +62 -1
  126. package/version.json +1 -1
@@ -0,0 +1,341 @@
1
+ // Giving `factoidal pack` a call stack big enough to finish, without
2
+ // asking the reader for a runtime flag.
3
+ // https://github.com/danbri/factoidal/issues/649
4
+ //
5
+ // THE DEFECT THIS FILE EXISTS FOR
6
+ // The pack fold in the engine recurses one frame per term in one block's
7
+ // local term index, and that is far deeper than a query recurses per row.
8
+ // On the DEFAULT stack of Node 22 and of Deno 2 the pack of anything
9
+ // above roughly 0.25 MB of Turtle ends with `Maximum call stack size
10
+ // exceeded` -- measured 2026-09-04 against the committed module, where
11
+ // `sequence_variant.ttl` (241,149 B) passes and `chromosome.ttl`
12
+ // (316,116 B) does not. Every one of those inputs packs correctly once
13
+ // the stack is raised, and the generation is byte-identical to
14
+ // `l4block-shard-pack` output. So the defect is the frame budget alone.
15
+ // The depth, what sets it, and why a browser cannot be given more are in
16
+ // `docs/designissues/2026-09-03-npm-pack-in-wasm.md`.
17
+ //
18
+ // WHAT THIS FILE IS ALLOWED TO DO
19
+ // Start the same pack on a thread or in a child process that has a
20
+ // bigger stack, carry the progress reports back, and re-raise the same
21
+ // error. It reads no RDF, decides no artifact name, and moves no
22
+ // artifact bytes: the worker writes the generation itself, through the
23
+ // same `store-host` primitives, exactly where the artifacts arrive.
24
+ // A reader must not be able to tell that a worker is involved, except
25
+ // that the pack finishes (iron rule 7 of CLAUDE.md).
26
+ //
27
+ // THE TWO ROUTES
28
+ // Node -- `worker_threads`, with `resourceLimits.stackSizeMb`. The
29
+ // engine loads INSIDE the worker: a loaded WebAssembly
30
+ // instance does not cross a thread boundary.
31
+ // Deno -- `worker_threads` is not the route, and a Deno `Worker` takes
32
+ // no stack size. The command re-executes itself once with
33
+ // `--v8-flags=--stack-size=...`, guarded by an environment
34
+ // variable so it can never loop. Confirmed honoured by
35
+ // deno 2.9.4 / V8 15.0.245.2 on 2026-09-04:
36
+ // `deno run -A --v8-flags=--stack-size=8000` packs
37
+ // `anatomical_structure.ttl` (3,811,378 B) in 13.7 s, where
38
+ // the same command without the flag overflows.
39
+ //
40
+ // WHAT IT DOES NOT FIX
41
+ // A browser tab has a fixed frame budget and no host flag, so neither
42
+ // route rescues an in-page packer. See the closing section of
43
+ // `docs/designissues/2026-09-03-npm-pack-in-wasm.md`.
44
+
45
+ import { StoreHostError, listGeneration, readWhole } from '../store-host/index.mjs'
46
+ import { fileUrlToPath, joinPath } from '../store-host/paths.mjs'
47
+ import { loadEngine } from './engine.mjs'
48
+ import { PackError, packFile, packSupported, verifyGeneration } from './pack.mjs'
49
+ import { openStore } from './store.mjs'
50
+
51
+ const isDeno = typeof globalThis.Deno !== 'undefined'
52
+
53
+ /**
54
+ * The worker's stack, in mebibytes.
55
+ *
56
+ * MEASURED, 2026-09-04, macOS 15 arm64, Node 22.22.2, against
57
+ * `examples/wikidata/subsets/lifesci-kgx/data/gene.ttl` -- 17,363,312
58
+ * bytes, 888,949 triples, the largest input the milestone carries:
59
+ *
60
+ * stackSizeMb verdict
61
+ * 4 (default) Maximum call stack size exceeded
62
+ * 6 Maximum call stack size exceeded
63
+ * 7 Maximum call stack size exceeded
64
+ * 8 pass, 42 s
65
+ * 10 pass
66
+ * 64 pass, 68 s
67
+ *
68
+ * 8 MiB is therefore the measured minimum, and 64 is eight times it.
69
+ * The headroom is deliberate and it is cheap: a thread stack is reserved
70
+ * address space, and only the pages actually touched become resident, so
71
+ * an unused 56 MiB costs nothing. The depth is set by the number of
72
+ * terms in ONE block's dictionary, not by the file size, so a file
73
+ * smaller than gene.ttl but with a wider dictionary needs more, and a
74
+ * value only just above the measured minimum would move the same defect
75
+ * to another input.
76
+ */
77
+ export const WORKER_STACK_MB = 64
78
+
79
+ /** The V8 stack limit the Deno re-exec asks for, in kibibytes -- the
80
+ * unit `--stack-size` takes. Same budget as WORKER_STACK_MB. */
81
+ export const REEXEC_STACK_KB = WORKER_STACK_MB * 1024
82
+
83
+ /** Set in the re-executed Deno child, so the guard can never loop. */
84
+ export const REEXEC_GUARD = 'FACTOIDAL_PACK_STACK_REEXEC'
85
+
86
+ /** Read one environment variable, or null where reading it is refused. */
87
+ function environment (name) {
88
+ try {
89
+ if (isDeno && globalThis.Deno.env) {
90
+ const value = globalThis.Deno.env.get(name)
91
+ return typeof value === 'string' && value.length > 0 ? value : null
92
+ }
93
+ } catch (_error) {
94
+ // Deno without --allow-env. An absent override is the normal case.
95
+ return null
96
+ }
97
+ const value = globalThis.process && globalThis.process.env
98
+ ? globalThis.process.env[name]
99
+ : undefined
100
+ return typeof value === 'string' && value.length > 0 ? value : null
101
+ }
102
+
103
+ /** The stack the worker asks for: `WORKER_STACK_MB`, or the override in
104
+ * FACTOIDAL_PACK_STACK_MB, which exists so the measurement above can be
105
+ * repeated without editing this file. */
106
+ export function workerStackMb () {
107
+ const override = environment('FACTOIDAL_PACK_STACK_MB')
108
+ if (override === null) return WORKER_STACK_MB
109
+ const value = Number(override)
110
+ return Number.isSafeInteger(value) && value > 0 ? value : WORKER_STACK_MB
111
+ }
112
+
113
+ /** Whether the caller asked for the in-process path. `--no-worker` on the
114
+ * command line, or FACTOIDAL_NO_WORKER in the environment. The old
115
+ * behaviour stays testable, and a platform where the worker fails has a
116
+ * way out. */
117
+ export function workerRefused (options) {
118
+ if (options && options.worker === false) return true
119
+ return environment('FACTOIDAL_NO_WORKER') !== null
120
+ }
121
+
122
+ // ------------------------------------------------- carrying an error back
123
+ //
124
+ // `commandPack` distinguishes a PackError and a StoreHostError from
125
+ // everything else, and prints the engine's own words in each case. A
126
+ // worker boundary must not change any of that, so the failure crosses as
127
+ // a plain description and is rebuilt on the other side.
128
+
129
+ /** Describe a thrown value so it survives `postMessage`. */
130
+ export function describeError (error) {
131
+ if (error instanceof PackError) {
132
+ return {
133
+ type: 'PackError',
134
+ message: error.message,
135
+ handle: error.handle,
136
+ notWired: error.notWired
137
+ }
138
+ }
139
+ if (error instanceof StoreHostError) {
140
+ return {
141
+ type: 'StoreHostError',
142
+ message: error.message,
143
+ code: error.code,
144
+ path: error.path === undefined ? null : error.path
145
+ }
146
+ }
147
+ return {
148
+ type: 'Error',
149
+ message: error && error.message ? String(error.message) : String(error)
150
+ }
151
+ }
152
+
153
+ /** Rebuild what `describeError` described. */
154
+ export function reviveError (description) {
155
+ if (description.type === 'PackError') {
156
+ return new PackError(description.message,
157
+ { handle: description.handle, notWired: description.notWired })
158
+ }
159
+ if (description.type === 'StoreHostError') {
160
+ const detail = description.path === null ? {} : { path: description.path }
161
+ return new StoreHostError(description.code, description.message, detail)
162
+ }
163
+ return new Error(description.message)
164
+ }
165
+
166
+ /** Whether a failure is the runtime running out of call stack. */
167
+ export function isStackOverflow (error) {
168
+ if (error instanceof RangeError) return true
169
+ const message = error && error.message ? String(error.message) : String(error)
170
+ return message.indexOf('call stack size exceeded') >= 0
171
+ }
172
+
173
+ // ------------------------------------------------------ the pack itself
174
+
175
+ /**
176
+ * Load the engine and run one deep task, on whatever stack the caller has.
177
+ *
178
+ * This is the body both routes run: the worker calls it on its own big
179
+ * stack, and `--no-worker` calls it in the process it was started in.
180
+ *
181
+ * `task.kind` selects the work. Both kinds recurse once per distinct term
182
+ * in a block's local term index, so both need the raised stack -- the
183
+ * verification `activate` runs decodes the same blocks the pack encoded.
184
+ * `activate` overflowed on a 112,742-row generation that `pack` had just
185
+ * written successfully, which is how the two came to share this path
186
+ * (measured 2026-09-04, https://github.com/danbri/factoidal/issues/649).
187
+ *
188
+ * @returns {{notWired: true}|{report: object}}
189
+ */
190
+ export async function packHere (task, onProgress) {
191
+ const engine = await loadEngine()
192
+ if (!packSupported(engine)) return { notWired: true }
193
+ if (task.kind === 'activate') {
194
+ return { report: activateHere(engine, task) }
195
+ }
196
+ const report = packFile(engine, task.input, task.output, {
197
+ syntax: task.syntax,
198
+ layout: task.layout,
199
+ base: task.base,
200
+ onProgress
201
+ })
202
+ return { report }
203
+ }
204
+
205
+ /**
206
+ * Verify one generation and answer the engine's verdict.
207
+ *
208
+ * The host reads the manifest and every artifact and hands them over; the
209
+ * engine checks each against the digest the manifest commits. Replacing
210
+ * CURRENT is NOT done here -- it is the caller's step, and it happens on
211
+ * the main thread only after this returns ok, so a worker that dies
212
+ * mid-verification can never leave a half-moved pointer.
213
+ */
214
+ function activateHere (engine, task) {
215
+ const store = openStore(task.root, task.generation)
216
+ const files = listGeneration(store.generationDir)
217
+ const artifacts = files
218
+ .filter((file) => file.name !== store.manifestName)
219
+ .map((file) => ({
220
+ key: file.name,
221
+ bytes: readWhole(joinPath(store.generationDir, file.name))
222
+ }))
223
+ return verifyGeneration(engine, store.manifestHex, artifacts)
224
+ }
225
+
226
+ /**
227
+ * Pack on a `worker_threads` thread with a raised stack. Node only.
228
+ *
229
+ * The worker writes every artifact itself, so no artifact byte crosses
230
+ * the thread boundary; what crosses is the task, the progress reports
231
+ * and the finish envelope.
232
+ *
233
+ * @returns {{notWired: true}|{report: object}}
234
+ * @throws whatever the pack threw, rebuilt
235
+ */
236
+ async function packInWorker (task, onProgress) {
237
+ const { Worker } = await import('node:worker_threads')
238
+ const entry = new URL('./pack-worker.mjs', import.meta.url)
239
+ const stackSizeMb = workerStackMb()
240
+ return await new Promise((resolve, reject) => {
241
+ const worker = new Worker(entry, {
242
+ workerData: { ...task, progress: typeof onProgress === 'function' },
243
+ resourceLimits: { stackSizeMb },
244
+ stdout: false,
245
+ stderr: false
246
+ })
247
+ let settled = false
248
+ const finish = (act) => {
249
+ if (settled) return
250
+ settled = true
251
+ worker.terminate().then(act, act)
252
+ }
253
+ worker.on('message', (message) => {
254
+ if (message.kind === 'progress') {
255
+ if (typeof onProgress === 'function') onProgress(message.progress)
256
+ return
257
+ }
258
+ if (message.kind === 'error') {
259
+ const failure = reviveError(message.error)
260
+ finish(() => reject(failure))
261
+ return
262
+ }
263
+ finish(() => resolve(message.kind === 'notWired'
264
+ ? { notWired: true }
265
+ : { report: message.report }))
266
+ })
267
+ worker.on('error', (error) => { finish(() => reject(error)) })
268
+ worker.on('exit', (code) => {
269
+ if (settled) return
270
+ settled = true
271
+ reject(new PackError(`the pack worker exited with code ${code}`))
272
+ })
273
+ })
274
+ }
275
+
276
+ /**
277
+ * Run one deep task, on the biggest stack this runtime will give it.
278
+ *
279
+ * On Node the worker route is the default. On Deno the caller has
280
+ * already re-executed (see `denoReexec`), so this runs in process.
281
+ *
282
+ * @param {object} task {kind, ...}: a pack takes
283
+ * {input, output, syntax, layout, base}, an activate takes
284
+ * {root, generation}
285
+ * @param {(progress: object) => void} [onProgress]
286
+ * @param {object} [options] {worker: false} forces the in-process path
287
+ */
288
+ export async function runPack (task, onProgress, options) {
289
+ if (isDeno || workerRefused(options)) return await packHere(task, onProgress)
290
+ return await packInWorker(task, onProgress)
291
+ }
292
+
293
+ // -------------------------------------------------------- the Deno route
294
+
295
+ /**
296
+ * Re-execute this command once under Deno with a raised V8 stack.
297
+ *
298
+ * Returns null when the re-exec is not available or not wanted, and the
299
+ * caller then packs in process. Returns the child's exit code when the
300
+ * child ran, and the caller exits with it. Permissions are NOT
301
+ * escalated: the child is given exactly the ones this process was
302
+ * granted, queried rather than requested so no prompt appears.
303
+ */
304
+ export async function denoReexec (options) {
305
+ if (!isDeno) return null
306
+ if (workerRefused(options)) return null
307
+ if (environment(REEXEC_GUARD) !== null) return null
308
+ const Deno = globalThis.Deno
309
+ let granted
310
+ try {
311
+ granted = ['read', 'write', 'env', 'run', 'net', 'ffi', 'sys']
312
+ .filter((name) => Deno.permissions.querySync({ name }).state === 'granted')
313
+ } catch (_error) {
314
+ return null
315
+ }
316
+ if (granted.indexOf('run') < 0 || granted.indexOf('read') < 0) return null
317
+ const main = Deno.mainModule.startsWith('file://')
318
+ ? fileUrlToPath(Deno.mainModule)
319
+ : Deno.mainModule
320
+ const args = [
321
+ 'run',
322
+ ...granted.map((name) => `--allow-${name}`),
323
+ `--v8-flags=--stack-size=${REEXEC_STACK_KB}`,
324
+ main,
325
+ ...Deno.args
326
+ ]
327
+ try {
328
+ const child = new Deno.Command(Deno.execPath(), {
329
+ args,
330
+ env: { [REEXEC_GUARD]: '1' },
331
+ stdin: 'inherit',
332
+ stdout: 'inherit',
333
+ stderr: 'inherit'
334
+ }).outputSync()
335
+ return child.code
336
+ } catch (_error) {
337
+ // No --allow-run at this exact path, or no executable. Pack here and
338
+ // let the overflow advice speak if the stack runs out.
339
+ return null
340
+ }
341
+ }
@@ -0,0 +1,31 @@
1
+ // The thread `factoidal pack` runs on under Node, so that the pack has a
2
+ // call stack big enough to finish.
3
+ // https://github.com/danbri/factoidal/issues/649
4
+ //
5
+ // WHAT THIS FILE IS ALLOWED TO DO
6
+ // Load the engine, run the same pack the in-process path runs, post the
7
+ // progress reports back, and post the failure back. It reads no RDF,
8
+ // names no artifact and encodes nothing: `pack.mjs` drives the engine
9
+ // and `store-host` writes the bytes, here exactly as they do on the
10
+ // main thread (iron rule 7 of CLAUDE.md).
11
+ //
12
+ // The engine loads HERE, not on the main thread: a loaded WebAssembly
13
+ // instance does not cross a thread boundary. The artifacts are written
14
+ // HERE too, where they arrive, so no artifact byte crosses it either.
15
+
16
+ import { parentPort, workerData } from 'node:worker_threads'
17
+ import { describeError, packHere } from './pack-host.mjs'
18
+
19
+ const port = parentPort
20
+ const task = workerData
21
+
22
+ packHere(task, task.progress
23
+ ? (progress) => port.postMessage({ kind: 'progress', progress })
24
+ : undefined)
25
+ .then((answer) => {
26
+ if (answer.notWired === true) port.postMessage({ kind: 'notWired' })
27
+ else port.postMessage({ kind: 'report', report: answer.report })
28
+ })
29
+ .catch((error) => {
30
+ port.postMessage({ kind: 'error', error: describeError(error) })
31
+ })
package/bin/pack.mjs ADDED
@@ -0,0 +1,174 @@
1
+ // Building a Shardborough generation from a JavaScript host, with the
2
+ // Lean engine running as WebAssembly.
3
+ // https://github.com/danbri/factoidal/issues/641 stage 3.
4
+ // Design record: docs/designissues/2026-09-03-npm-pack-in-wasm.md.
5
+ //
6
+ // WHAT THIS FILE IS ALLOWED TO DO
7
+ // Read the input file in chunks, hand each chunk to the engine, take back
8
+ // whatever artifacts the engine says are ready, and write each one under
9
+ // the name the engine gave it. It decides no artifact name, computes no
10
+ // digest, encodes no block and writes no manifest field. Every one of
11
+ // those is a format decision and it lives in the Lean source (iron rule
12
+ // 7 of CLAUDE.md). A reviewer who finds a magic number, a field offset,
13
+ // a file-name pattern or a hash in this file has found a violation.
14
+ //
15
+ // There is no file I/O inside the wasm module -- libuv is left out of the
16
+ // Emscripten build by design -- so the engine cannot write the generation
17
+ // itself. That is the whole reason this file exists.
18
+ //
19
+ // WHY IT STREAMS
20
+ // wasm32 addresses 4 GiB. The native packer's own second pass is already
21
+ // bounded: it publishes a batch of blocks every 64 input chunks and then
22
+ // drops the triples. This host drives that same fold one chunk at a time
23
+ // and writes each artifact as it appears, so neither side ever holds the
24
+ // whole input or the whole generation.
25
+
26
+ import { StoreHostError, readChunk, writeNew } from '../store-host/index.mjs'
27
+ import { joinPath } from '../store-host/paths.mjs'
28
+
29
+ /** The chunk the host feeds per call. The native packer reads 65,536
30
+ * bytes at a time; matching it keeps the two folds in step, which is
31
+ * what makes the byte-identity gate meaningful. */
32
+ export const FEED_BYTES = 65536
33
+
34
+ /** An error the pack operations reported, with the engine's own words. */
35
+ export class PackError extends Error {
36
+ constructor (message, detail = {}) {
37
+ super(message)
38
+ this.name = 'PackError'
39
+ this.handle = detail.handle === undefined ? null : detail.handle
40
+ this.notWired = detail.notWired === true
41
+ }
42
+ }
43
+
44
+ /**
45
+ * Whether the loaded engine carries the pack operations.
46
+ *
47
+ * The engine reports its own operation lists in the `ops` reflection
48
+ * envelope, so the command asks it rather than guessing from a package
49
+ * version. An engine built before stage 3 answers a package that has the
50
+ * subcommand, and this is how that pairing is detected.
51
+ */
52
+ export function packSupported (engine) {
53
+ let envelope
54
+ try {
55
+ envelope = engine.call('ops', [])
56
+ } catch (_error) {
57
+ return false
58
+ }
59
+ const named = (list) => Array.isArray(list) && list.indexOf('packBegin') >= 0
60
+ return named(envelope.ops) || named(envelope.blobOps) || named(envelope.blobIoOps)
61
+ }
62
+
63
+ /**
64
+ * Take every artifact the engine has ready and write it into `output`.
65
+ *
66
+ * `packNext` answers one artifact per call and reports when the queue is
67
+ * empty. The bytes come back in the module's out region, raw; nothing on
68
+ * this path encodes them.
69
+ *
70
+ * @returns {{names: string[], bytes: number}} what was written
71
+ */
72
+ function drain (engine, handle, output) {
73
+ const names = []
74
+ let total = 0
75
+ for (;;) {
76
+ const answer = engine.callBlobIO('packNext', [handle], new Uint8Array(0))
77
+ const envelope = answer.envelope
78
+ if (envelope.done === true) break
79
+ if (typeof envelope.name !== 'string') {
80
+ throw new PackError('packNext answered no artifact name', { handle })
81
+ }
82
+ writeNew(joinPath(output, envelope.name), answer.bytes)
83
+ names.push(envelope.name)
84
+ total += answer.bytes.length
85
+ }
86
+ return { names, bytes: total }
87
+ }
88
+
89
+ /**
90
+ * Build one immutable generation from one RDF file.
91
+ *
92
+ * @param {object} engine the loaded engine
93
+ * @param {string} input the RDF file to read
94
+ * @param {string} output the generation directory to fill; it must exist
95
+ * @param {object} options
96
+ * @param {string} options.layout `ibk3` or `ibk4`
97
+ * @param {string} options.syntax `turtle`, `trig` or `nquads`
98
+ * @param {string} [options.base] the base IRI relative IRIs resolve against;
99
+ * the empty string means no base
100
+ * @param {(progress: object) => void} [options.onProgress]
101
+ * @returns {object} the engine's own finish envelope, plus what was written
102
+ */
103
+ export function packFile (engine, input, output, options) {
104
+ const begun = engine.call('packBegin',
105
+ [options.syntax, options.layout, typeof options.base === 'string' ? options.base : ''])
106
+ const handle = begun.handle
107
+ if (typeof handle !== 'string') {
108
+ throw new PackError('packBegin answered no handle')
109
+ }
110
+ const written = []
111
+ let bytesWritten = 0
112
+ let bytesRead = 0
113
+ try {
114
+ // The prepass the native packer runs first -- the source digest and
115
+ // the generated-blank-node prefix -- is the engine's business, so the
116
+ // host simply feeds the file twice and lets the engine say when it
117
+ // has moved from one pass to the next.
118
+ for (let pass = 0; pass < 2; pass += 1) {
119
+ const passName = pass === 0 ? 'prepass' : 'ingest'
120
+ let offset = 0
121
+ for (;;) {
122
+ const chunk = readChunk(input, offset, FEED_BYTES)
123
+ if (chunk.length === 0) break
124
+ offset += chunk.length
125
+ bytesRead += chunk.length
126
+ engine.callBlobIO('packFeed', [handle], chunk)
127
+ const drained = drain(engine, handle, output)
128
+ written.push(...drained.names)
129
+ bytesWritten += drained.bytes
130
+ if (typeof options.onProgress === 'function') {
131
+ options.onProgress({ pass: passName, bytesRead, artifacts: written.length })
132
+ }
133
+ }
134
+ engine.call('packEndPass', [handle])
135
+ const drained = drain(engine, handle, output)
136
+ written.push(...drained.names)
137
+ bytesWritten += drained.bytes
138
+ }
139
+ const finished = engine.call('packFinish', [handle])
140
+ const last = drain(engine, handle, output)
141
+ written.push(...last.names)
142
+ bytesWritten += last.bytes
143
+ return { ...finished, written, bytesWritten, bytesRead }
144
+ } finally {
145
+ try { engine.call('packClose', [handle]) } catch (_error) { /* already gone */ }
146
+ }
147
+ }
148
+
149
+ /**
150
+ * Verify one generation and report the engine's verdict.
151
+ *
152
+ * The host reads the manifest and every artifact the engine asks for; the
153
+ * engine checks each one against the digest the manifest commits and
154
+ * every cross-artifact relation. Replacing CURRENT is the host's step,
155
+ * and it happens only on a verdict of ok.
156
+ */
157
+ export function verifyGeneration (engine, manifestHex, artifacts) {
158
+ const windows = []
159
+ let offset = 0
160
+ for (const artifact of artifacts) {
161
+ windows.push({ key: artifact.key, offset, len: artifact.bytes.length })
162
+ offset += artifact.bytes.length
163
+ }
164
+ const blob = new Uint8Array(offset)
165
+ let cursor = 0
166
+ for (const artifact of artifacts) {
167
+ blob.set(artifact.bytes, cursor)
168
+ cursor += artifact.bytes.length
169
+ }
170
+ return engine.callBlob('activateVerify',
171
+ [manifestHex, JSON.stringify(windows)], blob)
172
+ }
173
+
174
+ export { StoreHostError }
package/bin/store.mjs CHANGED
@@ -70,6 +70,38 @@ function asStoreError (error) {
70
70
  return new StoreOperationError(message)
71
71
  }
72
72
 
73
+ /**
74
+ * What to print when the runtime, not the engine, ran out of call stack.
75
+ *
76
+ * One copy of this text, used by every command that can hit the frame
77
+ * budget. `remedy` is the one line that differs: a query can be made
78
+ * smaller with a LIMIT, a pack cannot.
79
+ *
80
+ * The pack path normally never reaches this, because it runs on a worker
81
+ * thread with a raised stack (`bin/pack-host.mjs`,
82
+ * https://github.com/danbri/factoidal/issues/649). It is what a reader
83
+ * sees when the worker route is refused with --no-worker, is unavailable
84
+ * on their platform, or is not enough.
85
+ *
86
+ * @param {string} remedy
87
+ * @returns {string[]} the lines, in order, for stderr
88
+ */
89
+ export function stackLimitAdvice (remedy) {
90
+ return [
91
+ 'The runtime ran out of call stack inside the engine, not the store.',
92
+ 'Some engine paths recurse once per row or per input chunk, and enough',
93
+ "of them exceed the runtime's default WebAssembly frame budget.",
94
+ remedy
95
+ ]
96
+ }
97
+
98
+ /** The remedies for each command that can run out of frames. */
99
+ export const STACK_REMEDY = {
100
+ query: 'Raise it with node --stack-size=4000, add a LIMIT, or use Deno.',
101
+ pack: 'Raise it with node --stack-size=8000, or ' +
102
+ 'deno run --v8-flags=--stack-size=8000.'
103
+ }
104
+
73
105
  /**
74
106
  * Open a store and return its manifest bytes.
75
107
  *
@@ -37,7 +37,7 @@
37
37
  // bytes change.
38
38
 
39
39
  // Stamped by formal/lean4/Wasm/build-wasm.sh step 9 -- do not hand-edit.
40
- const WASM_VERSION = "9084903e6877";
40
+ const WASM_VERSION = "125d391e0ccc";
41
41
 
42
42
  import createModule from './l4factoidal.mjs';
43
43
 
@@ -187,6 +187,66 @@ export function loadL4() {
187
187
  }
188
188
  },
189
189
 
190
+ /**
191
+ * The dispatch ABI, plus ONE byte region IN and ONE byte region
192
+ * OUT.
193
+ *
194
+ * For the ops of `L4Wasm.blobIoOpNames` (the `ops` envelope lists
195
+ * them under `blobIoOps`), whose RESULT is bytes rather than
196
+ * text. The bytes leave the module raw — no hex, no base64 — and
197
+ * are copied out of the wasm heap into a fresh Uint8Array before
198
+ * the module's buffer is released. The copy is required: the heap
199
+ * is detached and replaced when the module grows, so a subarray
200
+ * view of it can go stale between calls.
201
+ *
202
+ * Every other op answers as `call` does, with an empty region.
203
+ *
204
+ * @param op the method name, e.g. "blobEcho"
205
+ * @param args array of positional STRING arguments
206
+ * @param blobIn Uint8Array (or ArrayBuffer) carried IN; may be omitted
207
+ * @returns { envelope, bytes } — the parsed {"ok":true,...}
208
+ * envelope and a Uint8Array of the out region
209
+ * @throws if the Lean side reports {"ok":false,"error":...}
210
+ */
211
+ callBlobIO(op, args, blobIn) {
212
+ const bytes = blobIn instanceof Uint8Array
213
+ ? blobIn
214
+ : new Uint8Array(blobIn ?? 0);
215
+ // Two 32-bit out parameters, uint8_t **out_ptr and size_t
216
+ // *out_len, in one 8-byte cell.
217
+ const outCell = Module._malloc(8);
218
+ if (!outCell) throw new Error('l4factoidal: could not allocate the out-parameter cell');
219
+ const blobPtr = bytes.length > 0 ? Module._malloc(bytes.length) : 0;
220
+ if (bytes.length > 0 && !blobPtr) {
221
+ Module._free(outCell);
222
+ throw new Error('l4factoidal: could not allocate a WASM blob buffer');
223
+ }
224
+ let outPtr = 0;
225
+ try {
226
+ Module.setValue(outCell, 0, 'i32');
227
+ Module.setValue(outCell + 4, 0, 'i32');
228
+ if (bytes.length > 0) Module.HEAPU8.set(bytes, blobPtr);
229
+ const resultPtr = callWithHeapStrings(
230
+ (opPtr, argsPtr) => Module._l4_call_blob_io_c(
231
+ opPtr, argsPtr, blobPtr, bytes.length, outCell, outCell + 4),
232
+ [op, JSON.stringify(args)]);
233
+ outPtr = Module.getValue(outCell, 'i32') >>> 0;
234
+ const outLen = Module.getValue(outCell + 4, 'i32') >>> 0;
235
+ const envelope = JSON.parse(take(resultPtr));
236
+ if (envelope.ok === false) throw new Error(`l4factoidal: ${envelope.error}`);
237
+ // slice() copies; HEAPU8 is replaced wholesale when the
238
+ // module's memory grows, so a view would not survive.
239
+ const region = outPtr !== 0 && outLen > 0
240
+ ? Module.HEAPU8.slice(outPtr, outPtr + outLen)
241
+ : new Uint8Array(0);
242
+ return { envelope, bytes: region };
243
+ } finally {
244
+ if (outPtr) Module._l4_free_blob(outPtr);
245
+ if (blobPtr) Module._free(blobPtr);
246
+ Module._free(outCell);
247
+ }
248
+ },
249
+
190
250
  /** Escape hatch for tests: the raw Emscripten module. */
191
251
  _module: Module,
192
252
  };