@factoidal/core 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +79 -0
- package/NOTICE +31 -0
- package/README.md +35 -1
- package/bin/factoidal.mjs +250 -14
- package/bin/pack-host.mjs +341 -0
- package/bin/pack-worker.mjs +31 -0
- package/bin/pack.mjs +174 -0
- package/bin/store.mjs +32 -0
- package/l4-assets/l4factoidal.js +61 -1
- package/l4-assets/l4factoidal.mjs +1 -1
- package/l4-assets/l4factoidal.wasm +0 -0
- package/l4-assets/version.json +4 -4
- package/package.json +10 -2
- package/sample-store/CURRENT +1 -0
- package/sample-store/gen-1/manifest.sbm2 +0 -0
- package/sample-store/gen-1/manifest.tsv +14 -0
- package/sample-store/gen-1/predicate-0.ibk3 +0 -0
- package/sample-store/gen-1/predicate-0.ibk3.merkle +1 -0
- package/sample-store/gen-1/predicate-0.ibk3.oli2 +0 -0
- package/sample-store/gen-1/predicate-0.ibk3.oli2.merkle +1 -0
- package/sample-store/gen-1/predicate-0.ibk3.sri2 +0 -0
- package/sample-store/gen-1/predicate-0.ibk3.sri2.merkle +1 -0
- package/sample-store/gen-1/predicate-0.ibk3.tli1 +0 -0
- package/sample-store/gen-1/predicate-0.ibk3.tli1.merkle +0 -0
- package/sample-store/gen-1/predicate-1.ibk3 +0 -0
- package/sample-store/gen-1/predicate-1.ibk3.merkle +0 -0
- package/sample-store/gen-1/predicate-1.ibk3.oli2 +0 -0
- package/sample-store/gen-1/predicate-1.ibk3.oli2.merkle +0 -0
- package/sample-store/gen-1/predicate-1.ibk3.sri2 +0 -0
- package/sample-store/gen-1/predicate-1.ibk3.sri2.merkle +1 -0
- package/sample-store/gen-1/predicate-1.ibk3.tli1 +0 -0
- package/sample-store/gen-1/predicate-1.ibk3.tli1.merkle +1 -0
- package/sample-store/gen-1/predicate-10.ibk3 +0 -0
- package/sample-store/gen-1/predicate-10.ibk3.merkle +1 -0
- package/sample-store/gen-1/predicate-10.ibk3.oli2 +0 -0
- package/sample-store/gen-1/predicate-10.ibk3.oli2.merkle +0 -0
- package/sample-store/gen-1/predicate-10.ibk3.sri2 +0 -0
- package/sample-store/gen-1/predicate-10.ibk3.sri2.merkle +1 -0
- package/sample-store/gen-1/predicate-10.ibk3.tli1 +0 -0
- package/sample-store/gen-1/predicate-10.ibk3.tli1.merkle +1 -0
- package/sample-store/gen-1/predicate-11.ibk3 +0 -0
- package/sample-store/gen-1/predicate-11.ibk3.merkle +0 -0
- package/sample-store/gen-1/predicate-11.ibk3.oli2 +0 -0
- package/sample-store/gen-1/predicate-11.ibk3.oli2.merkle +1 -0
- package/sample-store/gen-1/predicate-11.ibk3.sri2 +0 -0
- package/sample-store/gen-1/predicate-11.ibk3.sri2.merkle +1 -0
- package/sample-store/gen-1/predicate-11.ibk3.tli1 +0 -0
- package/sample-store/gen-1/predicate-11.ibk3.tli1.merkle +1 -0
- package/sample-store/gen-1/predicate-12.ibk3 +0 -0
- package/sample-store/gen-1/predicate-12.ibk3.merkle +1 -0
- package/sample-store/gen-1/predicate-12.ibk3.oli2 +0 -0
- package/sample-store/gen-1/predicate-12.ibk3.oli2.merkle +1 -0
- package/sample-store/gen-1/predicate-12.ibk3.sri2 +0 -0
- package/sample-store/gen-1/predicate-12.ibk3.sri2.merkle +1 -0
- package/sample-store/gen-1/predicate-12.ibk3.tli1 +0 -0
- package/sample-store/gen-1/predicate-12.ibk3.tli1.merkle +1 -0
- package/sample-store/gen-1/predicate-2.ibk3 +0 -0
- package/sample-store/gen-1/predicate-2.ibk3.merkle +1 -0
- package/sample-store/gen-1/predicate-2.ibk3.oli2 +0 -0
- package/sample-store/gen-1/predicate-2.ibk3.oli2.merkle +1 -0
- package/sample-store/gen-1/predicate-2.ibk3.sri2 +0 -0
- package/sample-store/gen-1/predicate-2.ibk3.sri2.merkle +0 -0
- package/sample-store/gen-1/predicate-2.ibk3.tli1 +0 -0
- package/sample-store/gen-1/predicate-2.ibk3.tli1.merkle +0 -0
- package/sample-store/gen-1/predicate-3.ibk3 +0 -0
- package/sample-store/gen-1/predicate-3.ibk3.merkle +1 -0
- package/sample-store/gen-1/predicate-3.ibk3.oli2 +0 -0
- package/sample-store/gen-1/predicate-3.ibk3.oli2.merkle +1 -0
- package/sample-store/gen-1/predicate-3.ibk3.sri2 +0 -0
- package/sample-store/gen-1/predicate-3.ibk3.sri2.merkle +1 -0
- package/sample-store/gen-1/predicate-3.ibk3.tli1 +0 -0
- package/sample-store/gen-1/predicate-3.ibk3.tli1.merkle +1 -0
- package/sample-store/gen-1/predicate-4.ibk3 +0 -0
- package/sample-store/gen-1/predicate-4.ibk3.merkle +0 -0
- package/sample-store/gen-1/predicate-4.ibk3.oli2 +0 -0
- package/sample-store/gen-1/predicate-4.ibk3.oli2.merkle +1 -0
- package/sample-store/gen-1/predicate-4.ibk3.sri2 +0 -0
- package/sample-store/gen-1/predicate-4.ibk3.sri2.merkle +1 -0
- package/sample-store/gen-1/predicate-4.ibk3.tli1 +0 -0
- package/sample-store/gen-1/predicate-4.ibk3.tli1.merkle +2 -0
- package/sample-store/gen-1/predicate-5.ibk3 +0 -0
- package/sample-store/gen-1/predicate-5.ibk3.merkle +1 -0
- package/sample-store/gen-1/predicate-5.ibk3.oli2 +0 -0
- package/sample-store/gen-1/predicate-5.ibk3.oli2.merkle +1 -0
- package/sample-store/gen-1/predicate-5.ibk3.sri2 +0 -0
- package/sample-store/gen-1/predicate-5.ibk3.sri2.merkle +1 -0
- package/sample-store/gen-1/predicate-5.ibk3.tli1 +0 -0
- package/sample-store/gen-1/predicate-5.ibk3.tli1.merkle +1 -0
- package/sample-store/gen-1/predicate-6.ibk3 +0 -0
- package/sample-store/gen-1/predicate-6.ibk3.merkle +1 -0
- package/sample-store/gen-1/predicate-6.ibk3.oli2 +0 -0
- package/sample-store/gen-1/predicate-6.ibk3.oli2.merkle +1 -0
- package/sample-store/gen-1/predicate-6.ibk3.sri2 +0 -0
- package/sample-store/gen-1/predicate-6.ibk3.sri2.merkle +1 -0
- package/sample-store/gen-1/predicate-6.ibk3.tli1 +0 -0
- package/sample-store/gen-1/predicate-6.ibk3.tli1.merkle +0 -0
- package/sample-store/gen-1/predicate-7.ibk3 +0 -0
- package/sample-store/gen-1/predicate-7.ibk3.merkle +0 -0
- package/sample-store/gen-1/predicate-7.ibk3.oli2 +0 -0
- package/sample-store/gen-1/predicate-7.ibk3.oli2.merkle +2 -0
- package/sample-store/gen-1/predicate-7.ibk3.sri2 +0 -0
- package/sample-store/gen-1/predicate-7.ibk3.sri2.merkle +1 -0
- package/sample-store/gen-1/predicate-7.ibk3.tli1 +0 -0
- package/sample-store/gen-1/predicate-7.ibk3.tli1.merkle +1 -0
- package/sample-store/gen-1/predicate-8.ibk3 +0 -0
- package/sample-store/gen-1/predicate-8.ibk3.merkle +1 -0
- package/sample-store/gen-1/predicate-8.ibk3.oli2 +0 -0
- package/sample-store/gen-1/predicate-8.ibk3.oli2.merkle +1 -0
- package/sample-store/gen-1/predicate-8.ibk3.sri2 +0 -0
- package/sample-store/gen-1/predicate-8.ibk3.sri2.merkle +1 -0
- package/sample-store/gen-1/predicate-8.ibk3.tli1 +0 -0
- package/sample-store/gen-1/predicate-8.ibk3.tli1.merkle +1 -0
- package/sample-store/gen-1/predicate-9.ibk3 +0 -0
- package/sample-store/gen-1/predicate-9.ibk3.merkle +1 -0
- package/sample-store/gen-1/predicate-9.ibk3.oli2 +0 -0
- package/sample-store/gen-1/predicate-9.ibk3.oli2.merkle +1 -0
- package/sample-store/gen-1/predicate-9.ibk3.sri2 +0 -0
- package/sample-store/gen-1/predicate-9.ibk3.sri2.merkle +3 -0
- package/sample-store/gen-1/predicate-9.ibk3.tli1 +0 -0
- package/sample-store/gen-1/predicate-9.ibk3.tli1.merkle +2 -0
- package/sample-store.d.ts +16 -0
- package/sample-store.mjs +37 -0
- package/store-host/deno.mjs +63 -0
- package/store-host/index.mjs +39 -0
- package/store-host/node.mjs +62 -1
- package/version.json +1 -1
|
@@ -0,0 +1,341 @@
|
|
|
1
|
+
// Giving `factoidal pack` a call stack big enough to finish, without
|
|
2
|
+
// asking the reader for a runtime flag.
|
|
3
|
+
// https://github.com/danbri/factoidal/issues/649
|
|
4
|
+
//
|
|
5
|
+
// THE DEFECT THIS FILE EXISTS FOR
|
|
6
|
+
// The pack fold in the engine recurses one frame per term in one block's
|
|
7
|
+
// local term index, and that is far deeper than a query recurses per row.
|
|
8
|
+
// On the DEFAULT stack of Node 22 and of Deno 2 the pack of anything
|
|
9
|
+
// above roughly 0.25 MB of Turtle ends with `Maximum call stack size
|
|
10
|
+
// exceeded` -- measured 2026-09-04 against the committed module, where
|
|
11
|
+
// `sequence_variant.ttl` (241,149 B) passes and `chromosome.ttl`
|
|
12
|
+
// (316,116 B) does not. Every one of those inputs packs correctly once
|
|
13
|
+
// the stack is raised, and the generation is byte-identical to
|
|
14
|
+
// `l4block-shard-pack` output. So the defect is the frame budget alone.
|
|
15
|
+
// The depth, what sets it, and why a browser cannot be given more are in
|
|
16
|
+
// `docs/designissues/2026-09-03-npm-pack-in-wasm.md`.
|
|
17
|
+
//
|
|
18
|
+
// WHAT THIS FILE IS ALLOWED TO DO
|
|
19
|
+
// Start the same pack on a thread or in a child process that has a
|
|
20
|
+
// bigger stack, carry the progress reports back, and re-raise the same
|
|
21
|
+
// error. It reads no RDF, decides no artifact name, and moves no
|
|
22
|
+
// artifact bytes: the worker writes the generation itself, through the
|
|
23
|
+
// same `store-host` primitives, exactly where the artifacts arrive.
|
|
24
|
+
// A reader must not be able to tell that a worker is involved, except
|
|
25
|
+
// that the pack finishes (iron rule 7 of CLAUDE.md).
|
|
26
|
+
//
|
|
27
|
+
// THE TWO ROUTES
|
|
28
|
+
// Node -- `worker_threads`, with `resourceLimits.stackSizeMb`. The
|
|
29
|
+
// engine loads INSIDE the worker: a loaded WebAssembly
|
|
30
|
+
// instance does not cross a thread boundary.
|
|
31
|
+
// Deno -- `worker_threads` is not the route, and a Deno `Worker` takes
|
|
32
|
+
// no stack size. The command re-executes itself once with
|
|
33
|
+
// `--v8-flags=--stack-size=...`, guarded by an environment
|
|
34
|
+
// variable so it can never loop. Confirmed honoured by
|
|
35
|
+
// deno 2.9.4 / V8 15.0.245.2 on 2026-09-04:
|
|
36
|
+
// `deno run -A --v8-flags=--stack-size=8000` packs
|
|
37
|
+
// `anatomical_structure.ttl` (3,811,378 B) in 13.7 s, where
|
|
38
|
+
// the same command without the flag overflows.
|
|
39
|
+
//
|
|
40
|
+
// WHAT IT DOES NOT FIX
|
|
41
|
+
// A browser tab has a fixed frame budget and no host flag, so neither
|
|
42
|
+
// route rescues an in-page packer. See the closing section of
|
|
43
|
+
// `docs/designissues/2026-09-03-npm-pack-in-wasm.md`.
|
|
44
|
+
|
|
45
|
+
import { StoreHostError, listGeneration, readWhole } from '../store-host/index.mjs'
|
|
46
|
+
import { fileUrlToPath, joinPath } from '../store-host/paths.mjs'
|
|
47
|
+
import { loadEngine } from './engine.mjs'
|
|
48
|
+
import { PackError, packFile, packSupported, verifyGeneration } from './pack.mjs'
|
|
49
|
+
import { openStore } from './store.mjs'
|
|
50
|
+
|
|
51
|
+
const isDeno = typeof globalThis.Deno !== 'undefined'
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* The worker's stack, in mebibytes.
|
|
55
|
+
*
|
|
56
|
+
* MEASURED, 2026-09-04, macOS 15 arm64, Node 22.22.2, against
|
|
57
|
+
* `examples/wikidata/subsets/lifesci-kgx/data/gene.ttl` -- 17,363,312
|
|
58
|
+
* bytes, 888,949 triples, the largest input the milestone carries:
|
|
59
|
+
*
|
|
60
|
+
* stackSizeMb verdict
|
|
61
|
+
* 4 (default) Maximum call stack size exceeded
|
|
62
|
+
* 6 Maximum call stack size exceeded
|
|
63
|
+
* 7 Maximum call stack size exceeded
|
|
64
|
+
* 8 pass, 42 s
|
|
65
|
+
* 10 pass
|
|
66
|
+
* 64 pass, 68 s
|
|
67
|
+
*
|
|
68
|
+
* 8 MiB is therefore the measured minimum, and 64 is eight times it.
|
|
69
|
+
* The headroom is deliberate and it is cheap: a thread stack is reserved
|
|
70
|
+
* address space, and only the pages actually touched become resident, so
|
|
71
|
+
* an unused 56 MiB costs nothing. The depth is set by the number of
|
|
72
|
+
* terms in ONE block's dictionary, not by the file size, so a file
|
|
73
|
+
* smaller than gene.ttl but with a wider dictionary needs more, and a
|
|
74
|
+
* value only just above the measured minimum would move the same defect
|
|
75
|
+
* to another input.
|
|
76
|
+
*/
|
|
77
|
+
export const WORKER_STACK_MB = 64
|
|
78
|
+
|
|
79
|
+
/** The V8 stack limit the Deno re-exec asks for, in kibibytes -- the
|
|
80
|
+
* unit `--stack-size` takes. Same budget as WORKER_STACK_MB. */
|
|
81
|
+
export const REEXEC_STACK_KB = WORKER_STACK_MB * 1024
|
|
82
|
+
|
|
83
|
+
/** Set in the re-executed Deno child, so the guard can never loop. */
|
|
84
|
+
export const REEXEC_GUARD = 'FACTOIDAL_PACK_STACK_REEXEC'
|
|
85
|
+
|
|
86
|
+
/** Read one environment variable, or null where reading it is refused. */
|
|
87
|
+
function environment (name) {
|
|
88
|
+
try {
|
|
89
|
+
if (isDeno && globalThis.Deno.env) {
|
|
90
|
+
const value = globalThis.Deno.env.get(name)
|
|
91
|
+
return typeof value === 'string' && value.length > 0 ? value : null
|
|
92
|
+
}
|
|
93
|
+
} catch (_error) {
|
|
94
|
+
// Deno without --allow-env. An absent override is the normal case.
|
|
95
|
+
return null
|
|
96
|
+
}
|
|
97
|
+
const value = globalThis.process && globalThis.process.env
|
|
98
|
+
? globalThis.process.env[name]
|
|
99
|
+
: undefined
|
|
100
|
+
return typeof value === 'string' && value.length > 0 ? value : null
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** The stack the worker asks for: `WORKER_STACK_MB`, or the override in
|
|
104
|
+
* FACTOIDAL_PACK_STACK_MB, which exists so the measurement above can be
|
|
105
|
+
* repeated without editing this file. */
|
|
106
|
+
export function workerStackMb () {
|
|
107
|
+
const override = environment('FACTOIDAL_PACK_STACK_MB')
|
|
108
|
+
if (override === null) return WORKER_STACK_MB
|
|
109
|
+
const value = Number(override)
|
|
110
|
+
return Number.isSafeInteger(value) && value > 0 ? value : WORKER_STACK_MB
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** Whether the caller asked for the in-process path. `--no-worker` on the
|
|
114
|
+
* command line, or FACTOIDAL_NO_WORKER in the environment. The old
|
|
115
|
+
* behaviour stays testable, and a platform where the worker fails has a
|
|
116
|
+
* way out. */
|
|
117
|
+
export function workerRefused (options) {
|
|
118
|
+
if (options && options.worker === false) return true
|
|
119
|
+
return environment('FACTOIDAL_NO_WORKER') !== null
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// ------------------------------------------------- carrying an error back
|
|
123
|
+
//
|
|
124
|
+
// `commandPack` distinguishes a PackError and a StoreHostError from
|
|
125
|
+
// everything else, and prints the engine's own words in each case. A
|
|
126
|
+
// worker boundary must not change any of that, so the failure crosses as
|
|
127
|
+
// a plain description and is rebuilt on the other side.
|
|
128
|
+
|
|
129
|
+
/** Describe a thrown value so it survives `postMessage`. */
|
|
130
|
+
export function describeError (error) {
|
|
131
|
+
if (error instanceof PackError) {
|
|
132
|
+
return {
|
|
133
|
+
type: 'PackError',
|
|
134
|
+
message: error.message,
|
|
135
|
+
handle: error.handle,
|
|
136
|
+
notWired: error.notWired
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
if (error instanceof StoreHostError) {
|
|
140
|
+
return {
|
|
141
|
+
type: 'StoreHostError',
|
|
142
|
+
message: error.message,
|
|
143
|
+
code: error.code,
|
|
144
|
+
path: error.path === undefined ? null : error.path
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
return {
|
|
148
|
+
type: 'Error',
|
|
149
|
+
message: error && error.message ? String(error.message) : String(error)
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/** Rebuild what `describeError` described. */
|
|
154
|
+
export function reviveError (description) {
|
|
155
|
+
if (description.type === 'PackError') {
|
|
156
|
+
return new PackError(description.message,
|
|
157
|
+
{ handle: description.handle, notWired: description.notWired })
|
|
158
|
+
}
|
|
159
|
+
if (description.type === 'StoreHostError') {
|
|
160
|
+
const detail = description.path === null ? {} : { path: description.path }
|
|
161
|
+
return new StoreHostError(description.code, description.message, detail)
|
|
162
|
+
}
|
|
163
|
+
return new Error(description.message)
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/** Whether a failure is the runtime running out of call stack. */
|
|
167
|
+
export function isStackOverflow (error) {
|
|
168
|
+
if (error instanceof RangeError) return true
|
|
169
|
+
const message = error && error.message ? String(error.message) : String(error)
|
|
170
|
+
return message.indexOf('call stack size exceeded') >= 0
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
// ------------------------------------------------------ the pack itself
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* Load the engine and run one deep task, on whatever stack the caller has.
|
|
177
|
+
*
|
|
178
|
+
* This is the body both routes run: the worker calls it on its own big
|
|
179
|
+
* stack, and `--no-worker` calls it in the process it was started in.
|
|
180
|
+
*
|
|
181
|
+
* `task.kind` selects the work. Both kinds recurse once per distinct term
|
|
182
|
+
* in a block's local term index, so both need the raised stack -- the
|
|
183
|
+
* verification `activate` runs decodes the same blocks the pack encoded.
|
|
184
|
+
* `activate` overflowed on a 112,742-row generation that `pack` had just
|
|
185
|
+
* written successfully, which is how the two came to share this path
|
|
186
|
+
* (measured 2026-09-04, https://github.com/danbri/factoidal/issues/649).
|
|
187
|
+
*
|
|
188
|
+
* @returns {{notWired: true}|{report: object}}
|
|
189
|
+
*/
|
|
190
|
+
export async function packHere (task, onProgress) {
|
|
191
|
+
const engine = await loadEngine()
|
|
192
|
+
if (!packSupported(engine)) return { notWired: true }
|
|
193
|
+
if (task.kind === 'activate') {
|
|
194
|
+
return { report: activateHere(engine, task) }
|
|
195
|
+
}
|
|
196
|
+
const report = packFile(engine, task.input, task.output, {
|
|
197
|
+
syntax: task.syntax,
|
|
198
|
+
layout: task.layout,
|
|
199
|
+
base: task.base,
|
|
200
|
+
onProgress
|
|
201
|
+
})
|
|
202
|
+
return { report }
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* Verify one generation and answer the engine's verdict.
|
|
207
|
+
*
|
|
208
|
+
* The host reads the manifest and every artifact and hands them over; the
|
|
209
|
+
* engine checks each against the digest the manifest commits. Replacing
|
|
210
|
+
* CURRENT is NOT done here -- it is the caller's step, and it happens on
|
|
211
|
+
* the main thread only after this returns ok, so a worker that dies
|
|
212
|
+
* mid-verification can never leave a half-moved pointer.
|
|
213
|
+
*/
|
|
214
|
+
function activateHere (engine, task) {
|
|
215
|
+
const store = openStore(task.root, task.generation)
|
|
216
|
+
const files = listGeneration(store.generationDir)
|
|
217
|
+
const artifacts = files
|
|
218
|
+
.filter((file) => file.name !== store.manifestName)
|
|
219
|
+
.map((file) => ({
|
|
220
|
+
key: file.name,
|
|
221
|
+
bytes: readWhole(joinPath(store.generationDir, file.name))
|
|
222
|
+
}))
|
|
223
|
+
return verifyGeneration(engine, store.manifestHex, artifacts)
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/**
|
|
227
|
+
* Pack on a `worker_threads` thread with a raised stack. Node only.
|
|
228
|
+
*
|
|
229
|
+
* The worker writes every artifact itself, so no artifact byte crosses
|
|
230
|
+
* the thread boundary; what crosses is the task, the progress reports
|
|
231
|
+
* and the finish envelope.
|
|
232
|
+
*
|
|
233
|
+
* @returns {{notWired: true}|{report: object}}
|
|
234
|
+
* @throws whatever the pack threw, rebuilt
|
|
235
|
+
*/
|
|
236
|
+
async function packInWorker (task, onProgress) {
|
|
237
|
+
const { Worker } = await import('node:worker_threads')
|
|
238
|
+
const entry = new URL('./pack-worker.mjs', import.meta.url)
|
|
239
|
+
const stackSizeMb = workerStackMb()
|
|
240
|
+
return await new Promise((resolve, reject) => {
|
|
241
|
+
const worker = new Worker(entry, {
|
|
242
|
+
workerData: { ...task, progress: typeof onProgress === 'function' },
|
|
243
|
+
resourceLimits: { stackSizeMb },
|
|
244
|
+
stdout: false,
|
|
245
|
+
stderr: false
|
|
246
|
+
})
|
|
247
|
+
let settled = false
|
|
248
|
+
const finish = (act) => {
|
|
249
|
+
if (settled) return
|
|
250
|
+
settled = true
|
|
251
|
+
worker.terminate().then(act, act)
|
|
252
|
+
}
|
|
253
|
+
worker.on('message', (message) => {
|
|
254
|
+
if (message.kind === 'progress') {
|
|
255
|
+
if (typeof onProgress === 'function') onProgress(message.progress)
|
|
256
|
+
return
|
|
257
|
+
}
|
|
258
|
+
if (message.kind === 'error') {
|
|
259
|
+
const failure = reviveError(message.error)
|
|
260
|
+
finish(() => reject(failure))
|
|
261
|
+
return
|
|
262
|
+
}
|
|
263
|
+
finish(() => resolve(message.kind === 'notWired'
|
|
264
|
+
? { notWired: true }
|
|
265
|
+
: { report: message.report }))
|
|
266
|
+
})
|
|
267
|
+
worker.on('error', (error) => { finish(() => reject(error)) })
|
|
268
|
+
worker.on('exit', (code) => {
|
|
269
|
+
if (settled) return
|
|
270
|
+
settled = true
|
|
271
|
+
reject(new PackError(`the pack worker exited with code ${code}`))
|
|
272
|
+
})
|
|
273
|
+
})
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/**
|
|
277
|
+
* Run one deep task, on the biggest stack this runtime will give it.
|
|
278
|
+
*
|
|
279
|
+
* On Node the worker route is the default. On Deno the caller has
|
|
280
|
+
* already re-executed (see `denoReexec`), so this runs in process.
|
|
281
|
+
*
|
|
282
|
+
* @param {object} task {kind, ...}: a pack takes
|
|
283
|
+
* {input, output, syntax, layout, base}, an activate takes
|
|
284
|
+
* {root, generation}
|
|
285
|
+
* @param {(progress: object) => void} [onProgress]
|
|
286
|
+
* @param {object} [options] {worker: false} forces the in-process path
|
|
287
|
+
*/
|
|
288
|
+
export async function runPack (task, onProgress, options) {
|
|
289
|
+
if (isDeno || workerRefused(options)) return await packHere(task, onProgress)
|
|
290
|
+
return await packInWorker(task, onProgress)
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
// -------------------------------------------------------- the Deno route
|
|
294
|
+
|
|
295
|
+
/**
|
|
296
|
+
* Re-execute this command once under Deno with a raised V8 stack.
|
|
297
|
+
*
|
|
298
|
+
* Returns null when the re-exec is not available or not wanted, and the
|
|
299
|
+
* caller then packs in process. Returns the child's exit code when the
|
|
300
|
+
* child ran, and the caller exits with it. Permissions are NOT
|
|
301
|
+
* escalated: the child is given exactly the ones this process was
|
|
302
|
+
* granted, queried rather than requested so no prompt appears.
|
|
303
|
+
*/
|
|
304
|
+
export async function denoReexec (options) {
|
|
305
|
+
if (!isDeno) return null
|
|
306
|
+
if (workerRefused(options)) return null
|
|
307
|
+
if (environment(REEXEC_GUARD) !== null) return null
|
|
308
|
+
const Deno = globalThis.Deno
|
|
309
|
+
let granted
|
|
310
|
+
try {
|
|
311
|
+
granted = ['read', 'write', 'env', 'run', 'net', 'ffi', 'sys']
|
|
312
|
+
.filter((name) => Deno.permissions.querySync({ name }).state === 'granted')
|
|
313
|
+
} catch (_error) {
|
|
314
|
+
return null
|
|
315
|
+
}
|
|
316
|
+
if (granted.indexOf('run') < 0 || granted.indexOf('read') < 0) return null
|
|
317
|
+
const main = Deno.mainModule.startsWith('file://')
|
|
318
|
+
? fileUrlToPath(Deno.mainModule)
|
|
319
|
+
: Deno.mainModule
|
|
320
|
+
const args = [
|
|
321
|
+
'run',
|
|
322
|
+
...granted.map((name) => `--allow-${name}`),
|
|
323
|
+
`--v8-flags=--stack-size=${REEXEC_STACK_KB}`,
|
|
324
|
+
main,
|
|
325
|
+
...Deno.args
|
|
326
|
+
]
|
|
327
|
+
try {
|
|
328
|
+
const child = new Deno.Command(Deno.execPath(), {
|
|
329
|
+
args,
|
|
330
|
+
env: { [REEXEC_GUARD]: '1' },
|
|
331
|
+
stdin: 'inherit',
|
|
332
|
+
stdout: 'inherit',
|
|
333
|
+
stderr: 'inherit'
|
|
334
|
+
}).outputSync()
|
|
335
|
+
return child.code
|
|
336
|
+
} catch (_error) {
|
|
337
|
+
// No --allow-run at this exact path, or no executable. Pack here and
|
|
338
|
+
// let the overflow advice speak if the stack runs out.
|
|
339
|
+
return null
|
|
340
|
+
}
|
|
341
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
// The thread `factoidal pack` runs on under Node, so that the pack has a
|
|
2
|
+
// call stack big enough to finish.
|
|
3
|
+
// https://github.com/danbri/factoidal/issues/649
|
|
4
|
+
//
|
|
5
|
+
// WHAT THIS FILE IS ALLOWED TO DO
|
|
6
|
+
// Load the engine, run the same pack the in-process path runs, post the
|
|
7
|
+
// progress reports back, and post the failure back. It reads no RDF,
|
|
8
|
+
// names no artifact and encodes nothing: `pack.mjs` drives the engine
|
|
9
|
+
// and `store-host` writes the bytes, here exactly as they do on the
|
|
10
|
+
// main thread (iron rule 7 of CLAUDE.md).
|
|
11
|
+
//
|
|
12
|
+
// The engine loads HERE, not on the main thread: a loaded WebAssembly
|
|
13
|
+
// instance does not cross a thread boundary. The artifacts are written
|
|
14
|
+
// HERE too, where they arrive, so no artifact byte crosses it either.
|
|
15
|
+
|
|
16
|
+
import { parentPort, workerData } from 'node:worker_threads'
|
|
17
|
+
import { describeError, packHere } from './pack-host.mjs'
|
|
18
|
+
|
|
19
|
+
const port = parentPort
|
|
20
|
+
const task = workerData
|
|
21
|
+
|
|
22
|
+
packHere(task, task.progress
|
|
23
|
+
? (progress) => port.postMessage({ kind: 'progress', progress })
|
|
24
|
+
: undefined)
|
|
25
|
+
.then((answer) => {
|
|
26
|
+
if (answer.notWired === true) port.postMessage({ kind: 'notWired' })
|
|
27
|
+
else port.postMessage({ kind: 'report', report: answer.report })
|
|
28
|
+
})
|
|
29
|
+
.catch((error) => {
|
|
30
|
+
port.postMessage({ kind: 'error', error: describeError(error) })
|
|
31
|
+
})
|
package/bin/pack.mjs
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
// Building a Shardborough generation from a JavaScript host, with the
|
|
2
|
+
// Lean engine running as WebAssembly.
|
|
3
|
+
// https://github.com/danbri/factoidal/issues/641 stage 3.
|
|
4
|
+
// Design record: docs/designissues/2026-09-03-npm-pack-in-wasm.md.
|
|
5
|
+
//
|
|
6
|
+
// WHAT THIS FILE IS ALLOWED TO DO
|
|
7
|
+
// Read the input file in chunks, hand each chunk to the engine, take back
|
|
8
|
+
// whatever artifacts the engine says are ready, and write each one under
|
|
9
|
+
// the name the engine gave it. It decides no artifact name, computes no
|
|
10
|
+
// digest, encodes no block and writes no manifest field. Every one of
|
|
11
|
+
// those is a format decision and it lives in the Lean source (iron rule
|
|
12
|
+
// 7 of CLAUDE.md). A reviewer who finds a magic number, a field offset,
|
|
13
|
+
// a file-name pattern or a hash in this file has found a violation.
|
|
14
|
+
//
|
|
15
|
+
// There is no file I/O inside the wasm module -- libuv is left out of the
|
|
16
|
+
// Emscripten build by design -- so the engine cannot write the generation
|
|
17
|
+
// itself. That is the whole reason this file exists.
|
|
18
|
+
//
|
|
19
|
+
// WHY IT STREAMS
|
|
20
|
+
// wasm32 addresses 4 GiB. The native packer's own second pass is already
|
|
21
|
+
// bounded: it publishes a batch of blocks every 64 input chunks and then
|
|
22
|
+
// drops the triples. This host drives that same fold one chunk at a time
|
|
23
|
+
// and writes each artifact as it appears, so neither side ever holds the
|
|
24
|
+
// whole input or the whole generation.
|
|
25
|
+
|
|
26
|
+
import { StoreHostError, readChunk, writeNew } from '../store-host/index.mjs'
|
|
27
|
+
import { joinPath } from '../store-host/paths.mjs'
|
|
28
|
+
|
|
29
|
+
/** The chunk the host feeds per call. The native packer reads 65,536
|
|
30
|
+
* bytes at a time; matching it keeps the two folds in step, which is
|
|
31
|
+
* what makes the byte-identity gate meaningful. */
|
|
32
|
+
export const FEED_BYTES = 65536
|
|
33
|
+
|
|
34
|
+
/** An error the pack operations reported, with the engine's own words. */
|
|
35
|
+
export class PackError extends Error {
|
|
36
|
+
constructor (message, detail = {}) {
|
|
37
|
+
super(message)
|
|
38
|
+
this.name = 'PackError'
|
|
39
|
+
this.handle = detail.handle === undefined ? null : detail.handle
|
|
40
|
+
this.notWired = detail.notWired === true
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Whether the loaded engine carries the pack operations.
|
|
46
|
+
*
|
|
47
|
+
* The engine reports its own operation lists in the `ops` reflection
|
|
48
|
+
* envelope, so the command asks it rather than guessing from a package
|
|
49
|
+
* version. An engine built before stage 3 answers a package that has the
|
|
50
|
+
* subcommand, and this is how that pairing is detected.
|
|
51
|
+
*/
|
|
52
|
+
export function packSupported (engine) {
|
|
53
|
+
let envelope
|
|
54
|
+
try {
|
|
55
|
+
envelope = engine.call('ops', [])
|
|
56
|
+
} catch (_error) {
|
|
57
|
+
return false
|
|
58
|
+
}
|
|
59
|
+
const named = (list) => Array.isArray(list) && list.indexOf('packBegin') >= 0
|
|
60
|
+
return named(envelope.ops) || named(envelope.blobOps) || named(envelope.blobIoOps)
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Take every artifact the engine has ready and write it into `output`.
|
|
65
|
+
*
|
|
66
|
+
* `packNext` answers one artifact per call and reports when the queue is
|
|
67
|
+
* empty. The bytes come back in the module's out region, raw; nothing on
|
|
68
|
+
* this path encodes them.
|
|
69
|
+
*
|
|
70
|
+
* @returns {{names: string[], bytes: number}} what was written
|
|
71
|
+
*/
|
|
72
|
+
function drain (engine, handle, output) {
|
|
73
|
+
const names = []
|
|
74
|
+
let total = 0
|
|
75
|
+
for (;;) {
|
|
76
|
+
const answer = engine.callBlobIO('packNext', [handle], new Uint8Array(0))
|
|
77
|
+
const envelope = answer.envelope
|
|
78
|
+
if (envelope.done === true) break
|
|
79
|
+
if (typeof envelope.name !== 'string') {
|
|
80
|
+
throw new PackError('packNext answered no artifact name', { handle })
|
|
81
|
+
}
|
|
82
|
+
writeNew(joinPath(output, envelope.name), answer.bytes)
|
|
83
|
+
names.push(envelope.name)
|
|
84
|
+
total += answer.bytes.length
|
|
85
|
+
}
|
|
86
|
+
return { names, bytes: total }
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Build one immutable generation from one RDF file.
|
|
91
|
+
*
|
|
92
|
+
* @param {object} engine the loaded engine
|
|
93
|
+
* @param {string} input the RDF file to read
|
|
94
|
+
* @param {string} output the generation directory to fill; it must exist
|
|
95
|
+
* @param {object} options
|
|
96
|
+
* @param {string} options.layout `ibk3` or `ibk4`
|
|
97
|
+
* @param {string} options.syntax `turtle`, `trig` or `nquads`
|
|
98
|
+
* @param {string} [options.base] the base IRI relative IRIs resolve against;
|
|
99
|
+
* the empty string means no base
|
|
100
|
+
* @param {(progress: object) => void} [options.onProgress]
|
|
101
|
+
* @returns {object} the engine's own finish envelope, plus what was written
|
|
102
|
+
*/
|
|
103
|
+
export function packFile (engine, input, output, options) {
|
|
104
|
+
const begun = engine.call('packBegin',
|
|
105
|
+
[options.syntax, options.layout, typeof options.base === 'string' ? options.base : ''])
|
|
106
|
+
const handle = begun.handle
|
|
107
|
+
if (typeof handle !== 'string') {
|
|
108
|
+
throw new PackError('packBegin answered no handle')
|
|
109
|
+
}
|
|
110
|
+
const written = []
|
|
111
|
+
let bytesWritten = 0
|
|
112
|
+
let bytesRead = 0
|
|
113
|
+
try {
|
|
114
|
+
// The prepass the native packer runs first -- the source digest and
|
|
115
|
+
// the generated-blank-node prefix -- is the engine's business, so the
|
|
116
|
+
// host simply feeds the file twice and lets the engine say when it
|
|
117
|
+
// has moved from one pass to the next.
|
|
118
|
+
for (let pass = 0; pass < 2; pass += 1) {
|
|
119
|
+
const passName = pass === 0 ? 'prepass' : 'ingest'
|
|
120
|
+
let offset = 0
|
|
121
|
+
for (;;) {
|
|
122
|
+
const chunk = readChunk(input, offset, FEED_BYTES)
|
|
123
|
+
if (chunk.length === 0) break
|
|
124
|
+
offset += chunk.length
|
|
125
|
+
bytesRead += chunk.length
|
|
126
|
+
engine.callBlobIO('packFeed', [handle], chunk)
|
|
127
|
+
const drained = drain(engine, handle, output)
|
|
128
|
+
written.push(...drained.names)
|
|
129
|
+
bytesWritten += drained.bytes
|
|
130
|
+
if (typeof options.onProgress === 'function') {
|
|
131
|
+
options.onProgress({ pass: passName, bytesRead, artifacts: written.length })
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
engine.call('packEndPass', [handle])
|
|
135
|
+
const drained = drain(engine, handle, output)
|
|
136
|
+
written.push(...drained.names)
|
|
137
|
+
bytesWritten += drained.bytes
|
|
138
|
+
}
|
|
139
|
+
const finished = engine.call('packFinish', [handle])
|
|
140
|
+
const last = drain(engine, handle, output)
|
|
141
|
+
written.push(...last.names)
|
|
142
|
+
bytesWritten += last.bytes
|
|
143
|
+
return { ...finished, written, bytesWritten, bytesRead }
|
|
144
|
+
} finally {
|
|
145
|
+
try { engine.call('packClose', [handle]) } catch (_error) { /* already gone */ }
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* Verify one generation and report the engine's verdict.
|
|
151
|
+
*
|
|
152
|
+
* The host reads the manifest and every artifact the engine asks for; the
|
|
153
|
+
* engine checks each one against the digest the manifest commits and
|
|
154
|
+
* every cross-artifact relation. Replacing CURRENT is the host's step,
|
|
155
|
+
* and it happens only on a verdict of ok.
|
|
156
|
+
*/
|
|
157
|
+
export function verifyGeneration (engine, manifestHex, artifacts) {
|
|
158
|
+
const windows = []
|
|
159
|
+
let offset = 0
|
|
160
|
+
for (const artifact of artifacts) {
|
|
161
|
+
windows.push({ key: artifact.key, offset, len: artifact.bytes.length })
|
|
162
|
+
offset += artifact.bytes.length
|
|
163
|
+
}
|
|
164
|
+
const blob = new Uint8Array(offset)
|
|
165
|
+
let cursor = 0
|
|
166
|
+
for (const artifact of artifacts) {
|
|
167
|
+
blob.set(artifact.bytes, cursor)
|
|
168
|
+
cursor += artifact.bytes.length
|
|
169
|
+
}
|
|
170
|
+
return engine.callBlob('activateVerify',
|
|
171
|
+
[manifestHex, JSON.stringify(windows)], blob)
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
export { StoreHostError }
|
package/bin/store.mjs
CHANGED
|
@@ -70,6 +70,38 @@ function asStoreError (error) {
|
|
|
70
70
|
return new StoreOperationError(message)
|
|
71
71
|
}
|
|
72
72
|
|
|
73
|
+
/**
|
|
74
|
+
* What to print when the runtime, not the engine, ran out of call stack.
|
|
75
|
+
*
|
|
76
|
+
* One copy of this text, used by every command that can hit the frame
|
|
77
|
+
* budget. `remedy` is the one line that differs: a query can be made
|
|
78
|
+
* smaller with a LIMIT, a pack cannot.
|
|
79
|
+
*
|
|
80
|
+
* The pack path normally never reaches this, because it runs on a worker
|
|
81
|
+
* thread with a raised stack (`bin/pack-host.mjs`,
|
|
82
|
+
* https://github.com/danbri/factoidal/issues/649). It is what a reader
|
|
83
|
+
* sees when the worker route is refused with --no-worker, is unavailable
|
|
84
|
+
* on their platform, or is not enough.
|
|
85
|
+
*
|
|
86
|
+
* @param {string} remedy
|
|
87
|
+
* @returns {string[]} the lines, in order, for stderr
|
|
88
|
+
*/
|
|
89
|
+
export function stackLimitAdvice (remedy) {
|
|
90
|
+
return [
|
|
91
|
+
'The runtime ran out of call stack inside the engine, not the store.',
|
|
92
|
+
'Some engine paths recurse once per row or per input chunk, and enough',
|
|
93
|
+
"of them exceed the runtime's default WebAssembly frame budget.",
|
|
94
|
+
remedy
|
|
95
|
+
]
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/** The remedies for each command that can run out of frames. */
|
|
99
|
+
export const STACK_REMEDY = {
|
|
100
|
+
query: 'Raise it with node --stack-size=4000, add a LIMIT, or use Deno.',
|
|
101
|
+
pack: 'Raise it with node --stack-size=8000, or ' +
|
|
102
|
+
'deno run --v8-flags=--stack-size=8000.'
|
|
103
|
+
}
|
|
104
|
+
|
|
73
105
|
/**
|
|
74
106
|
* Open a store and return its manifest bytes.
|
|
75
107
|
*
|
package/l4-assets/l4factoidal.js
CHANGED
|
@@ -37,7 +37,7 @@
|
|
|
37
37
|
// bytes change.
|
|
38
38
|
|
|
39
39
|
// Stamped by formal/lean4/Wasm/build-wasm.sh step 9 -- do not hand-edit.
|
|
40
|
-
const WASM_VERSION = "
|
|
40
|
+
const WASM_VERSION = "125d391e0ccc";
|
|
41
41
|
|
|
42
42
|
import createModule from './l4factoidal.mjs';
|
|
43
43
|
|
|
@@ -187,6 +187,66 @@ export function loadL4() {
|
|
|
187
187
|
}
|
|
188
188
|
},
|
|
189
189
|
|
|
190
|
+
/**
|
|
191
|
+
* The dispatch ABI, plus ONE byte region IN and ONE byte region
|
|
192
|
+
* OUT.
|
|
193
|
+
*
|
|
194
|
+
* For the ops of `L4Wasm.blobIoOpNames` (the `ops` envelope lists
|
|
195
|
+
* them under `blobIoOps`), whose RESULT is bytes rather than
|
|
196
|
+
* text. The bytes leave the module raw — no hex, no base64 — and
|
|
197
|
+
* are copied out of the wasm heap into a fresh Uint8Array before
|
|
198
|
+
* the module's buffer is released. The copy is required: the heap
|
|
199
|
+
* is detached and replaced when the module grows, so a subarray
|
|
200
|
+
* view of it can go stale between calls.
|
|
201
|
+
*
|
|
202
|
+
* Every other op answers as `call` does, with an empty region.
|
|
203
|
+
*
|
|
204
|
+
* @param op the method name, e.g. "blobEcho"
|
|
205
|
+
* @param args array of positional STRING arguments
|
|
206
|
+
* @param blobIn Uint8Array (or ArrayBuffer) carried IN; may be omitted
|
|
207
|
+
* @returns { envelope, bytes } — the parsed {"ok":true,...}
|
|
208
|
+
* envelope and a Uint8Array of the out region
|
|
209
|
+
* @throws if the Lean side reports {"ok":false,"error":...}
|
|
210
|
+
*/
|
|
211
|
+
callBlobIO(op, args, blobIn) {
|
|
212
|
+
const bytes = blobIn instanceof Uint8Array
|
|
213
|
+
? blobIn
|
|
214
|
+
: new Uint8Array(blobIn ?? 0);
|
|
215
|
+
// Two 32-bit out parameters, uint8_t **out_ptr and size_t
|
|
216
|
+
// *out_len, in one 8-byte cell.
|
|
217
|
+
const outCell = Module._malloc(8);
|
|
218
|
+
if (!outCell) throw new Error('l4factoidal: could not allocate the out-parameter cell');
|
|
219
|
+
const blobPtr = bytes.length > 0 ? Module._malloc(bytes.length) : 0;
|
|
220
|
+
if (bytes.length > 0 && !blobPtr) {
|
|
221
|
+
Module._free(outCell);
|
|
222
|
+
throw new Error('l4factoidal: could not allocate a WASM blob buffer');
|
|
223
|
+
}
|
|
224
|
+
let outPtr = 0;
|
|
225
|
+
try {
|
|
226
|
+
Module.setValue(outCell, 0, 'i32');
|
|
227
|
+
Module.setValue(outCell + 4, 0, 'i32');
|
|
228
|
+
if (bytes.length > 0) Module.HEAPU8.set(bytes, blobPtr);
|
|
229
|
+
const resultPtr = callWithHeapStrings(
|
|
230
|
+
(opPtr, argsPtr) => Module._l4_call_blob_io_c(
|
|
231
|
+
opPtr, argsPtr, blobPtr, bytes.length, outCell, outCell + 4),
|
|
232
|
+
[op, JSON.stringify(args)]);
|
|
233
|
+
outPtr = Module.getValue(outCell, 'i32') >>> 0;
|
|
234
|
+
const outLen = Module.getValue(outCell + 4, 'i32') >>> 0;
|
|
235
|
+
const envelope = JSON.parse(take(resultPtr));
|
|
236
|
+
if (envelope.ok === false) throw new Error(`l4factoidal: ${envelope.error}`);
|
|
237
|
+
// slice() copies; HEAPU8 is replaced wholesale when the
|
|
238
|
+
// module's memory grows, so a view would not survive.
|
|
239
|
+
const region = outPtr !== 0 && outLen > 0
|
|
240
|
+
? Module.HEAPU8.slice(outPtr, outPtr + outLen)
|
|
241
|
+
: new Uint8Array(0);
|
|
242
|
+
return { envelope, bytes: region };
|
|
243
|
+
} finally {
|
|
244
|
+
if (outPtr) Module._l4_free_blob(outPtr);
|
|
245
|
+
if (blobPtr) Module._free(blobPtr);
|
|
246
|
+
Module._free(outCell);
|
|
247
|
+
}
|
|
248
|
+
},
|
|
249
|
+
|
|
190
250
|
/** Escape hatch for tests: the raw Emscripten module. */
|
|
191
251
|
_module: Module,
|
|
192
252
|
};
|