@factoidal/core 0.13.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/README.md +58 -8
- package/bin/factoidal.mjs +6 -6
- package/browser.d.ts +10 -0
- package/browser.js +17 -0
- package/index.d.ts +279 -11
- package/index.mjs +26 -0
- package/l4-assets/l4factoidal.js +1 -1
- package/l4-assets/l4factoidal.mjs +1 -1
- package/l4-assets/l4factoidal.wasm +0 -0
- package/l4-assets/version.json +6 -6
- package/lib/api.mjs +55 -3
- package/lib/dataset.mjs +8 -0
- package/lib/evidence.mjs +93 -0
- package/lib/nodes.mjs +274 -0
- package/lib/sbm11-lws.mjs +200 -0
- package/lib/sbm11-node.mjs +61 -0
- package/lib/sbm11.mjs +155 -0
- package/lib/sync-http-node.mjs +71 -0
- package/lib/version.mjs +1 -1
- package/package.json +1 -1
- package/sample-store.mjs +1 -1
package/lib/nodes.mjs
ADDED
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
// The node-composition API (https://github.com/danbri/factoidal/issues/1002,
|
|
2
|
+
// step 2): registered Lean stages as nodes, each carrying its contract name
|
|
3
|
+
// and its proof status, with evidence flowing on the wires.
|
|
4
|
+
//
|
|
5
|
+
// The stage list is the engine's (op stageList, formal/lean4/Wasm/Ops/
|
|
6
|
+
// StageList.lean), read from the Lean registry
|
|
7
|
+
// L4Factoidal.Dataflow.registry: name, interface type, wire types, status,
|
|
8
|
+
// contract, named assumptions, and whether the stage may run at all
|
|
9
|
+
// (`runnable`, false for status `missing`). This file holds no copy of it.
|
|
10
|
+
//
|
|
11
|
+
// What this file adds is the binding of a stage NAME to the package
|
|
12
|
+
// function that runs it (RUNNERS below; each is the function that
|
|
13
|
+
// foundation/entrypoints.toml maps to that stage), and the wiring: a value
|
|
14
|
+
// on a wire is `{ graph, evidence }`, and each node appends one evidence
|
|
15
|
+
// record `{ stage, status, contract, assumptions, checked, artefact }` (plus
|
|
16
|
+
// `unchecked` and `reason` for a step run under run()'s `unchecked` opt-in;
|
|
17
|
+
// the record logic is in evidence.mjs).
|
|
18
|
+
//
|
|
19
|
+
// The type check and the assumption union are the engine's (op
|
|
20
|
+
// pipelineCheck, formal/lean4/Wasm/Ops/StageList.lean, computed by
|
|
21
|
+
// pipelineReport of formal/lean4/L4Factoidal/Dataflow/Pipeline.lean): a
|
|
22
|
+
// pipeline is well typed when each stage's output wire is the next stage's
|
|
23
|
+
// input wire or converts to it, and its `assumptions` are the premises of
|
|
24
|
+
// Pipeline.guarantee_sound, the guarantee of any pipeline of components with
|
|
25
|
+
// these signatures. This file does not compute the union. With an engine
|
|
26
|
+
// built before the op, pipe() and run() fall back to the union of the
|
|
27
|
+
// nodes' fields and report `wellTyped: null`.
|
|
28
|
+
//
|
|
29
|
+
// ES module, no Node built-ins: the browser loads this file as it is.
|
|
30
|
+
|
|
31
|
+
import { FactoidalError } from './errors.mjs';
|
|
32
|
+
import { refusalFor, makeRecord, summarize, unionAssumptions, unionLists } from './evidence.mjs';
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* @param {object} deps the package functions and the engine entry
|
|
36
|
+
* @returns {{stages: Function, node: Function, pipe: Function, run: Function}}
|
|
37
|
+
*/
|
|
38
|
+
export function buildNodes(deps) {
|
|
39
|
+
const { entry, entryResult, requireEntryFn, Dataset,
|
|
40
|
+
parse, closure, shaclValidate, query, serialize, canonicalize, explain } = deps;
|
|
41
|
+
|
|
42
|
+
const reader = (format) => async (wire, params) => {
|
|
43
|
+
if (typeof wire.graph !== 'string') {
|
|
44
|
+
throw new TypeError(`node: a reader takes text on its wire (got ${typeof wire.graph})`);
|
|
45
|
+
}
|
|
46
|
+
const ds = await parse(wire.graph, { format, ...params });
|
|
47
|
+
return { graph: ds, artefact: { format } };
|
|
48
|
+
};
|
|
49
|
+
const reasoner = (regime) => async (wire, params) => ({
|
|
50
|
+
graph: await closure(wire.graph, regime, params),
|
|
51
|
+
artefact: { regime },
|
|
52
|
+
});
|
|
53
|
+
const writer = (format) => async (wire, params) => ({
|
|
54
|
+
artefact: { format, text: await serialize(wire.graph, { ...params, format }) },
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
// Stage name (stageList `name`) -> the package function that runs it.
|
|
58
|
+
// A runner answers `{ graph?, artefact }`; no `graph` passes the input
|
|
59
|
+
// graph on unchanged.
|
|
60
|
+
const RUNNERS = {
|
|
61
|
+
ntriplesDocumentReader: reader('ntriples'),
|
|
62
|
+
nquadsDocumentReader: reader('nquads'),
|
|
63
|
+
turtleDocumentReader: reader('turtle'),
|
|
64
|
+
trigDocumentReader: reader('trig'),
|
|
65
|
+
rdfxmlDocumentReader: reader('rdfxml'),
|
|
66
|
+
jsonldDocumentReader: reader('jsonld'),
|
|
67
|
+
rdfsReasoner: reasoner('rdfs'),
|
|
68
|
+
owlRlReasoner: reasoner('owl-rl'),
|
|
69
|
+
rhoDfReasoner: reasoner('rho-df'),
|
|
70
|
+
shaclValidator: async (wire, params) => {
|
|
71
|
+
const { shapes, ...opts } = params;
|
|
72
|
+
if (shapes === undefined) throw new TypeError('node shaclValidator: params.shapes is required');
|
|
73
|
+
const r = await shaclValidate(wire.graph, shapes, opts);
|
|
74
|
+
return { artefact: { conforms: r.conforms, report: r.report } };
|
|
75
|
+
},
|
|
76
|
+
shippedEvaluator: async (wire, params) => {
|
|
77
|
+
const { query: sparql, ...opts } = params;
|
|
78
|
+
if (typeof sparql !== 'string') throw new TypeError('node shippedEvaluator: params.query must be a string');
|
|
79
|
+
const answer = await query(wire.graph, sparql, opts);
|
|
80
|
+
return answer instanceof Dataset ? { graph: answer, artefact: { form: 'graph' } }
|
|
81
|
+
: { artefact: { answer } };
|
|
82
|
+
},
|
|
83
|
+
ntriplesDocumentWriter: writer('ntriples'),
|
|
84
|
+
nquadsDocumentWriter: writer('nquads'),
|
|
85
|
+
turtleDocumentWriter: writer('turtle'),
|
|
86
|
+
trigDocumentWriter: writer('trig'),
|
|
87
|
+
rdfxmlDocumentWriter: writer('rdfxml'),
|
|
88
|
+
rdfcStage: async (wire, params) => ({
|
|
89
|
+
artefact: { canonical: await canonicalize(wire.graph, params) },
|
|
90
|
+
}),
|
|
91
|
+
rdfsExplainGraphStage: async (wire, params) => {
|
|
92
|
+
const { description, ...opts } = params;
|
|
93
|
+
if (description === undefined) {
|
|
94
|
+
throw new TypeError('node rdfsExplainGraphStage: params.description is required');
|
|
95
|
+
}
|
|
96
|
+
return { artefact: await explain(wire.graph, description, { ...opts, profile: 'rdfs' }) };
|
|
97
|
+
},
|
|
98
|
+
};
|
|
99
|
+
|
|
100
|
+
let stageCache = null;
|
|
101
|
+
let engine = null; // the entry stages() loaded; node() calls stages(), so pipe() has it
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* The engine's stage list (op stageList), one element per registry entry.
|
|
105
|
+
* @returns {Promise<object[]>}
|
|
106
|
+
*/
|
|
107
|
+
async function stages() {
|
|
108
|
+
if (!stageCache) {
|
|
109
|
+
stageCache = (async () => {
|
|
110
|
+
const e = await entry();
|
|
111
|
+
requireEntryFn(e, 'stageList', 'stages');
|
|
112
|
+
engine = e;
|
|
113
|
+
const r = entryResult(e.stageList(), 'stages');
|
|
114
|
+
return Object.freeze(r.stages.map((s) => Object.freeze({
|
|
115
|
+
...s,
|
|
116
|
+
theorems: Object.freeze([...s.theorems]),
|
|
117
|
+
assumptions: Object.freeze([...s.assumptions]),
|
|
118
|
+
tests: Object.freeze([...s.tests]),
|
|
119
|
+
bound: Object.prototype.hasOwnProperty.call(RUNNERS, s.name),
|
|
120
|
+
})));
|
|
121
|
+
})();
|
|
122
|
+
stageCache.catch(() => { stageCache = null; });
|
|
123
|
+
}
|
|
124
|
+
return stageCache;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
const NODE = Symbol('factoidal.node');
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* A node for the registered stage `name`, with its parameters (the
|
|
131
|
+
* second argument of the package function it calls, e.g. `{shapes}`,
|
|
132
|
+
* `{query}`, `{description}`, reader options).
|
|
133
|
+
* @param {string} name a stageList `name`
|
|
134
|
+
* @param {object} [params]
|
|
135
|
+
* @returns {Promise<object>}
|
|
136
|
+
*/
|
|
137
|
+
async function node(name, params) {
|
|
138
|
+
if (typeof name !== 'string') throw new TypeError('node: name must be a string');
|
|
139
|
+
if (params !== undefined && (params === null || typeof params !== 'object')) {
|
|
140
|
+
throw new TypeError('node: params must be an object');
|
|
141
|
+
}
|
|
142
|
+
const info = (await stages()).find((s) => s.name === name);
|
|
143
|
+
if (!info) {
|
|
144
|
+
throw new FactoidalError(`node: no registered stage named '${name}' (see stages())`,
|
|
145
|
+
{ code: 'option.invalid-value' });
|
|
146
|
+
}
|
|
147
|
+
return Object.freeze({
|
|
148
|
+
[NODE]: true,
|
|
149
|
+
stage: info.name,
|
|
150
|
+
kind: info.kind,
|
|
151
|
+
status: info.status,
|
|
152
|
+
contract: info.contract,
|
|
153
|
+
assumptions: info.assumptions,
|
|
154
|
+
wireIn: info.wireIn,
|
|
155
|
+
wireOut: info.wireOut,
|
|
156
|
+
runnable: info.runnable,
|
|
157
|
+
bound: info.bound,
|
|
158
|
+
params: Object.freeze({ ...(params || {}) }),
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
function nodesOf(items, who) {
|
|
163
|
+
const out = [];
|
|
164
|
+
for (const n of items) {
|
|
165
|
+
if (n && n[NODE]) out.push(n);
|
|
166
|
+
else if (n && Array.isArray(n.nodes)) out.push(...nodesOf(n.nodes, who));
|
|
167
|
+
else throw new TypeError(`${who}: expected nodes from node() or pipelines from pipe()`);
|
|
168
|
+
}
|
|
169
|
+
return out;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* The engine's check of the stages of `nodes` in order (op pipelineCheck),
|
|
174
|
+
* or null when the loaded engine has no such op.
|
|
175
|
+
*/
|
|
176
|
+
function check(nodes) {
|
|
177
|
+
if (!engine || typeof engine.pipelineCheck !== 'function') return null;
|
|
178
|
+
const r = entryResult(engine.pipelineCheck(JSON.stringify(nodes.map((n) => n.stage))), 'pipelineCheck');
|
|
179
|
+
return Object.freeze({
|
|
180
|
+
wellTyped: r.wellTyped,
|
|
181
|
+
wireIn: r.wireIn,
|
|
182
|
+
wireOut: r.wireOut,
|
|
183
|
+
mismatch: r.mismatch === null ? null : Object.freeze({ ...r.mismatch }),
|
|
184
|
+
conversions: Object.freeze(r.conversions.map((c) => Object.freeze({ ...c }))),
|
|
185
|
+
assumptions: Object.freeze([...r.assumptions]),
|
|
186
|
+
contract: Object.freeze({ theorem: r.contract.theorem, stages: Object.freeze([...r.contract.stages]) }),
|
|
187
|
+
stages: Object.freeze(r.stages.map((s) => Object.freeze({ ...s, assumptions: Object.freeze([...s.assumptions]) }))),
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/**
|
|
192
|
+
* A pipeline: the nodes in order (pipelines are flattened). `check` is
|
|
193
|
+
* the engine's op pipelineCheck on the nodes' stages; `wellTyped` and
|
|
194
|
+
* `assumptions` (the union of the nodes' named assumptions) are its
|
|
195
|
+
* fields. Without the op: `check` and `wellTyped` are null and
|
|
196
|
+
* `assumptions` is the union of the nodes' fields.
|
|
197
|
+
* @returns {{nodes: object[], assumptions: string[], wellTyped: boolean|null, check: object|null}}
|
|
198
|
+
*/
|
|
199
|
+
function pipe(...items) {
|
|
200
|
+
const nodes = nodesOf(items, 'pipe');
|
|
201
|
+
if (nodes.length === 0) throw new TypeError('pipe: no nodes');
|
|
202
|
+
const c = check(nodes);
|
|
203
|
+
return Object.freeze({
|
|
204
|
+
nodes: Object.freeze(nodes),
|
|
205
|
+
assumptions: c ? c.assumptions : Object.freeze(unionAssumptions(nodes)),
|
|
206
|
+
wellTyped: c ? c.wellTyped : null,
|
|
207
|
+
check: c,
|
|
208
|
+
});
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
const isWire = (v) => v !== null && typeof v === 'object' && !(v instanceof Dataset) &&
|
|
212
|
+
Object.prototype.hasOwnProperty.call(v, 'graph') && Array.isArray(v.evidence);
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* Run a node or a pipeline. `input` is a wire value `{graph, evidence}`
|
|
216
|
+
* or the graph itself (text for a reader, else a Dataset or data input).
|
|
217
|
+
* `options.unchecked: true` lets stages that are not `proved` run,
|
|
218
|
+
* `missing` ones too when the package has a runner for them; each such
|
|
219
|
+
* step's record says so (evidence.mjs). Without it, a `missing` stage
|
|
220
|
+
* refuses before any node runs, and so does a pipeline that the engine's
|
|
221
|
+
* check finds not well typed (op pipelineCheck); with it such a pipeline
|
|
222
|
+
* runs and `proved` is false.
|
|
223
|
+
* `assumptions`: the assumptions of the records on the input wire, then
|
|
224
|
+
* the engine's union for this pipeline (op pipelineCheck), no duplicate.
|
|
225
|
+
* @returns {Promise<{graph: any, evidence: object[], assumptions: string[],
|
|
226
|
+
* proved: boolean, uncheckedSteps: string[], wellTyped: boolean|null}>}
|
|
227
|
+
*/
|
|
228
|
+
async function run(pipeline, input, options) {
|
|
229
|
+
if (options !== undefined && (options === null || typeof options !== 'object')) {
|
|
230
|
+
throw new TypeError('run: options must be an object');
|
|
231
|
+
}
|
|
232
|
+
const unchecked = !!(options && options.unchecked);
|
|
233
|
+
const nodes = nodesOf([pipeline], 'run');
|
|
234
|
+
for (const n of nodes) {
|
|
235
|
+
const why = refusalFor(n, unchecked, Object.prototype.hasOwnProperty.call(RUNNERS, n.stage));
|
|
236
|
+
if (why) throw why;
|
|
237
|
+
}
|
|
238
|
+
const c = check(nodes);
|
|
239
|
+
if (c && !c.wellTyped && !unchecked) {
|
|
240
|
+
const m = c.mismatch;
|
|
241
|
+
throw new FactoidalError(`run: the pipeline is not well typed: stage ${m.at} ` +
|
|
242
|
+
`(${nodes[m.at].stage}) writes ${m.out}, stage ${m.at + 1} (${nodes[m.at + 1].stage}) ` +
|
|
243
|
+
`reads ${m.in} (pass { unchecked: true } to run it without a guarantee)`, { code: 'op.failed' });
|
|
244
|
+
}
|
|
245
|
+
const prior = isWire(input) ? input.evidence : [];
|
|
246
|
+
let wire = isWire(input) ? { graph: input.graph, evidence: [...input.evidence] }
|
|
247
|
+
: { graph: input, evidence: [] };
|
|
248
|
+
let owned = null; // a Dataset an earlier node of this run made
|
|
249
|
+
for (const n of nodes) {
|
|
250
|
+
const out = await RUNNERS[n.stage](wire, n.params);
|
|
251
|
+
const record = makeRecord(n, out.artefact, unchecked);
|
|
252
|
+
let graph = wire.graph;
|
|
253
|
+
if (out.graph !== undefined && out.graph !== wire.graph) {
|
|
254
|
+
if (owned && owned === wire.graph && typeof owned.dispose === 'function') owned.dispose();
|
|
255
|
+
graph = out.graph;
|
|
256
|
+
owned = out.graph instanceof Dataset ? out.graph : null;
|
|
257
|
+
}
|
|
258
|
+
wire = { graph, evidence: [...wire.evidence, record] };
|
|
259
|
+
}
|
|
260
|
+
const s = summarize(wire.evidence);
|
|
261
|
+
if (!c) return { graph: wire.graph, evidence: wire.evidence, ...s, wellTyped: null };
|
|
262
|
+
return {
|
|
263
|
+
graph: wire.graph,
|
|
264
|
+
evidence: wire.evidence,
|
|
265
|
+
assumptions: prior.length === 0 ? [...c.assumptions]
|
|
266
|
+
: unionLists([unionAssumptions(prior), c.assumptions]),
|
|
267
|
+
proved: s.proved && c.wellTyped,
|
|
268
|
+
uncheckedSteps: s.uncheckedSteps,
|
|
269
|
+
wellTyped: c.wellTyped,
|
|
270
|
+
};
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
return { stages, node, pipe, run };
|
|
274
|
+
}
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
// The LWS reader of an SBM11 generation for lib/sbm11.mjs: the files of a
|
|
2
|
+
// generation (or of a collection root with its CURRENT pointer) stored as
|
|
3
|
+
// Linked Web Storage resources, one resource per file, under one container
|
|
4
|
+
// URL (https://github.com/danbri/factoidal/issues/1010 KR3, read part;
|
|
5
|
+
// LWS 1.0: https://w3c.github.io/lws-protocol/lws10-core/).
|
|
6
|
+
//
|
|
7
|
+
// WHAT THIS FILE IS ALLOWED TO DO
|
|
8
|
+
// Move bytes. The engine names every file and checks every length against
|
|
9
|
+
// the manifest (formal/lean4/L4Factoidal/Storage/Sbm11/HostSession.lean,
|
|
10
|
+
// through the storage medium of formal/lean4/L4Factoidal/Storage/Medium.lean).
|
|
11
|
+
// This file maps a file name to a resource URL, makes HTTP GETs, and hands
|
|
12
|
+
// the bytes over. It reads no manifest and no byte of a file.
|
|
13
|
+
//
|
|
14
|
+
// HTTP (RFC 9110 section 14): each read asks for `Range: bytes=a-b`. A
|
|
15
|
+
// partial answer is used only when it is status 206 and its Content-Range is
|
|
16
|
+
// exactly `bytes a-b/N`. Any other successful answer is the whole
|
|
17
|
+
// representation: it is kept in memory and every later read of that file is
|
|
18
|
+
// cut from it, with no further request. A changed entity tag between two
|
|
19
|
+
// reads of one file refuses the read (the bytes of two versions are never
|
|
20
|
+
// mixed). Limits: no Merkle check here (the engine's Verify.none path), no
|
|
21
|
+
// eviction of the in-memory copies, no retry.
|
|
22
|
+
//
|
|
23
|
+
// EVIDENCE
|
|
24
|
+
// `evidence()` lists each resource read: { resource, etag, lastModified,
|
|
25
|
+
// requests, partial, bytes } -- `partial` the number of 206 answers. The
|
|
26
|
+
// store from openSbm11 answers it as `store.evidence`.
|
|
27
|
+
//
|
|
28
|
+
// SYNCHRONOUS TRANSPORT
|
|
29
|
+
// The engine is inside WebAssembly and asks for bytes synchronously, so the
|
|
30
|
+
// reader needs a synchronous GET: `transport(url, headers) -> { status,
|
|
31
|
+
// header(name) -> string | null, bytes: Uint8Array }`. In Node the default
|
|
32
|
+
// is lib/sync-http-node.mjs (fetch in a worker thread, Atomics.wait); in a
|
|
33
|
+
// browser worker it is a synchronous XMLHttpRequest. A browser window
|
|
34
|
+
// thread has neither: pass `transport` or run the engine in a worker.
|
|
35
|
+
|
|
36
|
+
function containerUrl (url) {
|
|
37
|
+
const s = String(url)
|
|
38
|
+
return s.endsWith('/') ? s : s + '/'
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function resourceUrl (base, file) {
|
|
42
|
+
return new URL(String(file).split('/').map(encodeURIComponent).join('/'), base).href
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** A synchronous GET with XMLHttpRequest (a browser worker). */
|
|
46
|
+
export function xhrSyncTransport (url, headers) {
|
|
47
|
+
const x = new XMLHttpRequest()
|
|
48
|
+
x.open('GET', url, false)
|
|
49
|
+
x.responseType = 'arraybuffer'
|
|
50
|
+
for (const [k, v] of Object.entries(headers)) x.setRequestHeader(k, v)
|
|
51
|
+
x.send()
|
|
52
|
+
return {
|
|
53
|
+
status: x.status,
|
|
54
|
+
header: (name) => x.getResponseHeader(name),
|
|
55
|
+
bytes: new Uint8Array(x.response || new ArrayBuffer(0))
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function parseContentRange (value) {
|
|
60
|
+
const m = /^bytes\s+(\d+)-(\d+)\/(\d+|\*)$/.exec(String(value || '').trim())
|
|
61
|
+
if (!m) return null
|
|
62
|
+
return { first: Number(m[1]), last: Number(m[2]), total: m[3] === '*' ? null : Number(m[3]) }
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function unsatisfiedTotal (value) {
|
|
66
|
+
const m = /^bytes\s+\*\/(\d+)$/.exec(String(value || '').trim())
|
|
67
|
+
return m ? Number(m[1]) : null
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* A reader ({ readAt, size, evidence, close }) of the SBM11 files held as
|
|
72
|
+
* LWS resources under the container `url`.
|
|
73
|
+
*
|
|
74
|
+
* @param {string} url the container that holds CURRENT and the generation
|
|
75
|
+
* (or the generation container itself)
|
|
76
|
+
* @param {object} [opts]
|
|
77
|
+
* @param {Function} [opts.transport] the synchronous GET (see above)
|
|
78
|
+
* @param {string} [opts.auth] a value for the Authorization header, such as
|
|
79
|
+
* `Bearer <access token>` (an LWS access token)
|
|
80
|
+
* @param {object} [opts.headers] further request headers
|
|
81
|
+
*/
|
|
82
|
+
export function lwsSbm11Reader (url, opts = {}) {
|
|
83
|
+
const base = containerUrl(url)
|
|
84
|
+
let transport = opts.transport
|
|
85
|
+
if (typeof transport !== 'function') {
|
|
86
|
+
if (typeof XMLHttpRequest === 'function' && typeof document === 'undefined') {
|
|
87
|
+
transport = xhrSyncTransport
|
|
88
|
+
} else {
|
|
89
|
+
throw new TypeError('lwsSbm11Reader: no synchronous transport here; pass opts.transport (Node: @factoidal/core gives one)')
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
const extra = { ...(opts.headers || {}) }
|
|
93
|
+
if (opts.auth) extra.authorization = String(opts.auth)
|
|
94
|
+
// file -> { url, etag, lastModified, total, whole, requests, partial, bytes }
|
|
95
|
+
const files = new Map()
|
|
96
|
+
const errors = []
|
|
97
|
+
|
|
98
|
+
function entry (file) {
|
|
99
|
+
let e = files.get(file)
|
|
100
|
+
if (!e) {
|
|
101
|
+
e = { url: resourceUrl(base, file), etag: null, lastModified: null, total: undefined, whole: null, requests: 0, partial: 0, bytes: 0 }
|
|
102
|
+
files.set(file, e)
|
|
103
|
+
}
|
|
104
|
+
return e
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function fail (file, message) {
|
|
108
|
+
errors.push(`${file}: ${message}`)
|
|
109
|
+
return null
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
function get (e, file, first, last) {
|
|
113
|
+
const headers = { ...extra, range: `bytes=${first}-${last}` }
|
|
114
|
+
let r
|
|
115
|
+
try {
|
|
116
|
+
r = transport(e.url, headers)
|
|
117
|
+
} catch (error) {
|
|
118
|
+
fail(file, `GET ${e.url}: ${error && error.message ? error.message : error}`)
|
|
119
|
+
return null
|
|
120
|
+
}
|
|
121
|
+
e.requests += 1
|
|
122
|
+
const tag = r.header('etag')
|
|
123
|
+
if (r.status >= 200 && r.status < 300) {
|
|
124
|
+
if (e.etag !== null && tag !== e.etag) {
|
|
125
|
+
fail(file, `the entity tag changed from ${e.etag} to ${tag}`)
|
|
126
|
+
return null
|
|
127
|
+
}
|
|
128
|
+
e.etag = tag
|
|
129
|
+
e.lastModified = r.header('last-modified')
|
|
130
|
+
e.bytes += r.bytes.length
|
|
131
|
+
}
|
|
132
|
+
return r
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
function sizeOf (file) {
|
|
136
|
+
const e = entry(file)
|
|
137
|
+
if (e.total !== undefined) return e.total
|
|
138
|
+
const r = get(e, file, 0, 0)
|
|
139
|
+
if (r === null) return null
|
|
140
|
+
const cr = parseContentRange(r.header('content-range'))
|
|
141
|
+
if (r.status === 206 && cr && cr.first === 0 && cr.last === 0 && cr.total !== null && r.bytes.length === 1) {
|
|
142
|
+
e.partial += 1
|
|
143
|
+
e.total = cr.total
|
|
144
|
+
} else if (r.status === 416 && unsatisfiedTotal(r.header('content-range')) !== null) {
|
|
145
|
+
e.total = unsatisfiedTotal(r.header('content-range'))
|
|
146
|
+
e.whole = new Uint8Array(0)
|
|
147
|
+
} else if (r.status === 200) {
|
|
148
|
+
e.whole = r.bytes
|
|
149
|
+
e.total = r.bytes.length
|
|
150
|
+
} else if (r.status === 404 || r.status === 410) {
|
|
151
|
+
e.total = null
|
|
152
|
+
} else {
|
|
153
|
+
fail(file, `GET ${e.url}: status ${r.status}`)
|
|
154
|
+
return null
|
|
155
|
+
}
|
|
156
|
+
return e.total
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
return {
|
|
160
|
+
readAt (file, offset, length) {
|
|
161
|
+
const e = entry(file)
|
|
162
|
+
if (length === 0) return new Uint8Array(0)
|
|
163
|
+
if (e.whole !== null) {
|
|
164
|
+
return offset + length <= e.whole.length ? e.whole.subarray(offset, offset + length) : null
|
|
165
|
+
}
|
|
166
|
+
const last = offset + length - 1
|
|
167
|
+
const r = get(e, file, offset, last)
|
|
168
|
+
if (r === null) return null
|
|
169
|
+
const cr = parseContentRange(r.header('content-range'))
|
|
170
|
+
if (r.status === 206 && cr && cr.first === offset && cr.last === last && r.bytes.length === length) {
|
|
171
|
+
e.partial += 1
|
|
172
|
+
return r.bytes
|
|
173
|
+
}
|
|
174
|
+
if (r.status === 200) {
|
|
175
|
+
e.whole = r.bytes
|
|
176
|
+
e.total = r.bytes.length
|
|
177
|
+
return offset + length <= r.bytes.length ? r.bytes.subarray(offset, offset + length) : null
|
|
178
|
+
}
|
|
179
|
+
return fail(file, `GET ${e.url} bytes ${offset}-${last}: status ${r.status}`)
|
|
180
|
+
},
|
|
181
|
+
size (file) {
|
|
182
|
+
return sizeOf(file)
|
|
183
|
+
},
|
|
184
|
+
/** Each resource read: { resource, etag, lastModified, requests, partial, bytes }. */
|
|
185
|
+
evidence () {
|
|
186
|
+
const out = []
|
|
187
|
+
for (const e of files.values()) {
|
|
188
|
+
if (e.requests === 0 || e.etag === null) continue
|
|
189
|
+
out.push({ resource: e.url, etag: e.etag, lastModified: e.lastModified, requests: e.requests, partial: e.partial, bytes: e.bytes })
|
|
190
|
+
}
|
|
191
|
+
return out
|
|
192
|
+
},
|
|
193
|
+
/** The transport failures, as text (a failed read reaches the engine as no bytes). */
|
|
194
|
+
errors () { return errors.slice() },
|
|
195
|
+
close () {
|
|
196
|
+
files.clear()
|
|
197
|
+
if (typeof opts.onClose === 'function') opts.onClose()
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
// The Node reader of an SBM11 generation directory for lib/sbm11.mjs: the
|
|
2
|
+
// source is a directory path, and a file of the generation is read with
|
|
3
|
+
// fs.readSync at an offset (https://github.com/danbri/factoidal/issues/1009
|
|
4
|
+
// KR1). It moves bytes only (iron rule 7); the engine checks every length.
|
|
5
|
+
//
|
|
6
|
+
// One open file descriptor per file, kept while the reader lives: a query
|
|
7
|
+
// makes many small reads of the same few files.
|
|
8
|
+
|
|
9
|
+
import { closeSync, fstatSync, openSync, readSync, statSync } from 'node:fs'
|
|
10
|
+
import { join } from 'node:path'
|
|
11
|
+
|
|
12
|
+
export function nodeDirectoryReader (dir) {
|
|
13
|
+
const fds = new Map()
|
|
14
|
+
function fdOf (file) {
|
|
15
|
+
let fd = fds.get(file)
|
|
16
|
+
if (fd === undefined) {
|
|
17
|
+
fd = openSync(join(dir, file), 'r')
|
|
18
|
+
fds.set(file, fd)
|
|
19
|
+
}
|
|
20
|
+
return fd
|
|
21
|
+
}
|
|
22
|
+
return {
|
|
23
|
+
readAt (file, offset, length) {
|
|
24
|
+
const out = new Uint8Array(length)
|
|
25
|
+
const fd = fdOf(file)
|
|
26
|
+
let done = 0
|
|
27
|
+
while (done < length) {
|
|
28
|
+
const n = readSync(fd, out, done, length - done, offset + done)
|
|
29
|
+
if (n === 0) break
|
|
30
|
+
done += n
|
|
31
|
+
}
|
|
32
|
+
return done === length ? out : null
|
|
33
|
+
},
|
|
34
|
+
size (file) {
|
|
35
|
+
try {
|
|
36
|
+
const fd = fds.get(file)
|
|
37
|
+
const st = fd === undefined ? statSync(join(dir, file)) : fstatSync(fd)
|
|
38
|
+
return st.isFile() ? st.size : null
|
|
39
|
+
} catch {
|
|
40
|
+
return null
|
|
41
|
+
}
|
|
42
|
+
},
|
|
43
|
+
close () {
|
|
44
|
+
for (const fd of fds.values()) closeSync(fd)
|
|
45
|
+
fds.clear()
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** A default-reader factory: one reader per directory, kept. */
|
|
51
|
+
export function nodeDefaultReaderFactory () {
|
|
52
|
+
const byDir = new Map()
|
|
53
|
+
return (source) => {
|
|
54
|
+
let r = byDir.get(source)
|
|
55
|
+
if (!r) {
|
|
56
|
+
r = nodeDirectoryReader(source)
|
|
57
|
+
byDir.set(source, r)
|
|
58
|
+
}
|
|
59
|
+
return r
|
|
60
|
+
}
|
|
61
|
+
}
|
package/lib/sbm11.mjs
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
// SPARQL over an SBM11 generation, in Node and in the browser, through the
|
|
2
|
+
// wasm ops sbm11Open / sbm11Query / sbm11Close
|
|
3
|
+
// (formal/lean4/Wasm/Ops/Sbm11.lean; https://github.com/danbri/factoidal/issues/1009 KR1).
|
|
4
|
+
//
|
|
5
|
+
// WHAT THIS FILE IS ALLOWED TO DO
|
|
6
|
+
// Move bytes. The engine asks for (source, file, offset, length) through the
|
|
7
|
+
// bridge globalThis.__factoidalReadAt, and for the size of a file through
|
|
8
|
+
// globalThis.__factoidalFileSize (the EM_JS bodies in
|
|
9
|
+
// formal/lean4/ffi/l4_hostread.c call them synchronously). This file finds
|
|
10
|
+
// the reader that serves `source` and answers. It never reads a manifest,
|
|
11
|
+
// checks a length or a digest, or decides which file a query needs: the
|
|
12
|
+
// engine names every file and checks every answer against the manifest
|
|
13
|
+
// (formal/lean4/L4Factoidal/Storage/Sbm11/HostSession.lean). Iron rule 7.
|
|
14
|
+
//
|
|
15
|
+
// READERS
|
|
16
|
+
// A reader is { readAt(file, offset, length) -> Uint8Array, size(file) ->
|
|
17
|
+
// number | null }, both synchronous: the engine is inside WebAssembly and
|
|
18
|
+
// cannot wait for a promise. Node: lib/sbm11-node.mjs reads a directory with
|
|
19
|
+
// fs. A browser worker: a reader over OPFS FileSystemSyncAccessHandle
|
|
20
|
+
// (stage 4 of https://github.com/danbri/factoidal/issues/1007). A caller
|
|
21
|
+
// registers a reader under a source name; a source with no registered reader
|
|
22
|
+
// goes to the default reader (Node: the directory of that name).
|
|
23
|
+
|
|
24
|
+
const readers = new Map()
|
|
25
|
+
let defaultReader = null
|
|
26
|
+
let bridgeInstalled = false
|
|
27
|
+
|
|
28
|
+
function readerFor (source) {
|
|
29
|
+
const r = readers.get(source)
|
|
30
|
+
if (r) return r
|
|
31
|
+
if (defaultReader) return defaultReader(source)
|
|
32
|
+
return null
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function installBridge () {
|
|
36
|
+
if (bridgeInstalled) return
|
|
37
|
+
bridgeInstalled = true
|
|
38
|
+
// A thrown error and a wrong length both reach Lean as "the host gave no
|
|
39
|
+
// bytes"; the engine then refuses the read with the file name.
|
|
40
|
+
globalThis.__factoidalReadAt = (source, file, offset, length) => {
|
|
41
|
+
const r = readerFor(source)
|
|
42
|
+
if (!r) return null
|
|
43
|
+
return r.readAt(file, offset, length)
|
|
44
|
+
}
|
|
45
|
+
globalThis.__factoidalFileSize = (source, file) => {
|
|
46
|
+
const r = readerFor(source)
|
|
47
|
+
if (!r) return null
|
|
48
|
+
return r.size(file)
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Serve `source` with `reader` ({ readAt(file, offset, length), size(file) },
|
|
54
|
+
* both synchronous). `null` removes the registration.
|
|
55
|
+
*/
|
|
56
|
+
export function registerSbm11Source (source, reader) {
|
|
57
|
+
installBridge()
|
|
58
|
+
if (reader === null) { readers.delete(String(source)); return }
|
|
59
|
+
if (!reader || typeof reader.readAt !== 'function' || typeof reader.size !== 'function') {
|
|
60
|
+
throw new TypeError('registerSbm11Source: reader needs readAt(file, offset, length) and size(file)')
|
|
61
|
+
}
|
|
62
|
+
readers.set(String(source), reader)
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** The reader for a source with no registration: `(source) => reader | null`. */
|
|
66
|
+
export function setDefaultSbm11Reader (factory) {
|
|
67
|
+
installBridge()
|
|
68
|
+
defaultReader = factory
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function envelope (raw) {
|
|
72
|
+
const j = typeof raw === 'string' ? JSON.parse(raw) : raw
|
|
73
|
+
if (!j || j.ok !== true) {
|
|
74
|
+
const e = new Error((j && j.error) || 'sbm11: unknown error')
|
|
75
|
+
e.code = 'SBM11'
|
|
76
|
+
throw e
|
|
77
|
+
}
|
|
78
|
+
return j
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* The SBM11 functions over an engine loader: `loadEngine()` resolves to the
|
|
83
|
+
* engine handle (`call(op, args)`).
|
|
84
|
+
*/
|
|
85
|
+
export function sbm11Api (loadEngine) {
|
|
86
|
+
/**
|
|
87
|
+
* Open an SBM11 generation (a generation directory, or a collection root
|
|
88
|
+
* whose CURRENT file names one). Returns { generation, terms, query(sparql,
|
|
89
|
+
* options), close() }. `query` answers { path, rows, format, result, json }:
|
|
90
|
+
* `result` is the text `l4sbm11 query` writes (SPARQL JSON for SELECT and
|
|
91
|
+
* ASK, N-Triples for CONSTRUCT and DESCRIBE), `json` its parse for
|
|
92
|
+
* SELECT and ASK.
|
|
93
|
+
*/
|
|
94
|
+
async function openSbm11 (source, options = {}) {
|
|
95
|
+
installBridge()
|
|
96
|
+
if (options.reader) registerSbm11Source(source, options.reader)
|
|
97
|
+
const eng = await loadEngine()
|
|
98
|
+
const opened = envelope(eng.call('sbm11Open', [String(source)]))
|
|
99
|
+
const handle = opened.handle
|
|
100
|
+
let open = true
|
|
101
|
+
const reader = options.reader || readers.get(String(source))
|
|
102
|
+
return {
|
|
103
|
+
handle,
|
|
104
|
+
/** The resources the reader read (`lwsSbm11Reader`): `[{ resource,
|
|
105
|
+
* etag, lastModified, requests, partial, bytes }]`; undefined for a
|
|
106
|
+
* reader that records none. */
|
|
107
|
+
get evidence () {
|
|
108
|
+
return reader && typeof reader.evidence === 'function' ? reader.evidence() : undefined
|
|
109
|
+
},
|
|
110
|
+
generation: opened.generation,
|
|
111
|
+
terms: Number(opened.terms),
|
|
112
|
+
query (sparql, qopts = {}) {
|
|
113
|
+
if (!open) throw new Error('sbm11: the store is closed')
|
|
114
|
+
const o = {}
|
|
115
|
+
if (qopts.base != null) o.base = String(qopts.base)
|
|
116
|
+
if (qopts.sparql12) o.sparql12 = true
|
|
117
|
+
if (qopts.reference) o.reference = true
|
|
118
|
+
const args = [handle, String(sparql), JSON.stringify(o)]
|
|
119
|
+
// The byte-region entry returns the result bytes outside the
|
|
120
|
+
// envelope; a large answer is then not escaped into a JSON string
|
|
121
|
+
// and parsed again (Wasm/Ops/Sbm11.lean sbm11QueryBlob).
|
|
122
|
+
let j, bytes
|
|
123
|
+
if (typeof eng.callBlobIO === 'function') {
|
|
124
|
+
const answer = eng.callBlobIO('sbm11Query', args, new Uint8Array(0))
|
|
125
|
+
j = envelope(answer.envelope)
|
|
126
|
+
bytes = answer.bytes
|
|
127
|
+
} else {
|
|
128
|
+
j = envelope(eng.call('sbm11Query', args))
|
|
129
|
+
}
|
|
130
|
+
let text = bytes === undefined ? j.result : undefined
|
|
131
|
+
return {
|
|
132
|
+
path: j.path,
|
|
133
|
+
rows: Number(j.rows),
|
|
134
|
+
format: j.format,
|
|
135
|
+
bytesRead: Number(j.bytesRead),
|
|
136
|
+
reads: Number(j.reads),
|
|
137
|
+
/** the answer as bytes (UTF-8) */
|
|
138
|
+
get bytes () { return bytes ?? new TextEncoder().encode(text) },
|
|
139
|
+
/** the answer as text */
|
|
140
|
+
get result () {
|
|
141
|
+
if (text === undefined) text = new TextDecoder().decode(bytes)
|
|
142
|
+
return text
|
|
143
|
+
},
|
|
144
|
+
get json () { return j.format === 'srj' ? JSON.parse(this.result) : undefined }
|
|
145
|
+
}
|
|
146
|
+
},
|
|
147
|
+
close () {
|
|
148
|
+
if (!open) return
|
|
149
|
+
open = false
|
|
150
|
+
envelope(eng.call('sbm11Close', [handle]))
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
return { openSbm11 }
|
|
155
|
+
}
|