@factoidal/core 0.2.0 → 0.4.0

package/bin/store.mjs

@@ -0,0 +1,238 @@
+// Driving the three WebAssembly store operations from a JavaScript host.
+// https://github.com/danbri/factoidal/issues/641
+//
+// WHAT THIS FILE IS ALLOWED TO DO
+// Read CURRENT, read a manifest file by name, read the artifact files the
+// engine's plan named, concatenate their bytes, and hand them over. It
+// never parses a manifest, never verifies a digest, never decodes a block
+// and never decides which artifact answers a query. Every one of those is
+// a format decision and it lives in `formal/lean4/Wasm/Ops/Store.lean`
+// (iron rule 7 of CLAUDE.md). A reviewer who finds a magic number, a
+// field offset or a hash in this file has found a rule violation.
+//
+// The operations and their envelopes are described in
+// `docs/designissues/2026-09-03-wasm-shardborough-store-ops.md`.
+
+import { openCollection, readWhole } from '../store-host/index.mjs'
+import { joinPath } from '../store-host/paths.mjs'
+import { hexOfBytes } from './engine.mjs'
+
+/** The manifest file names a generation directory can carry, in the order
+ *  `Harness.ShardMerklePread.readManifest` tries them. */
+const MANIFEST_NAMES = ['manifest.sbm2', 'manifest.sbm1']
+
+/**
+ * An error the store operations reported. `capName` is set when the
+ * refusal was one of the operation's caps; `message` is always the
+ * engine's own words.
+ */
+export class StoreOperationError extends Error {
+  constructor (message, detail = {}) {
+    super(message)
+    this.name = 'StoreOperationError'
+    this.capValue = detail.capValue === undefined ? null : detail.capValue
+    this.capLimit = detail.capLimit === undefined ? null : detail.capLimit
+    this.digestKey = detail.digestKey === undefined ? null : detail.digestKey
+    this.stackLimit = detail.stackLimit === true
+  }
+}
+
+// The engine's refusals arrive as an Error whose message is
+// "l4factoidal: <the operation's own text>". These two patterns say only
+// WHICH refusal it was, so the command can add a next step; the text the
+// user sees is always the engine's, never a rewrite of it.
+const CAP_PATTERN = /the plan selects (\d+) (?:artifacts|artifact bytes|rows), the cap is (\d+)/
+const DIGEST_PATTERN = /artifact '([^']*)' does not match the SHA-256/
+
+function asStoreError (error) {
+  const raw = error && error.message ? String(error.message) : String(error)
+  const message = raw.replace(/^l4factoidal:\s*/, '')
+  // Not a refusal by the engine: the host runtime ran out of call stack
+  // inside the wasm module. Some evaluator paths recurse once per row.
+  // Measured 2026-09-03 against the committed wasm, on a 6455-row
+  // generation: `SELECT ?s ?p ?o WHERE { ?s ?p ?o }` and the same query
+  // with `ORDER BY` overflow under Node's default WebAssembly frame
+  // budget, while `SELECT *`, and either query with a LIMIT, do not.
+  // `node --stack-size=4000` clears all of them, and Deno clears them at
+  // its own default.
+  if (error instanceof RangeError || message.indexOf('call stack size exceeded') >= 0) {
+    return new StoreOperationError(message, { stackLimit: true })
+  }
+  const cap = CAP_PATTERN.exec(message)
+  if (cap !== null) {
+    return new StoreOperationError(message,
+      { capValue: Number(cap[1]), capLimit: Number(cap[2]) })
+  }
+  const digest = DIGEST_PATTERN.exec(message)
+  if (digest !== null) {
+    return new StoreOperationError(message, { digestKey: digest[1] })
+  }
+  return new StoreOperationError(message)
+}
+
+/**
+ * What to print when the runtime, not the engine, ran out of call stack.
+ *
+ * One copy of this text, used by every command that can hit the frame
+ * budget. `remedy` is the one line that differs: a query can be made
+ * smaller with a LIMIT, a pack cannot.
+ *
+ * The pack path normally never reaches this, because it runs on a worker
+ * thread with a raised stack (`bin/pack-host.mjs`,
+ * https://github.com/danbri/factoidal/issues/649). It is what a reader
+ * sees when the worker route is refused with --no-worker, is unavailable
+ * on their platform, or is not enough.
+ *
+ * @param {string} remedy
+ * @returns {string[]} the lines, in order, for stderr
+ */
+export function stackLimitAdvice (remedy) {
+  return [
+    'The runtime ran out of call stack inside the engine, not the store.',
+    'Some engine paths recurse once per row or per input chunk, and enough',
+    "of them exceed the runtime's default WebAssembly frame budget.",
+    remedy
+  ]
+}
+
+/** The remedies for each command that can run out of frames. */
+export const STACK_REMEDY = {
+  query: 'Raise it with node --stack-size=4000, add a LIMIT, or use Deno.',
+  pack: 'Raise it with node --stack-size=8000, or ' +
+    'deno run --v8-flags=--stack-size=8000.'
+}
+
+/**
+ * Open a store and return its manifest bytes.
+ *
+ * With no `generationName` the activated generation is opened through
+ * CURRENT. With one, that generation is opened directly, which is how a
+ * generation that has not been activated is inspected.
+ *
+ * @param {string} root the collection root that holds CURRENT
+ * @param {string|null} generationName
+ * @returns {{root: string, generation: string, generationDir: string,
+ *            manifestName: string, manifest: Uint8Array,
+ *            manifestHex: string, activated: boolean}}
+ */
+export function openStore (root, generationName = null) {
+  if (typeof generationName !== 'string') {
+    const opened = openCollection(root)
+    return { ...opened, manifestHex: hexOfBytes(opened.manifest), activated: true }
+  }
+  const generationDir = joinPath(root, generationName)
+  for (const manifestName of MANIFEST_NAMES) {
+    let manifest
+    try {
+      manifest = readWhole(joinPath(generationDir, manifestName))
+    } catch (_error) {
+      continue
+    }
+    return {
+      root,
+      generation: generationName,
+      generationDir,
+      manifestName,
+      manifest,
+      manifestHex: hexOfBytes(manifest),
+      activated: false
+    }
+  }
+  throw new StoreOperationError(
+    `${generationDir} has none of ${MANIFEST_NAMES.join(', ')}`)
+}
+
+/** `storeManifestInspect` — decode one manifest. */
+export function inspectManifest (engine, store) {
+  try {
+    return engine.call('storeManifestInspect', [store.manifestHex])
+  } catch (error) {
+    throw asStoreError(error)
+  }
+}
+
+/** `storeQueryPlan` — the artifact keys this query needs, and the mode. */
+export function planQuery (engine, store, sparql) {
+  try {
+    return engine.call('storeQueryPlan', [store.manifestHex, sparql])
+  } catch (error) {
+    throw asStoreError(error)
+  }
+}
+
+/**
+ * Ask the engine to decide the caps before any file is read.
+ *
+ * `storeQuery` checks its three caps against the manifest's own
+ * declarations BEFORE it looks at the artifact descriptors, so a call
+ * carrying an empty descriptor list gets the cap decision without moving
+ * a byte. The caps themselves stay where they are defined
+ * (`Wasm/Ops/Store.lean`); this host holds none of their values.
+ *
+ * @returns the envelope when the plan needs no artifact at all, else null
+ * @throws {StoreOperationError} when a cap refused the plan
+ */
+function capDecision (engine, store, sparql) {
+  try {
+    return engine.callBlob('storeQuery',
+      [store.manifestHex, sparql, '[]'], new Uint8Array(0))
+  } catch (error) {
+    const refusal = asStoreError(error)
+    if (refusal.message.indexOf('no bytes were supplied for artifact') >= 0) {
+      return null
+    }
+    throw refusal
+  }
+}
+
+/**
+ * Evaluate one SPARQL query against one generation.
+ *
+ * The sequence is: plan, cap decision, read exactly the artifacts the
+ * plan named, concatenate them into one buffer, and call `storeQuery`
+ * with a `{"key","offset","len"}` window per artifact. The buffer is
+ * written straight into the wasm heap by `engine.callBlob` with no
+ * encoding, and the engine bounds-checks every window.
+ *
+ * @returns {{plan: object, result: object, blobBytes: number,
+ *            artifacts: {key: string, offset: number, len: number}[]}}
+ */
+export function queryStore (engine, store, sparql) {
+  const plan = planQuery(engine, store, sparql)
+  const empty = capDecision(engine, store, sparql)
+  if (empty !== null) {
+    return { plan, result: empty, blobBytes: 0, artifacts: [] }
+  }
+  const chunks = plan.keys.map((key) => readWhole(joinPath(store.generationDir, key)))
+  let total = 0
+  for (const chunk of chunks) total += chunk.length
+  const blob = new Uint8Array(total)
+  const artifacts = []
+  let offset = 0
+  for (let index = 0; index < chunks.length; index += 1) {
+    blob.set(chunks[index], offset)
+    artifacts.push({ key: plan.keys[index], offset, len: chunks[index].length })
+    offset += chunks[index].length
+  }
+  let result
+  try {
+    result = engine.callBlob('storeQuery',
+      [store.manifestHex, sparql, JSON.stringify(artifacts)], blob)
+  } catch (error) {
+    throw asStoreError(error)
+  }
+  return { plan, result, blobBytes: total, artifacts }
+}
+
+/**
+ * `serializeTurtle` — the engine's own Turtle writer, over the N-Quads a
+ * CONSTRUCT answered. Named graphs are flattened into the default graph
+ * on this path; `--format nquads` is the fidelity-preserving one.
+ */
+export function turtleOfNQuads (engine, nquads) {
+  try {
+    return engine.call('serializeTurtle', [nquads]).turtle
+  } catch (error) {
+    throw asStoreError(error)
+  }
+}

package/l4-assets/l4factoidal.js

@@ -37,7 +37,7 @@
 // bytes change.
 
 // Stamped by formal/lean4/Wasm/build-wasm.sh step 9 -- do not hand-edit.
-const WASM_VERSION = "91fb323ec932";
+const WASM_VERSION = "125d391e0ccc";
 
 import createModule from './l4factoidal.mjs';
 
@@ -72,8 +72,6 @@ export function loadL4() {
     const Module = await createModule(moduleArg);
 
     const cVersion = Module.cwrap('l4_version_c', 'number', []);
-    const cBgpQuery = Module.cwrap('l4_bgp_query_c', 'number', ['string', 'string']);
-    const cCall = Module.cwrap('l4_call_c', 'number', ['string', 'string']);
     const cFree = Module.cwrap('l4_free_result', null, ['number']);
     const cInit = Module.cwrap('l4_init', 'number', []);
 
@@ -89,6 +87,27 @@ export function loadL4() {
 
     const asJson = (v) => (typeof v === 'string' ? v : JSON.stringify(v));
 
+    // Emscripten's cwrap `string` converter uses the WebAssembly stack.
+    // Multi-megabyte RDF/block requests can therefore overflow STACK_SIZE
+    // before Lean sees them. Allocate input UTF-8 on the wasm heap instead;
+    // the C shim copies each input into a Lean String synchronously, so these
+    // buffers can be released as soon as the exported call returns.
+    const callWithHeapStrings = (fn, texts) => {
+      const pointers = [];
+      try {
+        for (const text of texts) {
+          const size = Module.lengthBytesUTF8(text) + 1;
+          const ptr = Module._malloc(size);
+          if (!ptr) throw new Error('l4factoidal: could not allocate a WASM input buffer');
+          Module.stringToUTF8(text, ptr, size);
+          pointers.push(ptr);
+        }
+        return fn(...pointers);
+      } finally {
+        for (let i = pointers.length - 1; i >= 0; i--) Module._free(pointers[i]);
+      }
+    };
+
     return {
       /** The Lean-side ABI version string. */
       version() { return take(cVersion()); },
@@ -105,7 +124,9 @@ export function loadL4() {
        * @throws      if the Lean side reports a decoding error
        */
       bgpQuery(data, bgp) {
-        const parsed = JSON.parse(take(cBgpQuery(asJson(data), asJson(bgp))));
+        const resultPtr = callWithHeapStrings(Module._l4_bgp_query_c,
+          [asJson(data), asJson(bgp)]);
+        const parsed = JSON.parse(take(resultPtr));
         if (parsed.error) throw new Error(`l4factoidal: ${parsed.error}`);
         return parsed;
       },
@@ -122,11 +143,110 @@ export function loadL4() {
        * @throws      if the Lean side reports {"ok":false,"error":...}
        */
       call(op, args) {
-        const parsed = JSON.parse(take(cCall(op, JSON.stringify(args))));
+        const resultPtr = callWithHeapStrings(Module._l4_call_c,
+          [op, JSON.stringify(args)]);
+        const parsed = JSON.parse(take(resultPtr));
         if (parsed.ok === false) throw new Error(`l4factoidal: ${parsed.error}`);
         return parsed;
       },
 
+      /**
+       * The dispatch ABI, plus ONE contiguous byte region.
+       *
+       * For ops whose input is block bytes rather than text
+       * (`storeQuery`; the `ops` reflection lists them under
+       * `blobOps`). The bytes are written straight into the wasm heap
+       * with no encoding — no hex, no base64 — and copied once into a
+       * Lean ByteArray on the Lean side. Which bytes belong to which
+       * artifact is said in `args`, as {"key","offset","len"} windows
+       * into the region; Lean bounds-checks every one of them, so this
+       * call cannot pass a stale or out-of-range pointer.
+       *
+       * @param op    the method name, e.g. "storeQuery"
+       * @param args  array of positional STRING arguments
+       * @param blob  Uint8Array (or ArrayBuffer) of the concatenated bytes
+       * @returns     the parsed {"ok":true,...} envelope
+       * @throws      if the Lean side reports {"ok":false,"error":...}
+       */
+      callBlob(op, args, blob) {
+        const bytes = blob instanceof Uint8Array ? blob : new Uint8Array(blob ?? 0);
+        const blobPtr = bytes.length > 0 ? Module._malloc(bytes.length) : 0;
+        if (bytes.length > 0 && !blobPtr) {
+          throw new Error('l4factoidal: could not allocate a WASM blob buffer');
+        }
+        try {
+          if (bytes.length > 0) Module.HEAPU8.set(bytes, blobPtr);
+          const resultPtr = callWithHeapStrings(
+            (opPtr, argsPtr) => Module._l4_call_blob_c(opPtr, argsPtr, blobPtr, bytes.length),
+            [op, JSON.stringify(args)]);
+          const parsed = JSON.parse(take(resultPtr));
+          if (parsed.ok === false) throw new Error(`l4factoidal: ${parsed.error}`);
+          return parsed;
+        } finally {
+          if (blobPtr) Module._free(blobPtr);
+        }
+      },
+
+      /**
+       * The dispatch ABI, plus ONE byte region IN and ONE byte region
+       * OUT.
+       *
+       * For the ops of `L4Wasm.blobIoOpNames` (the `ops` envelope lists
+       * them under `blobIoOps`), whose RESULT is bytes rather than
+       * text. The bytes leave the module raw — no hex, no base64 — and
+       * are copied out of the wasm heap into a fresh Uint8Array before
+       * the module's buffer is released. The copy is required: the heap
+       * is detached and replaced when the module grows, so a subarray
+       * view of it can go stale between calls.
+       *
+       * Every other op answers as `call` does, with an empty region.
+       *
+       * @param op     the method name, e.g. "blobEcho"
+       * @param args   array of positional STRING arguments
+       * @param blobIn Uint8Array (or ArrayBuffer) carried IN; may be omitted
+       * @returns      { envelope, bytes } — the parsed {"ok":true,...}
+       *               envelope and a Uint8Array of the out region
+       * @throws       if the Lean side reports {"ok":false,"error":...}
+       */
+      callBlobIO(op, args, blobIn) {
+        const bytes = blobIn instanceof Uint8Array
+          ? blobIn
+          : new Uint8Array(blobIn ?? 0);
+        // Two 32-bit out parameters, uint8_t **out_ptr and size_t
+        // *out_len, in one 8-byte cell.
+        const outCell = Module._malloc(8);
+        if (!outCell) throw new Error('l4factoidal: could not allocate the out-parameter cell');
+        const blobPtr = bytes.length > 0 ? Module._malloc(bytes.length) : 0;
+        if (bytes.length > 0 && !blobPtr) {
+          Module._free(outCell);
+          throw new Error('l4factoidal: could not allocate a WASM blob buffer');
+        }
+        let outPtr = 0;
+        try {
+          Module.setValue(outCell, 0, 'i32');
+          Module.setValue(outCell + 4, 0, 'i32');
+          if (bytes.length > 0) Module.HEAPU8.set(bytes, blobPtr);
+          const resultPtr = callWithHeapStrings(
+            (opPtr, argsPtr) => Module._l4_call_blob_io_c(
+              opPtr, argsPtr, blobPtr, bytes.length, outCell, outCell + 4),
+            [op, JSON.stringify(args)]);
+          outPtr = Module.getValue(outCell, 'i32') >>> 0;
+          const outLen = Module.getValue(outCell + 4, 'i32') >>> 0;
+          const envelope = JSON.parse(take(resultPtr));
+          if (envelope.ok === false) throw new Error(`l4factoidal: ${envelope.error}`);
+          // slice() copies; HEAPU8 is replaced wholesale when the
+          // module's memory grows, so a view would not survive.
+          const region = outPtr !== 0 && outLen > 0
+            ? Module.HEAPU8.slice(outPtr, outPtr + outLen)
+            : new Uint8Array(0);
+          return { envelope, bytes: region };
+        } finally {
+          if (outPtr) Module._l4_free_blob(outPtr);
+          if (blobPtr) Module._free(blobPtr);
+          Module._free(outCell);
+        }
+      },
+
       /** Escape hatch for tests: the raw Emscripten module. */
       _module: Module,
     };