@factoidal/core 0.5.1 → 0.6.0

package/bin/pack-host.mjs

@@ -46,7 +46,7 @@ import { StoreHostError, listGeneration, readWhole } from '../store-host/index.m
 import { fileUrlToPath, joinPath } from '../store-host/paths.mjs'
 import { loadEngine } from './engine.mjs'
 import { PackError, packFile, packSupported, verifyGeneration } from './pack.mjs'
-import { openStore } from './store.mjs'
+import { StoreOperationError, openStore } from './store.mjs'
 
 const isDeno = typeof globalThis.Deno !== 'undefined'
 
@@ -144,6 +144,16 @@ export function describeError (error) {
       path: error.path === undefined ? null : error.path
     }
   }
+  if (error instanceof StoreOperationError) {
+    return {
+      type: 'StoreOperationError',
+      message: error.message,
+      capValue: error.capValue,
+      capLimit: error.capLimit,
+      digestKey: error.digestKey,
+      stackLimit: error.stackLimit
+    }
+  }
   return {
     type: 'Error',
     message: error && error.message ? String(error.message) : String(error)
@@ -160,6 +170,14 @@ export function reviveError (description) {
     const detail = description.path === null ? {} : { path: description.path }
     return new StoreHostError(description.code, description.message, detail)
   }
+  if (description.type === 'StoreOperationError') {
+    return new StoreOperationError(description.message, {
+      capValue: description.capValue === null ? undefined : description.capValue,
+      capLimit: description.capLimit === null ? undefined : description.capLimit,
+      digestKey: description.digestKey === null ? undefined : description.digestKey,
+      stackLimit: description.stackLimit
+    })
+  }
   return new Error(description.message)
 }
 

package/bin/store-worker-host.mjs

@@ -0,0 +1,397 @@
+// Giving the store operations a call stack big enough to finish, without
+// asking the reader for a runtime flag.
+// https://github.com/danbri/factoidal/issues/653
+//
+// THE DEFECT THIS FILE EXISTS FOR
+// Several engine paths recurse once per manifest entry or once per row.
+// On a large generation that exceeds the default call stack of Node and
+// of Deno, and the failure is `Maximum call stack size exceeded` rather
+// than an engine refusal. MEASURED 2026-09-05 against the committed
+// module, on a 7,315,251-quad collection of 3,286 blocks in 204 graphs
+// (`/Users/danbri/working/factoidal-skosfull`): `storeQueryPlan` alone
+// overflows on plain `node`, before a single artifact byte is read, and
+// `node --stack-size=60000` clears the plan, the open and every query.
+// Earlier the same limit was narrowed to any query that materialises
+// about 14,576 rows or more, whatever it returns.
+//
+// `factoidal pack` had the same defect and it was solved host-side, on a
+// worker thread with a raised stack (`bin/pack-host.mjs`,
+// https://github.com/danbri/factoidal/issues/649). This file carries that
+// route to the store operations.
+//
+// WHY THE HANDLE ITSELF LIVES IN THE WORKER
+// A handle is STATE INSIDE THE WASM INSTANCE -- the verified, decoded,
+// indexed blocks that `storeOpen` retained. A WebAssembly instance does
+// not cross a thread boundary, so a handle opened on the main thread
+// cannot be queried from a worker, and the reverse. Only two designs are
+// available:
+//
+//   (a) the handle lives in the worker, and `query()` and `close()` are
+//       messages to it;
+//   (b) each query spawns a worker, which re-reads and re-decodes every
+//       artifact.
+//
+// (b) destroys the reason a handle exists: `storeOpen` is the expensive
+// call (5.4 s against the collection above, where a query is 0.6 s), and
+// paying it per query is worse than the stateless `queryStore` path. So
+// (a) is what this file implements.
+//
+// THE COST OF (a), STATED
+//   - One worker thread per SESSION, not per handle. A session holds one
+//     engine and as many handles as the engine's own handle cap allows,
+//     so a caller that opens several stores pays for one thread and one
+//     copy of the module. `openStoreHandleOnWorker` uses a shared session
+//     by default; `{ownWorker: true}` gives a handle its own thread, and
+//     its own copy of the 5.6 MB module, when a caller wants failure
+//     isolation between stores.
+//   - Every call is ASYNCHRONOUS, where the in-process handle is
+//     synchronous. A message round trip is the price of the bigger stack.
+//   - An extension function registered on the MAIN thread's engine
+//     (`bin/ext.mjs`) is not visible to the worker's engine. Register it
+//     inside the worker, or use the in-process handle.
+//   - The worker keeps the process alive until `close()` or
+//     `terminate()`. `unref()` is not used: a server that dropped its
+//     last reference would otherwise lose its store silently.
+//
+// WHAT A ONE-SHOT QUERY DOES INSTEAD
+// A single `factoidal query` builds no handle and drops whatever it
+// loads, so a worker is measured overhead it cannot amortise. It runs
+// IN PROCESS, and only when the runtime runs out of frames does it retry
+// once on a worker (`queryStoreOnWorker`). The normal case pays nothing.
+
+import { loadEngine } from './engine.mjs'
+import {
+  WORKER_STACK_MB, isStackOverflow, reviveError, workerRefused, workerStackMb
+} from './pack-host.mjs'
+import {
+  StoreOperationError, openStore, openStoreHandle, planQuery, queryStore
+} from './store.mjs'
+
+const isDeno = typeof globalThis.Deno !== 'undefined'
+
+export { WORKER_STACK_MB, isStackOverflow, workerRefused, workerStackMb }
+
+/**
+ * Whether this runtime has the `worker_threads` route at all.
+ *
+ * Deno has a `node:worker_threads` shim, and it accepts
+ * `resourceLimits.stackSizeMb` and does almost nothing with it. MEASURED
+ * 2026-09-05, deno 2.9.4 / V8 15.0.245.2 against Node 22.22.2, by
+ * counting frames to the overflow inside the worker:
+ *
+ *   runtime  default   stackSizeMb 64
+ *   node     41,195    696,555
+ *   deno     10,835     13,837
+ *
+ * A 1.28-times raise does not carry a store open that needs sixteen
+ * times the default, so Deno takes the re-exec route in
+ * `bin/pack-host.mjs` instead: the command runs itself again once with
+ * `--v8-flags=--stack-size=...`, which needs --allow-run and --allow-env.
+ * A Deno LIBRARY caller cannot re-execute its own host, so
+ * `openStoreHandleOnWorker` gives it an in-process handle behind the same
+ * asynchronous interface, and the process must be started with
+ *   deno run --allow-read --v8-flags=--stack-size=65536
+ */
+export function workerRouteAvailable () {
+  return !isDeno
+}
+
+// ------------------------------------------------------------- a session
+//
+// One worker thread, one engine, many handles.
+
+let nextRequestId = 1
+
+/**
+ * A worker thread holding one engine and the handles opened through it.
+ *
+ * Hold one of these for as long as a process answers questions about a
+ * store. `terminate()` drops the thread and everything in it.
+ */
+export class StoreWorkerSession {
+  constructor (worker) {
+    this.worker = worker
+    this.pending = new Map()
+    this.dead = null
+    this.handles = new Set()
+    worker.on('message', (message) => {
+      const waiting = this.pending.get(message.id)
+      if (waiting === undefined) return
+      this.pending.delete(message.id)
+      if (message.kind === 'error') waiting.reject(reviveError(message.error))
+      else waiting.resolve(message.answer)
+    })
+    worker.on('error', (error) => { this.fail(error) })
+    worker.on('exit', (code) => {
+      this.fail(new StoreOperationError(
+        `the store worker exited with code ${code}`))
+    })
+  }
+
+  /** Fail every outstanding request and refuse every later one. */
+  fail (error) {
+    if (this.dead === null) this.dead = error
+    for (const waiting of this.pending.values()) waiting.reject(error)
+    this.pending.clear()
+  }
+
+  /** Send one request and await its answer. */
+  send (request) {
+    if (this.dead !== null) return Promise.reject(this.dead)
+    const id = nextRequestId
+    nextRequestId += 1
+    return new Promise((resolve, reject) => {
+      this.pending.set(id, { resolve, reject })
+      this.worker.postMessage({ ...request, id })
+    })
+  }
+
+  /**
+   * Open one store handle inside this session's worker.
+   *
+   * @param {string} root the collection root that holds CURRENT
+   * @param {{generation?: string, keys?: string[], sparql?: string}} options
+   *   the same options `openStoreHandle` takes, plus `generation` to open
+   *   a generation that has not been activated
+   * @returns {Promise<WorkerStoreHandle>}
+   */
+  async open (root, options = {}) {
+    const envelope = await this.send({
+      kind: 'open',
+      root,
+      generation: typeof options.generation === 'string' ? options.generation : null,
+      keys: Array.isArray(options.keys) ? options.keys : null,
+      sparql: typeof options.sparql === 'string' ? options.sparql : null
+    })
+    const handle = new WorkerStoreHandle(this, envelope)
+    this.handles.add(handle)
+    return handle
+  }
+
+  /** `storeHandleList` from inside the worker: what this session holds. */
+  async list () {
+    return (await this.send({ kind: 'list' })).list
+  }
+
+  /** Stop the thread. Every handle it held is gone with it. */
+  async terminate () {
+    if (this.dead === null) this.dead = new StoreOperationError(
+      'this store worker session is terminated')
+    for (const handle of this.handles) handle.closed = true
+    this.handles.clear()
+    await this.worker.terminate()
+  }
+}
+
+/**
+ * A store handle held inside a worker thread.
+ *
+ * The same envelope fields as the in-process `StoreHandle`, and the same
+ * two methods, except that both are asynchronous. The wasm module is
+ * single-threaded, so two `query()` calls on one session are served in
+ * the order they were sent.
+ */
+export class WorkerStoreHandle {
+  constructor (session, envelope) {
+    this.session = session
+    this.handle = envelope.handle
+    this.identity = envelope.identity
+    this.layout = envelope.layout
+    this.wireVersion = envelope.wireVersion
+    this.artifacts = envelope.artifacts
+    this.bytes = envelope.bytes
+    this.rows = envelope.rows
+    this.closed = false
+    // Set when this handle owns its session, so `close()` stops the thread.
+    this.ownSession = false
+  }
+
+  /** One SPARQL query against the retained blocks. The envelope is the
+   *  one the in-process handle answers with. */
+  async query (sparql) {
+    if (this.closed) {
+      throw new StoreOperationError(`store handle ${this.handle} is closed`)
+    }
+    return (await this.session.send({
+      kind: 'query', handle: this.handle, sparql
+    })).result
+  }
+
+  /** Drop the handle and everything it retained. Idempotent. */
+  async close () {
+    if (this.closed) return
+    this.closed = true
+    this.session.handles.delete(this)
+    if (this.ownSession) {
+      await this.session.terminate()
+      return
+    }
+    if (this.session.dead !== null) return
+    await this.session.send({ kind: 'close', handle: this.handle })
+  }
+}
+
+// ------------------------------------------------------- starting one
+
+/** Start a worker thread with a stack big enough for the store paths. */
+async function startWorker () {
+  if (!workerRouteAvailable()) {
+    throw new StoreOperationError(
+      'this runtime has no worker_threads route for the store operations; ' +
+      'raise the call stack itself instead')
+  }
+  const { Worker } = await import('node:worker_threads')
+  const entry = new URL('./store-worker.mjs', import.meta.url)
+  const worker = new Worker(entry, {
+    resourceLimits: { stackSizeMb: workerStackMb() }
+  })
+  const session = new StoreWorkerSession(worker)
+  // Load the engine now, so the first query measures the query.
+  await session.send({ kind: 'warm' })
+  return session
+}
+
+/** Start a session of this session's own. */
+export async function openStoreWorkerSession () {
+  return await startWorker()
+}
+
+let sharedSession = null
+
+/** The session `openStoreHandleOnWorker` uses when the caller names none:
+ *  one thread, one engine, shared by every handle in the process. */
+export async function sharedStoreWorkerSession () {
+  if (sharedSession === null) sharedSession = startWorker()
+  try {
+    return await sharedSession
+  } catch (error) {
+    sharedSession = null
+    throw error
+  }
+}
+
+/** Stop the shared session, if one was started. */
+export async function closeSharedStoreWorkerSession () {
+  if (sharedSession === null) return
+  const session = sharedSession
+  sharedSession = null
+  await (await session).terminate()
+}
+
+/**
+ * Open one store handle on a worker thread with a raised call stack.
+ *
+ * This is the handle a long-lived process wants against a large store: an
+ * MCP server, a chat tool loop, a SPARQL endpoint. It is asynchronous
+ * where `openStoreHandle` is synchronous, and it needs no runtime flag.
+ *
+ * @param {string} root the collection root that holds CURRENT
+ * @param {{generation?: string, keys?: string[], sparql?: string,
+ *          session?: StoreWorkerSession, ownWorker?: boolean}} options
+ * @returns {Promise<WorkerStoreHandle>}
+ */
+export async function openStoreHandleOnWorker (root, options = {}) {
+  if (!workerRouteAvailable() || workerRefused(options)) {
+    return await openHere(root, options)
+  }
+  if (options.session instanceof StoreWorkerSession) {
+    return await options.session.open(root, options)
+  }
+  if (options.ownWorker === true) {
+    const session = await startWorker()
+    const handle = await session.open(root, options)
+    handle.ownSession = true
+    return handle
+  }
+  const session = await sharedStoreWorkerSession()
+  return await session.open(root, options)
+}
+
+// -------------------------------- the in-process facade, for Deno
+//
+// Same two methods, same envelopes, no thread. It exists so one caller
+// works on both runtimes; on Deno the stack must come from the host
+// (`--v8-flags=--stack-size=65536`), because Deno's worker_threads shim
+// does not raise it (see `workerRouteAvailable`).
+
+/** A handle held in this process, behind the asynchronous interface. */
+export class LocalStoreHandle {
+  constructor (handle) {
+    this.local = handle
+    this.handle = handle.handle
+    this.identity = handle.identity
+    this.layout = handle.layout
+    this.wireVersion = handle.wireVersion
+    this.artifacts = handle.artifacts
+    this.bytes = handle.bytes
+    this.rows = handle.rows
+  }
+
+  get closed () { return this.local.closed }
+
+  async query (sparql) { return this.local.query(sparql) }
+
+  async close () { this.local.close() }
+}
+
+/** Open a handle in this process, behind the asynchronous interface. */
+async function openHere (root, options) {
+  const engine = await loadEngine()
+  const store = openStore(root,
+    typeof options.generation === 'string' ? options.generation : null)
+  const pick = {}
+  if (Array.isArray(options.keys)) pick.keys = options.keys
+  else if (typeof options.sparql === 'string') pick.sparql = options.sparql
+  return new LocalStoreHandle(openStoreHandle(engine, store, pick))
+}
+
+// ------------------------------------------------- the one-shot retry
+
+/**
+ * Run one stateless query on a worker thread, then stop the thread.
+ *
+ * What `factoidal query` retries with after the in-process attempt ran
+ * out of frames. It answers exactly what `queryStore` answers.
+ *
+ * @returns {Promise<{plan: object, result: object, blobBytes: number}>}
+ */
+export async function queryStoreOnWorker (root, generation, sparql) {
+  if (!workerRouteAvailable()) {
+    const engine = await loadEngine()
+    const store = openStore(root, generation ?? null)
+    const answer = queryStore(engine, store, sparql)
+    return { plan: answer.plan, result: answer.result, blobBytes: answer.blobBytes }
+  }
+  const session = await startWorker()
+  try {
+    return await session.send({
+      kind: 'query-once', root, generation: generation ?? null, sparql
+    })
+  } finally {
+    await session.terminate()
+  }
+}
+
+/**
+ * Plan one query on a worker thread, then stop the thread.
+ *
+ * `storeQueryPlan` walks the manifest, and on a collection of a few
+ * thousand blocks that walk alone exceeds the default stack. What
+ * `factoidal query --explain` retries with.
+ *
+ * @returns {Promise<object>} the plan envelope
+ */
+export async function planQueryOnWorker (root, generation, sparql) {
+  if (!workerRouteAvailable()) {
+    const engine = await loadEngine()
+    return planQuery(engine, openStore(root, generation ?? null), sparql)
+  }
+  const session = await startWorker()
+  try {
+    return (await session.send({
+      kind: 'plan', root, generation: generation ?? null, sparql
+    })).plan
+  } finally {
+    await session.terminate()
+  }
+}

package/bin/store-worker.mjs

@@ -0,0 +1,100 @@
+// The thread the store operations run on under Node, so that a query
+// against a large generation has a call stack big enough to finish.
+// https://github.com/danbri/factoidal/issues/653
+//
+// WHAT THIS FILE IS ALLOWED TO DO
+// Load the engine, open a store, hold store handles, run the same store
+// operations the in-process path runs, and post the answers back. It
+// reads no manifest field, verifies no digest, decodes no block and
+// chooses no artifact: `bin/store.mjs` drives the engine here exactly as
+// it does on the main thread (iron rule 7 of CLAUDE.md).
+//
+// THE ENGINE AND THE HANDLES LIVE HERE, NOT ON THE MAIN THREAD.
+// A loaded WebAssembly instance does not cross a thread boundary, and a
+// store handle is state inside that instance: the verified, decoded,
+// indexed blocks. So a handle opened here can only be queried from here.
+// That is why the protocol below carries `query` and `close` as messages
+// rather than returning a handle object. One worker holds many handles,
+// up to the engine's own handle cap.
+
+import { parentPort } from 'node:worker_threads'
+import { loadEngine } from './engine.mjs'
+import { describeError } from './pack-host.mjs'
+import {
+  listStoreHandles, openStore, openStoreHandle, planQuery, queryStore
+} from './store.mjs'
+
+const port = parentPort
+
+/** The handles this worker holds, by the id the engine gave them. */
+const handles = new Map()
+
+let engine = null
+async function theEngine () {
+  if (engine === null) engine = await loadEngine()
+  return engine
+}
+
+/** One request. Every branch answers with a plain object, so nothing
+ *  that crosses the boundary is an engine reference. */
+async function serve (request) {
+  const active = await theEngine()
+  if (request.kind === 'open') {
+    const store = openStore(request.root, request.generation ?? null)
+    const options = {}
+    if (Array.isArray(request.keys)) options.keys = request.keys
+    else if (typeof request.sparql === 'string') options.sparql = request.sparql
+    const handle = openStoreHandle(active, store, options)
+    handles.set(handle.handle, handle)
+    return {
+      handle: handle.handle,
+      identity: handle.identity,
+      layout: handle.layout,
+      wireVersion: handle.wireVersion,
+      artifacts: handle.artifacts,
+      bytes: handle.bytes,
+      rows: handle.rows
+    }
+  }
+  if (request.kind === 'query') {
+    const handle = handles.get(request.handle)
+    if (handle === undefined) {
+      throw new Error(`this worker holds no store handle ${request.handle}`)
+    }
+    return { result: handle.query(request.sparql) }
+  }
+  if (request.kind === 'close') {
+    const handle = handles.get(request.handle)
+    if (handle === undefined) return { closed: false }
+    handle.close()
+    handles.delete(request.handle)
+    return { closed: true }
+  }
+  if (request.kind === 'list') {
+    return { list: listStoreHandles(active) }
+  }
+  if (request.kind === 'plan') {
+    const store = openStore(request.root, request.generation ?? null)
+    return { plan: planQuery(active, store, request.sparql) }
+  }
+  if (request.kind === 'query-once') {
+    // The stateless path: read the artifacts the plan names, hand them
+    // over, drop them. Used by the one-shot `factoidal query` after an
+    // in-process attempt ran out of frames.
+    const store = openStore(request.root, request.generation ?? null)
+    const answer = queryStore(active, store, request.sparql)
+    return {
+      plan: answer.plan,
+      result: answer.result,
+      blobBytes: answer.blobBytes
+    }
+  }
+  if (request.kind === 'warm') return { warm: true }
+  throw new Error(`the store worker was sent an unknown request "${request.kind}"`)
+}
+
+port.on('message', (request) => {
+  serve(request).then(
+    (answer) => port.postMessage({ id: request.id, kind: 'ok', answer }),
+    (error) => port.postMessage({ id: request.id, kind: 'error', error: describeError(error) }))
+})

package/bin/store.mjs

@@ -97,7 +97,10 @@ export function stackLimitAdvice (remedy) {
 
 /** The remedies for each command that can run out of frames. */
 export const STACK_REMEDY = {
-  query: 'Raise it with node --stack-size=4000, add a LIMIT, or use Deno.',
+  query: 'A worker thread with a raised stack normally runs this again ' +
+    '(bin/store-worker-host.mjs). It was refused or is unavailable here: ' +
+    'drop --no-worker, give Deno --allow-run and --allow-env, raise the ' +
+    'stack with node --stack-size=60000, or add a LIMIT.',
   pack: 'Raise it with node --stack-size=8000, or ' +
     'deno run --v8-flags=--stack-size=8000.'
 }
@@ -174,8 +177,12 @@ export function planQuery (engine, store, sparql) {
  */
 function capDecision (engine, store, sparql) {
   try {
-    return engine.callBlob('storeQuery',
-      [store.manifestHex, sparql, '[]'], new Uint8Array(0))
+    // callBlobIO, not callBlob: the blob-IO entry is the one that reaches
+    // the SPARQL 1.1 section 17.6 extension registry (Wasm/Dispatch.lean's
+    // callBlobIO), so a caller registration is in scope on the stateless
+    // store path too. The envelope is identical.
+    return engine.callBlobIO('storeQuery',
+      [store.manifestHex, sparql, '[]'], new Uint8Array(0)).envelope
   } catch (error) {
     const refusal = asStoreError(error)
     if (refusal.message.indexOf('no bytes were supplied for artifact') >= 0) {
@@ -191,7 +198,7 @@ function capDecision (engine, store, sparql) {
  * The sequence is: plan, cap decision, read exactly the artifacts the
  * plan named, concatenate them into one buffer, and call `storeQuery`
  * with a `{"key","offset","len"}` window per artifact. The buffer is
- * written straight into the wasm heap by `engine.callBlob` with no
+ * written straight into the wasm heap by `engine.callBlobIO` with no
  * encoding, and the engine bounds-checks every window.
  *
  * @returns {{plan: object, result: object, blobBytes: number,
@@ -216,8 +223,8 @@ export function queryStore (engine, store, sparql) {
   }
   let result
   try {
-    result = engine.callBlob('storeQuery',
-      [store.manifestHex, sparql, JSON.stringify(artifacts)], blob)
+    result = engine.callBlobIO('storeQuery',
+      [store.manifestHex, sparql, JSON.stringify(artifacts)], blob).envelope
   } catch (error) {
     throw asStoreError(error)
   }
@@ -258,9 +265,21 @@ export function turtleOfNQuads (engine, nquads) {
 // The stateless path above stays exactly as it was. A one-shot CLI query
 // should not pay to build a handle it will drop.
 
-/** The artifact keys a manifest declares, in manifest order. */
+/**
+ * The artifact keys a manifest declares, in manifest order: every block, and
+ * every index sidecar the entry names. The KEYS come from the engine; this
+ * function chooses none of them. A sidecar is what lets a literal search skip
+ * the scan (SBM8's LGI1) or a geometry FILTER skip the WKT parse (SBM9's
+ * GBI1); a generation that declares none is unaffected. New sidecar roles
+ * need no change here: the engine names them under `entry.sidecars`.
+ */
 function manifestKeys (engine, store) {
-  return inspectManifest(engine, store).entries.map((entry) => entry.key)
+  const keys = []
+  for (const entry of inspectManifest(engine, store).entries) {
+    keys.push(entry.key)
+    for (const key of Object.values(entry.sidecars ?? {})) keys.push(key)
+  }
+  return keys
 }
 
 /** Read the named artifacts and concatenate them into one region. */
@@ -346,7 +365,8 @@ export function openStoreHandle (engine, store, options = {}) {
   if (Array.isArray(options.keys)) {
     keys = options.keys
   } else if (typeof options.sparql === 'string') {
-    keys = planQuery(engine, store, options.sparql).keys
+    const plan = planQuery(engine, store, options.sparql)
+    keys = plan.keys.concat(plan.sidecarKeys ?? [])
   } else {
     keys = manifestKeys(engine, store)
   }

package/l4-assets/l4factoidal.js

@@ -37,7 +37,7 @@
 // bytes change.
 
 // Stamped by formal/lean4/Wasm/build-wasm.sh step 9 -- do not hand-edit.
-const WASM_VERSION = "ffbb3a035705";
+const WASM_VERSION = "0de098a8aad0";
 
 import createModule from './l4factoidal.mjs';