@factoidal/core 0.5.0 → 0.6.0

package/bin/factoidal.mjs

@@ -28,6 +28,9 @@ import { loadEngine } from './engine.mjs'
 import { sampleStoreFacts, sampleStorePath } from '../sample-store.mjs'
 import { PackError, packSupported, verifyGeneration } from './pack.mjs'
 import { denoReexec, isStackOverflow, runPack } from './pack-host.mjs'
+import {
+  planQueryOnWorker, queryStoreOnWorker, workerRouteAvailable
+} from './store-worker-host.mjs'
 import {
   STACK_REMEDY, StoreOperationError, inspectManifest, openStore, planQuery,
   queryStore, stackLimitAdvice, turtleOfNQuads
@@ -179,6 +182,21 @@ options:
   --generation NAME  query this generation instead of the activated one
   --json             shorthand for --format json
   --quiet            print the result only, no plan line on stderr
+  --no-worker        never retry on a worker thread (see below)
+
+Several engine paths recurse once per manifest entry or once per row, and
+against a large collection that exceeds the runtime's default call stack.
+A one-shot query builds no store handle, so it runs IN PROCESS and pays
+nothing for a worker it would drop; only when the runtime runs out of
+frames does it run again on a worker thread with a raised stack, or under
+Deno re-execute itself once with a raised V8 stack, which needs
+--allow-run and --allow-env (https://github.com/danbri/factoidal/issues/653).
+--no-worker turns the retry off, and the frame budget is then reported.
+
+A process that asks MANY questions about one store should hold a store
+handle instead: openStoreHandleOnWorker in bin/store-worker-host.mjs
+verifies and decodes once, on a thread with the raised stack, and answers
+every later query from what it retained.
 
 formats:
   table    a human display of the engine's SPARQL Query Results JSON;
@@ -532,6 +550,36 @@ function reportStoreFailure (error) {
   return EXIT_FAILURE
 }
 
+/**
+ * Run one store call again on a bigger call stack, after the in-process
+ * attempt ran out of frames.
+ *
+ * A one-shot query builds no handle and drops what it loads, so it runs
+ * IN PROCESS first and pays nothing for a worker it would throw away.
+ * This is the retry, and it is reached only by the overflow.
+ *
+ * Node takes the worker route; Deno has no `worker_threads` route with a
+ * stack size, so it re-executes this command once with a raised V8 stack
+ * (`denoReexec`), which needs --allow-run and --allow-env.
+ *
+ * @returns the retried answer, or a number to exit with (the Deno child's
+ *   code), or null when no bigger stack is available here
+ */
+async function onBiggerStack (options, quiet, retry) {
+  const host = { worker: options['no-worker'] !== true }
+  if (host.worker === false) return null
+  if (isDeno) {
+    const code = await denoReexec(host)
+    return typeof code === 'number' ? code : null
+  }
+  if (!workerRouteAvailable()) return null
+  if (!quiet) {
+    err('The runtime ran out of call stack; running again on a worker thread ' +
+      'with a bigger one.')
+  }
+  return await retry()
+}
+
 async function commandQuery (positional, options) {
   if (positional.length < 1) throw new UsageError('query needs a STORE')
   if (typeof options.base === 'string') {
@@ -554,8 +602,13 @@ async function commandQuery (positional, options) {
     try {
       plan = planQuery(engine, store, sparql)
     } catch (error) {
-      if (error instanceof StoreOperationError) return reportStoreFailure(error)
-      throw error
+      if (!(error instanceof StoreOperationError)) throw error
+      if (!error.stackLimit) return reportStoreFailure(error)
+      const retried = await onBiggerStack(options, quiet,
+        () => planQueryOnWorker(root, named, sparql))
+      if (retried === null) return reportStoreFailure(error)
+      if (typeof retried === 'number') return retried
+      plan = retried
     }
     if (format === 'json' || options.json === true) {
       out(JSON.stringify(plan, null, 2))
@@ -572,8 +625,13 @@ async function commandQuery (positional, options) {
   try {
     answer = queryStore(engine, store, sparql)
   } catch (error) {
-    if (error instanceof StoreOperationError) return reportStoreFailure(error)
-    throw error
+    if (!(error instanceof StoreOperationError)) throw error
+    if (!error.stackLimit) return reportStoreFailure(error)
+    const retried = await onBiggerStack(options, quiet,
+      () => queryStoreOnWorker(root, named, sparql))
+    if (retried === null) return reportStoreFailure(error)
+    if (typeof retried === 'number') return retried
+    answer = retried
   }
   const { plan, result, blobBytes } = answer
   if (!quiet) {

package/bin/pack-host.mjs

@@ -46,7 +46,7 @@ import { StoreHostError, listGeneration, readWhole } from '../store-host/index.m
 import { fileUrlToPath, joinPath } from '../store-host/paths.mjs'
 import { loadEngine } from './engine.mjs'
 import { PackError, packFile, packSupported, verifyGeneration } from './pack.mjs'
-import { openStore } from './store.mjs'
+import { StoreOperationError, openStore } from './store.mjs'
 
 const isDeno = typeof globalThis.Deno !== 'undefined'
 
@@ -144,6 +144,16 @@ export function describeError (error) {
       path: error.path === undefined ? null : error.path
     }
   }
+  if (error instanceof StoreOperationError) {
+    return {
+      type: 'StoreOperationError',
+      message: error.message,
+      capValue: error.capValue,
+      capLimit: error.capLimit,
+      digestKey: error.digestKey,
+      stackLimit: error.stackLimit
+    }
+  }
   return {
     type: 'Error',
     message: error && error.message ? String(error.message) : String(error)
@@ -160,6 +170,14 @@ export function reviveError (description) {
     const detail = description.path === null ? {} : { path: description.path }
     return new StoreHostError(description.code, description.message, detail)
   }
+  if (description.type === 'StoreOperationError') {
+    return new StoreOperationError(description.message, {
+      capValue: description.capValue === null ? undefined : description.capValue,
+      capLimit: description.capLimit === null ? undefined : description.capLimit,
+      digestKey: description.digestKey === null ? undefined : description.digestKey,
+      stackLimit: description.stackLimit
+    })
+  }
   return new Error(description.message)
 }
 

package/bin/store-worker-host.mjs

@@ -0,0 +1,397 @@
+// Giving the store operations a call stack big enough to finish, without
+// asking the reader for a runtime flag.
+// https://github.com/danbri/factoidal/issues/653
+//
+// THE DEFECT THIS FILE EXISTS FOR
+// Several engine paths recurse once per manifest entry or once per row.
+// On a large generation that exceeds the default call stack of Node and
+// of Deno, and the failure is `Maximum call stack size exceeded` rather
+// than an engine refusal. MEASURED 2026-09-05 against the committed
+// module, on a 7,315,251-quad collection of 3,286 blocks in 204 graphs
+// (`/Users/danbri/working/factoidal-skosfull`): `storeQueryPlan` alone
+// overflows on plain `node`, before a single artifact byte is read, and
+// `node --stack-size=60000` clears the plan, the open and every query.
+// Earlier the same limit was narrowed to any query that materialises
+// about 14,576 rows or more, whatever it returns.
+//
+// `factoidal pack` had the same defect and it was solved host-side, on a
+// worker thread with a raised stack (`bin/pack-host.mjs`,
+// https://github.com/danbri/factoidal/issues/649). This file carries that
+// route to the store operations.
+//
+// WHY THE HANDLE ITSELF LIVES IN THE WORKER
+// A handle is STATE INSIDE THE WASM INSTANCE -- the verified, decoded,
+// indexed blocks that `storeOpen` retained. A WebAssembly instance does
+// not cross a thread boundary, so a handle opened on the main thread
+// cannot be queried from a worker, and the reverse. Only two designs are
+// available:
+//
+//   (a) the handle lives in the worker, and `query()` and `close()` are
+//       messages to it;
+//   (b) each query spawns a worker, which re-reads and re-decodes every
+//       artifact.
+//
+// (b) destroys the reason a handle exists: `storeOpen` is the expensive
+// call (5.4 s against the collection above, where a query is 0.6 s), and
+// paying it per query is worse than the stateless `queryStore` path. So
+// (a) is what this file implements.
+//
+// THE COST OF (a), STATED
+//   - One worker thread per SESSION, not per handle. A session holds one
+//     engine and as many handles as the engine's own handle cap allows,
+//     so a caller that opens several stores pays for one thread and one
+//     copy of the module. `openStoreHandleOnWorker` uses a shared session
+//     by default; `{ownWorker: true}` gives a handle its own thread, and
+//     its own copy of the 5.6 MB module, when a caller wants failure
+//     isolation between stores.
+//   - Every call is ASYNCHRONOUS, where the in-process handle is
+//     synchronous. A message round trip is the price of the bigger stack.
+//   - An extension function registered on the MAIN thread's engine
+//     (`bin/ext.mjs`) is not visible to the worker's engine. Register it
+//     inside the worker, or use the in-process handle.
+//   - The worker keeps the process alive until `close()` or
+//     `terminate()`. `unref()` is not used: a server that dropped its
+//     last reference would otherwise lose its store silently.
+//
+// WHAT A ONE-SHOT QUERY DOES INSTEAD
+// A single `factoidal query` builds no handle and drops whatever it
+// loads, so a worker is measured overhead it cannot amortise. It runs
+// IN PROCESS, and only when the runtime runs out of frames does it retry
+// once on a worker (`queryStoreOnWorker`). The normal case pays nothing.
+
+import { loadEngine } from './engine.mjs'
+import {
+  WORKER_STACK_MB, isStackOverflow, reviveError, workerRefused, workerStackMb
+} from './pack-host.mjs'
+import {
+  StoreOperationError, openStore, openStoreHandle, planQuery, queryStore
+} from './store.mjs'
+
+const isDeno = typeof globalThis.Deno !== 'undefined'
+
+export { WORKER_STACK_MB, isStackOverflow, workerRefused, workerStackMb }
+
+/**
+ * Whether this runtime has the `worker_threads` route at all.
+ *
+ * Deno has a `node:worker_threads` shim, and it accepts
+ * `resourceLimits.stackSizeMb` and does almost nothing with it. MEASURED
+ * 2026-09-05, deno 2.9.4 / V8 15.0.245.2 against Node 22.22.2, by
+ * counting frames to the overflow inside the worker:
+ *
+ *   runtime  default   stackSizeMb 64
+ *   node     41,195    696,555
+ *   deno     10,835     13,837
+ *
+ * A 1.28-times raise does not carry a store open that needs sixteen
+ * times the default, so Deno takes the re-exec route in
+ * `bin/pack-host.mjs` instead: the command runs itself again once with
+ * `--v8-flags=--stack-size=...`, which needs --allow-run and --allow-env.
+ * A Deno LIBRARY caller cannot re-execute its own host, so
+ * `openStoreHandleOnWorker` gives it an in-process handle behind the same
+ * asynchronous interface, and the process must be started with
+ *   deno run --allow-read --v8-flags=--stack-size=65536
+ */
+export function workerRouteAvailable () {
+  return !isDeno
+}
+
+// ------------------------------------------------------------- a session
+//
+// One worker thread, one engine, many handles.
+
+let nextRequestId = 1
+
+/**
+ * A worker thread holding one engine and the handles opened through it.
+ *
+ * Hold one of these for as long as a process answers questions about a
+ * store. `terminate()` drops the thread and everything in it.
+ */
+export class StoreWorkerSession {
+  constructor (worker) {
+    this.worker = worker
+    this.pending = new Map()
+    this.dead = null
+    this.handles = new Set()
+    worker.on('message', (message) => {
+      const waiting = this.pending.get(message.id)
+      if (waiting === undefined) return
+      this.pending.delete(message.id)
+      if (message.kind === 'error') waiting.reject(reviveError(message.error))
+      else waiting.resolve(message.answer)
+    })
+    worker.on('error', (error) => { this.fail(error) })
+    worker.on('exit', (code) => {
+      this.fail(new StoreOperationError(
+        `the store worker exited with code ${code}`))
+    })
+  }
+
+  /** Fail every outstanding request and refuse every later one. */
+  fail (error) {
+    if (this.dead === null) this.dead = error
+    for (const waiting of this.pending.values()) waiting.reject(error)
+    this.pending.clear()
+  }
+
+  /** Send one request and await its answer. */
+  send (request) {
+    if (this.dead !== null) return Promise.reject(this.dead)
+    const id = nextRequestId
+    nextRequestId += 1
+    return new Promise((resolve, reject) => {
+      this.pending.set(id, { resolve, reject })
+      this.worker.postMessage({ ...request, id })
+    })
+  }
+
+  /**
+   * Open one store handle inside this session's worker.
+   *
+   * @param {string} root the collection root that holds CURRENT
+   * @param {{generation?: string, keys?: string[], sparql?: string}} options
+   *   the same options `openStoreHandle` takes, plus `generation` to open
+   *   a generation that has not been activated
+   * @returns {Promise<WorkerStoreHandle>}
+   */
+  async open (root, options = {}) {
+    const envelope = await this.send({
+      kind: 'open',
+      root,
+      generation: typeof options.generation === 'string' ? options.generation : null,
+      keys: Array.isArray(options.keys) ? options.keys : null,
+      sparql: typeof options.sparql === 'string' ? options.sparql : null
+    })
+    const handle = new WorkerStoreHandle(this, envelope)
+    this.handles.add(handle)
+    return handle
+  }
+
+  /** `storeHandleList` from inside the worker: what this session holds. */
+  async list () {
+    return (await this.send({ kind: 'list' })).list
+  }
+
+  /** Stop the thread. Every handle it held is gone with it. */
+  async terminate () {
+    if (this.dead === null) this.dead = new StoreOperationError(
+      'this store worker session is terminated')
+    for (const handle of this.handles) handle.closed = true
+    this.handles.clear()
+    await this.worker.terminate()
+  }
+}
+
+/**
+ * A store handle held inside a worker thread.
+ *
+ * The same envelope fields as the in-process `StoreHandle`, and the same
+ * two methods, except that both are asynchronous. The wasm module is
+ * single-threaded, so two `query()` calls on one session are served in
+ * the order they were sent.
+ */
+export class WorkerStoreHandle {
+  constructor (session, envelope) {
+    this.session = session
+    this.handle = envelope.handle
+    this.identity = envelope.identity
+    this.layout = envelope.layout
+    this.wireVersion = envelope.wireVersion
+    this.artifacts = envelope.artifacts
+    this.bytes = envelope.bytes
+    this.rows = envelope.rows
+    this.closed = false
+    // Set when this handle owns its session, so `close()` stops the thread.
+    this.ownSession = false
+  }
+
+  /** One SPARQL query against the retained blocks. The envelope is the
+   *  one the in-process handle answers with. */
+  async query (sparql) {
+    if (this.closed) {
+      throw new StoreOperationError(`store handle ${this.handle} is closed`)
+    }
+    return (await this.session.send({
+      kind: 'query', handle: this.handle, sparql
+    })).result
+  }
+
+  /** Drop the handle and everything it retained. Idempotent. */
+  async close () {
+    if (this.closed) return
+    this.closed = true
+    this.session.handles.delete(this)
+    if (this.ownSession) {
+      await this.session.terminate()
+      return
+    }
+    if (this.session.dead !== null) return
+    await this.session.send({ kind: 'close', handle: this.handle })
+  }
+}
+
+// ------------------------------------------------------- starting one
+
+/** Start a worker thread with a stack big enough for the store paths. */
+async function startWorker () {
+  if (!workerRouteAvailable()) {
+    throw new StoreOperationError(
+      'this runtime has no worker_threads route for the store operations; ' +
+      'raise the call stack itself instead')
+  }
+  const { Worker } = await import('node:worker_threads')
+  const entry = new URL('./store-worker.mjs', import.meta.url)
+  const worker = new Worker(entry, {
+    resourceLimits: { stackSizeMb: workerStackMb() }
+  })
+  const session = new StoreWorkerSession(worker)
+  // Load the engine now, so the first query measures the query.
+  await session.send({ kind: 'warm' })
+  return session
+}
+
+/** Start a session of this session's own. */
+export async function openStoreWorkerSession () {
+  return await startWorker()
+}
+
+let sharedSession = null
+
+/** The session `openStoreHandleOnWorker` uses when the caller names none:
+ *  one thread, one engine, shared by every handle in the process. */
+export async function sharedStoreWorkerSession () {
+  if (sharedSession === null) sharedSession = startWorker()
+  try {
+    return await sharedSession
+  } catch (error) {
+    sharedSession = null
+    throw error
+  }
+}
+
+/** Stop the shared session, if one was started. */
+export async function closeSharedStoreWorkerSession () {
+  if (sharedSession === null) return
+  const session = sharedSession
+  sharedSession = null
+  await (await session).terminate()
+}
+
+/**
+ * Open one store handle on a worker thread with a raised call stack.
+ *
+ * This is the handle a long-lived process wants against a large store: an
+ * MCP server, a chat tool loop, a SPARQL endpoint. It is asynchronous
+ * where `openStoreHandle` is synchronous, and it needs no runtime flag.
+ *
+ * @param {string} root the collection root that holds CURRENT
+ * @param {{generation?: string, keys?: string[], sparql?: string,
+ *          session?: StoreWorkerSession, ownWorker?: boolean}} options
+ * @returns {Promise<WorkerStoreHandle>}
+ */
+export async function openStoreHandleOnWorker (root, options = {}) {
+  if (!workerRouteAvailable() || workerRefused(options)) {
+    return await openHere(root, options)
+  }
+  if (options.session instanceof StoreWorkerSession) {
+    return await options.session.open(root, options)
+  }
+  if (options.ownWorker === true) {
+    const session = await startWorker()
+    const handle = await session.open(root, options)
+    handle.ownSession = true
+    return handle
+  }
+  const session = await sharedStoreWorkerSession()
+  return await session.open(root, options)
+}
+
+// -------------------------------- the in-process facade, for Deno
+//
+// Same two methods, same envelopes, no thread. It exists so one caller
+// works on both runtimes; on Deno the stack must come from the host
+// (`--v8-flags=--stack-size=65536`), because Deno's worker_threads shim
+// does not raise it (see `workerRouteAvailable`).
+
+/** A handle held in this process, behind the asynchronous interface. */
+export class LocalStoreHandle {
+  constructor (handle) {
+    this.local = handle
+    this.handle = handle.handle
+    this.identity = handle.identity
+    this.layout = handle.layout
+    this.wireVersion = handle.wireVersion
+    this.artifacts = handle.artifacts
+    this.bytes = handle.bytes
+    this.rows = handle.rows
+  }
+
+  get closed () { return this.local.closed }
+
+  async query (sparql) { return this.local.query(sparql) }
+
+  async close () { this.local.close() }
+}
+
+/** Open a handle in this process, behind the asynchronous interface. */
+async function openHere (root, options) {
+  const engine = await loadEngine()
+  const store = openStore(root,
+    typeof options.generation === 'string' ? options.generation : null)
+  const pick = {}
+  if (Array.isArray(options.keys)) pick.keys = options.keys
+  else if (typeof options.sparql === 'string') pick.sparql = options.sparql
+  return new LocalStoreHandle(openStoreHandle(engine, store, pick))
+}
+
+// ------------------------------------------------- the one-shot retry
+
+/**
+ * Run one stateless query on a worker thread, then stop the thread.
+ *
+ * What `factoidal query` retries with after the in-process attempt ran
+ * out of frames. It answers exactly what `queryStore` answers.
+ *
+ * @returns {Promise<{plan: object, result: object, blobBytes: number}>}
+ */
+export async function queryStoreOnWorker (root, generation, sparql) {
+  if (!workerRouteAvailable()) {
+    const engine = await loadEngine()
+    const store = openStore(root, generation ?? null)
+    const answer = queryStore(engine, store, sparql)
+    return { plan: answer.plan, result: answer.result, blobBytes: answer.blobBytes }
+  }
+  const session = await startWorker()
+  try {
+    return await session.send({
+      kind: 'query-once', root, generation: generation ?? null, sparql
+    })
+  } finally {
+    await session.terminate()
+  }
+}
+
+/**
+ * Plan one query on a worker thread, then stop the thread.
+ *
+ * `storeQueryPlan` walks the manifest, and on a collection of a few
+ * thousand blocks that walk alone exceeds the default stack. What
+ * `factoidal query --explain` retries with.
+ *
+ * @returns {Promise<object>} the plan envelope
+ */
+export async function planQueryOnWorker (root, generation, sparql) {
+  if (!workerRouteAvailable()) {
+    const engine = await loadEngine()
+    return planQuery(engine, openStore(root, generation ?? null), sparql)
+  }
+  const session = await startWorker()
+  try {
+    return (await session.send({
+      kind: 'plan', root, generation: generation ?? null, sparql
+    })).plan
+  } finally {
+    await session.terminate()
+  }
+}

package/bin/store-worker.mjs

@@ -0,0 +1,100 @@
+// The thread the store operations run on under Node, so that a query
+// against a large generation has a call stack big enough to finish.
+// https://github.com/danbri/factoidal/issues/653
+//
+// WHAT THIS FILE IS ALLOWED TO DO
+// Load the engine, open a store, hold store handles, run the same store
+// operations the in-process path runs, and post the answers back. It
+// reads no manifest field, verifies no digest, decodes no block and
+// chooses no artifact: `bin/store.mjs` drives the engine here exactly as
+// it does on the main thread (iron rule 7 of CLAUDE.md).
+//
+// THE ENGINE AND THE HANDLES LIVE HERE, NOT ON THE MAIN THREAD.
+// A loaded WebAssembly instance does not cross a thread boundary, and a
+// store handle is state inside that instance: the verified, decoded,
+// indexed blocks. So a handle opened here can only be queried from here.
+// That is why the protocol below carries `query` and `close` as messages
+// rather than returning a handle object. One worker holds many handles,
+// up to the engine's own handle cap.
+
+import { parentPort } from 'node:worker_threads'
+import { loadEngine } from './engine.mjs'
+import { describeError } from './pack-host.mjs'
+import {
+  listStoreHandles, openStore, openStoreHandle, planQuery, queryStore
+} from './store.mjs'
+
+const port = parentPort
+
+/** The handles this worker holds, by the id the engine gave them. */
+const handles = new Map()
+
+let engine = null
+async function theEngine () {
+  if (engine === null) engine = await loadEngine()
+  return engine
+}
+
+/** One request. Every branch answers with a plain object, so nothing
+ *  that crosses the boundary is an engine reference. */
+async function serve (request) {
+  const active = await theEngine()
+  if (request.kind === 'open') {
+    const store = openStore(request.root, request.generation ?? null)
+    const options = {}
+    if (Array.isArray(request.keys)) options.keys = request.keys
+    else if (typeof request.sparql === 'string') options.sparql = request.sparql
+    const handle = openStoreHandle(active, store, options)
+    handles.set(handle.handle, handle)
+    return {
+      handle: handle.handle,
+      identity: handle.identity,
+      layout: handle.layout,
+      wireVersion: handle.wireVersion,
+      artifacts: handle.artifacts,
+      bytes: handle.bytes,
+      rows: handle.rows
+    }
+  }
+  if (request.kind === 'query') {
+    const handle = handles.get(request.handle)
+    if (handle === undefined) {
+      throw new Error(`this worker holds no store handle ${request.handle}`)
+    }
+    return { result: handle.query(request.sparql) }
+  }
+  if (request.kind === 'close') {
+    const handle = handles.get(request.handle)
+    if (handle === undefined) return { closed: false }
+    handle.close()
+    handles.delete(request.handle)
+    return { closed: true }
+  }
+  if (request.kind === 'list') {
+    return { list: listStoreHandles(active) }
+  }
+  if (request.kind === 'plan') {
+    const store = openStore(request.root, request.generation ?? null)
+    return { plan: planQuery(active, store, request.sparql) }
+  }
+  if (request.kind === 'query-once') {
+    // The stateless path: read the artifacts the plan names, hand them
+    // over, drop them. Used by the one-shot `factoidal query` after an
+    // in-process attempt ran out of frames.
+    const store = openStore(request.root, request.generation ?? null)
+    const answer = queryStore(active, store, request.sparql)
+    return {
+      plan: answer.plan,
+      result: answer.result,
+      blobBytes: answer.blobBytes
+    }
+  }
+  if (request.kind === 'warm') return { warm: true }
+  throw new Error(`the store worker was sent an unknown request "${request.kind}"`)
+}
+
+port.on('message', (request) => {
+  serve(request).then(
+    (answer) => port.postMessage({ id: request.id, kind: 'ok', answer }),
+    (error) => port.postMessage({ id: request.id, kind: 'error', error: describeError(error) }))
+})