@factoidal/core 0.7.1 → 0.11.0

package/solid/server/index.mjs

@@ -0,0 +1,395 @@
+// A Node HTTP host for the Solid Protocol v0.11.0 server conformance class.
+// https://github.com/danbri/factoidal/issues/659
+// Design record: docs/designissues/2026-09-06-lws-and-solid-protocols.md
+// Specification: https://solidproject.org/TR/protocol
+//
+// WHAT THIS FILE IS ALLOWED TO DO
+// Accept a socket, turn one HTTP request into the JSON request record,
+// call the engine's `solidStep` on a handle, and write the JSON response
+// record back. Nothing else.
+//
+// It decides no status code, builds no `Link` header, writes no
+// containment triple, reads no ACL and answers no OPTIONS. Storage
+// discovery, containment, slash semantics, the auxiliary lifecycle, N3
+// Patch, Web Access Control, CORS and the LDN inbox are all in
+// `formal/lean4/L4Factoidal/Solid/Server/` (iron rule 7 of CLAUDE.md).
+// The CORS PREFLIGHT is not an exception: an OPTIONS request is passed
+// to `solidStep` like every other request and the engine's headers are
+// written back unread. The only status this file writes is 500, for a
+// failure of the host itself.
+//
+// THE WASM CONTRACT
+// Fixed in docs/lws-solid-conformance.md, section "wasm ABI".
+//   solidOpen  [configJson]            -> { ok, handle, root, storage }
+//   solidStep  [handle, requestJson]   -> { ok, response }
+//   solidClose [handle]                -> { ok }
+//   solidFetched [handle, documentJson] -> { ok, fetch? | installed? | refused? }
+//
+// Solid-OIDC (issue 899): with `auth` in the configuration the engine
+// verifies the DPoP-bound access token itself
+// (formal/lean4/L4Factoidal/Solid/Server/Oidc.lean). When it needs a
+// document it does not hold -- an issuer's OpenID configuration, its JWK
+// Set, or a WebID profile -- `solidStep` answers `{ fetch: {kind, key,
+// url} }`. This host fetches exactly that URL, hands status and body to
+// `solidFetched` unread, follows any `fetch` that answers, and steps the
+// request again. It chooses no URL and reads no document.
+//
+//   request  { method, target, headers: [[name, value]], body }
+//   response { status, headers: [[name, value]], body }
+//
+// The config members are `baseIri` (the absolute IRI the storage root
+// maps to), `now` (the engine clock in seconds, which advances by one
+// second per mutation so a write sequence has increasing Last-Modified
+// values with no real clock), `owner` and `agent`. All are optional.
+//
+// `target` is a path within the storage, so Node's `request.url` is
+// passed through unchanged and this host composes no IRI. With an
+// ephemeral port the origin is not known until the socket is bound, so
+// the storage handle is opened after binding rather than before.
+//
+// `agent` is the WebID of the requesting agent. It stands only when the
+// configuration has no `auth` (or has `insecureNoAuth`); with `auth` the
+// engine decides the WebID from the credential.
+//
+// This file shares no code with `../client/index.mjs` beyond the engine
+// loader, by the design record's rule that the two conformance classes
+// stay distinct.
+
+import { loadEngine } from '../../bin/engine.mjs'
+import { readSnapshot, persistIfChanged } from '../../lws/persist.mjs'
+
+/** The ops this host needs from the engine. */
+export const SOLID_SERVER_OPS = ['solidOpen', 'solidStep', 'solidClose', 'solidSnapshot',
+  'solidFetched']
+
+/** The most documents one request may make this host fetch. The engine
+ *  names at most three per request (issuer metadata, key set, profile);
+ *  this bound only stops a defective module from looping the host. */
+const MAX_FETCHES = 8
+
+/** Fetch one document the engine named. Pure I/O; a network failure is
+ *  status 0, which the engine refuses like any failed fetch. */
+async function fetchDocument (url) {
+  try {
+    const answer = await fetch(url, {
+      redirect: 'error',
+      headers: { accept: 'text/turtle, application/json;q=0.9, */*;q=0.1' }
+    })
+    return { status: answer.status, body: await answer.text() }
+  } catch (error) {
+    return { status: 0, body: String(error && error.message ? error.message : error) }
+  }
+}
+
+/** An error raised by the Solid server host. `unknownOp` is set when the
+ *  loaded WebAssembly module does not carry the operation at all, which
+ *  is what a test skips on. */
+export class SolidServerHostError extends Error {
+  constructor (message, detail = {}) {
+    super(message)
+    this.name = 'SolidServerHostError'
+    this.unknownOp = detail.unknownOp === true
+  }
+}
+
+function isUnknownOp (error) {
+  const text = error && error.message ? String(error.message) : String(error)
+  return text.indexOf('unknown op') >= 0
+}
+
+/**
+ * Whether the loaded engine carries the Solid server operations.
+ *
+ * @param {object} engine the handle from `loadEngine()`
+ * @returns {Promise<{available: boolean, reason: string}>}
+ */
+export async function solidServerOpsAvailable (engine) {
+  try {
+    const opened = engine.call('solidOpen', [JSON.stringify({ probe: true })])
+    const handle = handleOf(opened)
+    if (handle !== null) {
+      try { engine.call('solidClose', [handle]) } catch (_error) { /* probe only */ }
+    }
+    return { available: true, reason: 'solidOpen answered' }
+  } catch (error) {
+    if (isUnknownOp(error)) {
+      return {
+        available: false,
+        reason: 'the loaded l4factoidal WebAssembly module does not carry ' +
+          'the solid server ops (rebuild formal/lean4/Wasm)'
+      }
+    }
+    return { available: true, reason: 'solidOpen answered with an engine error' }
+  }
+}
+
+function handleOf (envelope) {
+  if (envelope === null || typeof envelope !== 'object') return null
+  for (const name of ['handle', 'id']) {
+    const value = envelope[name]
+    if (typeof value === 'string' && value.length > 0) return value
+    if (typeof value === 'number') return String(value)
+  }
+  return null
+}
+
+function responseOf (envelope, bytes) {
+  if (envelope === null || typeof envelope !== 'object') {
+    throw new SolidServerHostError('solidStep answered with no envelope')
+  }
+  const record = (envelope.response !== null && typeof envelope.response === 'object')
+    ? envelope.response
+    : envelope
+  if (typeof record.status !== 'number') {
+    throw new SolidServerHostError(
+      'solidStep answered with no status in its response record')
+  }
+  return {
+    status: record.status,
+    headers: Array.isArray(record.headers) ? record.headers : [],
+    body: bytes instanceof Uint8Array ? bytes : new Uint8Array(0)
+  }
+}
+
+/** The request content as bytes: a byte array is passed as it is, and a
+ *  string (a JavaScript caller's convenience) as its UTF-8 bytes. Bodies
+ *  are bytes end to end (https://github.com/danbri/factoidal/issues/889). */
+function contentBytes (request) {
+  if (request.body instanceof Uint8Array) return request.body
+  if (typeof request.body === 'string') return new TextEncoder().encode(request.body)
+  return new Uint8Array(0)
+}
+
+/**
+ * Open one Solid storage on the engine.
+ *
+ * @param {object} options
+ * @param {string} [options.baseIri] the absolute IRI the storage root maps to
+ * @param {number} [options.now] the engine clock, seconds since the epoch
+ * @param {string} [options.owner] the WebID the storage advertises
+ * @param {string} [options.agent] the WebID of the requesting agent
+ * @param {boolean} [options.realClock] stamp each request with this host's clock
+ * @param {object} [options.engine] an engine handle to reuse
+ * @returns {Promise<{step: Function, close: Function, handle: string,
+ *                    engine: object}>}
+ */
+export async function openSolidStorage (options = {}) {
+  const engine = options.engine !== undefined ? options.engine : await loadEngine()
+  const config = {}
+  if (typeof options.baseIri === 'string') config.baseIri = options.baseIri
+  if (typeof options.now === 'number') config.now = options.now
+  if (typeof options.owner === 'string') config.owner = options.owner
+  if (typeof options.agent === 'string') config.agent = options.agent
+  // Solid-OIDC: `auth` is passed through unread; `insecureNoAuth` is the
+  // explicit development mode.
+  if (options.auth !== undefined && options.auth !== null) config.auth = options.auth
+  if (options.insecureNoAuth === true) config.insecureNoAuth = true
+  if (typeof options.enforceWac === 'boolean') config.enforceWac = options.enforceWac
+  if (typeof options.rootAcl === 'string') config.rootAcl = options.rootAcl
+  // `dir` names the storage directory. The snapshot found there, if any,
+  // is the storage to continue from; the engine checks it.
+  const dir = typeof options.dir === 'string' ? options.dir : null
+  if (dir !== null) {
+    const snapshot = readSnapshot(dir)
+    if (snapshot !== null) config.snapshot = snapshot
+  }
+  let opened
+  try {
+    opened = engine.call('solidOpen', [JSON.stringify(config)])
+  } catch (error) {
+    if (isUnknownOp(error)) {
+      throw new SolidServerHostError(
+        'the loaded l4factoidal WebAssembly module does not carry solidOpen',
+        { unknownOp: true })
+    }
+    throw error
+  }
+  const handle = handleOf(opened)
+  if (handle === null) {
+    throw new SolidServerHostError('solidOpen answered with no handle')
+  }
+  return {
+    engine,
+    handle,
+    storageIri: typeof opened.storage === 'string' ? opened.storage : null,
+    /**
+     * One protocol step.
+     * @param {{method: string, target: string,
+     *          headers: Array<[string, string]>, body: Uint8Array|string}} request
+     * @returns {{status: number, headers: Array<[string, string]>, body: Uint8Array}}
+     */
+    async step (request) {
+      const content = contentBytes(request)
+      const { body: _body, ...fields } = request
+      const record = options.realClock === true
+        ? { ...fields, now: Math.floor(Date.now() / 1000) }
+        : fields
+      // The byte-region entry: request content in, response content out.
+      const callStep = () =>
+        engine.callBlobIO('solidStep', [handle, JSON.stringify(record)], content)
+      let { envelope, bytes } = callStep()
+      let fetches = 0
+      while (envelope !== null && typeof envelope === 'object' &&
+             envelope.fetch !== undefined && envelope.fetch !== null) {
+        let wanted = envelope.fetch
+        while (wanted !== undefined && wanted !== null) {
+          fetches += 1
+          if (fetches > MAX_FETCHES) {
+            throw new SolidServerHostError('the engine asked for more documents than one request may fetch')
+          }
+          const got = await fetchDocument(wanted.url)
+          const told = engine.call('solidFetched', [handle, JSON.stringify({
+            kind: wanted.kind, key: wanted.key, url: wanted.url, status: got.status, body: got.body,
+            ...(typeof record.now === 'number' ? { now: record.now } : {})
+          })])
+          wanted = told !== null && typeof told === 'object' ? told.fetch : null
+        }
+        ({ envelope, bytes } = callStep())
+      }
+      const answer = responseOf(envelope, bytes)
+      if (dir !== null) persistIfChanged(engine, 'solidSnapshot', handle, envelope, dir)
+      return answer
+    },
+    close () {
+      try { engine.call('solidClose', [handle]) } catch (_error) { /* already gone */ }
+    }
+  }
+}
+
+function readBody (request) {
+  return new Promise((resolve, reject) => {
+    const chunks = []
+    request.on('data', (chunk) => chunks.push(chunk))
+    request.on('error', reject)
+    request.on('end', () => resolve(new Uint8Array(Buffer.concat(chunks))))
+  })
+}
+
+function headerPairs (raw) {
+  const pairs = []
+  for (let index = 0; index + 1 < raw.length; index += 2) {
+    pairs.push([raw[index], raw[index + 1]])
+  }
+  return pairs
+}
+
+/**
+ * A Node `http.Server` that answers every request through `solidStep`.
+ *
+ * Returned unlistened, so a test can take an ephemeral port. The storage
+ * handle is opened on demand, because `baseIri` is not known until the
+ * socket is bound.
+ *
+ * @param {object} options as `openSolidStorage`, plus `onError`
+ * @returns {Promise<{server: object, storage: Function, close: Function,
+ *                    setBaseIri: Function}>}
+ */
+export async function createSolidServer (options = {}) {
+  const http = await import('node:http')
+  const onError = typeof options.onError === 'function'
+    ? options.onError
+    : (error) => { console.error(`solid-serve: ${error.message}`) }
+
+  let settings = { ...options }
+  let opening = null
+  let opened = null
+
+  function storage () {
+    if (opening === null) {
+      opening = openSolidStorage(settings).then((value) => { opened = value; return value })
+    }
+    return opening
+  }
+
+  const server = http.createServer(async (request, response) => {
+    let record
+    try {
+      record = {
+        method: request.method,
+        target: request.url,
+        headers: headerPairs(request.rawHeaders),
+        body: await readBody(request)
+      }
+    } catch (error) {
+      onError(error)
+      response.writeHead(500, { 'content-type': 'text/plain' })
+      response.end('solid-serve: could not read the request body\n')
+      return
+    }
+    let answer
+    try {
+      answer = await (await storage()).step(record)
+    } catch (error) {
+      onError(error)
+      response.writeHead(500, { 'content-type': 'text/plain' })
+      response.end(`solid-serve: ${error.message}\n`)
+      return
+    }
+    writeHeaders(response, answer.headers)
+    response.statusCode = answer.status
+    response.end(answer.body)
+  })
+
+  return {
+    server,
+    storage,
+    setBaseIri (value) {
+      if (opening !== null) {
+        throw new SolidServerHostError('the storage is already open; set baseIri first')
+      }
+      settings = { ...settings, baseIri: value }
+    },
+    close () {
+      return new Promise((resolve) => {
+        server.close(() => {
+          if (opened !== null) opened.close()
+          resolve()
+        })
+      })
+    }
+  }
+}
+
+/** Write one response record's header pairs, keeping repeats. */
+function writeHeaders (response, pairs) {
+  for (const pair of pairs) {
+    if (!Array.isArray(pair) || pair.length < 2) continue
+    const existing = response.getHeader(pair[0])
+    if (existing === undefined) {
+      response.setHeader(pair[0], String(pair[1]))
+    } else if (Array.isArray(existing)) {
+      response.setHeader(pair[0], existing.concat([String(pair[1])]))
+    } else {
+      response.setHeader(pair[0], [String(existing), String(pair[1])])
+    }
+  }
+}
+
+/**
+ * Start a Solid server and resolve once it is listening and the storage
+ * is open, so that a module without the Solid ops is reported by the
+ * command that started the server rather than by a later request.
+ *
+ * @param {object} options as `createSolidServer`, plus `port` and `host`
+ * @returns {Promise<{server: object, storage: Function, port: number,
+ *                    origin: string, close: Function}>}
+ */
+export async function listen (options = {}) {
+  const port = typeof options.port === 'number' ? options.port : 0
+  const host = typeof options.host === 'string' ? options.host : '127.0.0.1'
+  const created = await createSolidServer(options)
+  await new Promise((resolve, reject) => {
+    created.server.once('error', reject)
+    created.server.listen(port, host, resolve)
+  })
+  const bound = created.server.address()
+  const origin = `http://${host}:${bound.port}`
+  if (typeof options.baseIri !== 'string') created.setBaseIri(`${origin}/`)
+  try {
+    await created.storage()
+  } catch (error) {
+    await new Promise((resolve) => created.server.close(resolve))
+    throw error
+  }
+  return { ...created, port: bound.port, origin }
+}

package/store-host/deno.mjs

@@ -239,6 +239,14 @@ export function atomicReplace (path, bytes) {
   }
 }
 
+export function listNames (directory) {
+  try {
+    return [...Deno.readDirSync(directory)].map((entry) => entry.name).sort()
+  } catch (cause) {
+    throw wrap('DIR_READ_FAILED', `cannot list ${directory}`, directory, cause)
+  }
+}
+
 export function listGeneration (directory) {
   const out = []
   let entries

package/store-host/index.mjs

@@ -190,6 +190,18 @@ export function listGeneration (directory) {
   return impl.listGeneration(directory)
 }
 
+/**
+ * The names (files and subdirectories) in one directory, sorted. A host
+ * primitive for the engine, which decides what the names mean (the current
+ * generation of a Jena TDB2 database: `tdb2ServeKeys`).
+ * @param {string} directory
+ * @returns {string[]}
+ */
+export function listNames (directory) {
+  requirePath(directory, 'directory')
+  return impl.listNames(directory)
+}
+
 // The manifest file names a generation directory can carry, in the order
 // `Harness.ShardMerklePread.readManifest` tries them. This module reads
 // whichever exists and returns its bytes untouched; it does not look

package/store-host/node.mjs

@@ -251,6 +251,14 @@ export function atomicReplace (path, bytes) {
   }
 }
 
+export function listNames (directory) {
+  try {
+    return readdirSync(directory).sort()
+  } catch (cause) {
+    throw wrap('DIR_READ_FAILED', `cannot list ${directory}`, directory, cause)
+  }
+}
+
 export function listGeneration (directory) {
   let names
   try {

package/xmpp/README.md

@@ -0,0 +1,107 @@
+# `@factoidal/core/xmpp`
+
+A functional-programming-style JavaScript API over `L4Factoidal.XMPP`
+(Lean 4, compiled to WebAssembly) — the fresh, GC3-targeting XMPP
+implementation described in
+[`formal/lean4/L4Factoidal/XMPP/README.md`](../../../formal/lean4/L4Factoidal/XMPP/README.md).
+
+```js
+import { parseJid, parseStreamHeader, featuresFor, parseStanza } from '@factoidal/core/xmpp'
+
+const jid = await parseJid('romeo@example.net/orchard')
+// { ok: true, value: { localpart: 'romeo', domainpart: 'example.net',
+//                       resourcepart: 'orchard', wellFormed: true } }
+
+const bad = await parseJid('@example.com')
+// { ok: false, error: 'not a structurally valid JID: @example.com', unknownOp: false }
+
+const features = await featuresFor('postTls', ['SCRAM-SHA-256', 'PLAIN'])
+// { ok: true, value: { startTlsRequired: null,
+//                       saslMechanisms: ['SCRAM-SHA-256', 'PLAIN'],
+//                       canBind: false, canSession: false,
+//                       xml: '<stream:features>...</stream:features>' } }
+```
+
+## Design: functions in, results out, nothing thrown for an expected outcome
+
+Every function here is pure: plain data in, plain data out, no classes,
+nothing to construct or hold onto. A malformed JID, a stanza that isn't
+`message`/`presence`/`iq`, an unknown negotiation stage — these are
+**ordinary answers**, not exceptional control flow, so every function
+returns
+
+```ts
+{ ok: true, value: T } | { ok: false, error: string, unknownOp: boolean }
+```
+
+rather than throwing. The one thing that still throws is a genuine
+host/environment failure — the wasm module not being resolvable or
+loadable at all (`loadXmppEngine()`), which is not a protocol-level
+outcome and has no sensible data shape to return instead.
+
+## API
+
+| Function | Wraps op | Notes |
+|---|---|---|
+| `parseJid(jid, options?)` | `xmppJidParse` | RFC 7622 structural + length well-formedness; PRECIS not yet enforced (see below) |
+| `renderJid(parts, options?)` | `xmppJidRender` | `{localpart?, domainpart, resourcepart?}` → string |
+| `parseStreamHeader(xml, options?)` | `xmppStreamHeaderParse` | tolerates a leading `<?xml ...?>` declaration |
+| `renderStreamHeader(header, options?)` | `xmppStreamHeaderRender` | inverse |
+| `featuresFor(stage, mechanisms?, options?)` | `xmppFeaturesFor` | `stage` is one of `preTls`/`postTls`/`authenticated`/`bound` |
+| `parseStanza(xml, options?)` | `xmppStanzaParse` | `message`/`presence`/`iq` only; payload children come back as one serialized `payloadXml` string, not a tree |
+
+Every function accepts an optional `{ engine }` in its last argument to
+reuse an already-loaded engine handle (`await loadXmppEngine()`) instead
+of paying the load cost per call — see `loadXmppEngine()`'s doc comment.
+
+`xmppOpsAvailable(options?)` reports whether the resolved wasm module
+carries these ops at all, so a caller (or a test) can skip gracefully
+against an older committed bundle instead of getting a confusing
+"unknown op" error.
+
+## What this does not cover yet, on purpose
+
+- **Stanza construction from JSON.** `parseStanza` has no inverse: building
+  a stanza's XML back from a JSON payload description needs a
+  sibling-node XML constructor the Lean side doesn't have yet.
+- **GC3 itself.** Still an unfinished, early-stage upstream XSF spec —
+  see the design record.
+- **SASL/SCRAM computation.** The message *format* exists on the Lean
+  side (`L4Factoidal.XMPP.Sasl`); the actual hash/HMAC computation
+  doesn't yet (no HACL* HMAC-SHA-256 binding — see the design record's
+  crypto section).
+- **MUC/MIX interop modules.**
+
+## Upstream limit found via this module's tests (2026-09-07), repaired 2026-10-10
+
+`parseStanza`/`parseStreamHeader` delegate general XML parsing to the
+shared `L4Factoidal.XML` module (reused rather than duplicated — see
+the design record). Until 2026-10-10 its scan of a text run or a start
+tag for references (`XML.scanRefs`) used one stack frame per reference:
+a text body with about 2,000 consecutive entity references (for example
+`"&amp;".repeat(2000)`) overflowed the wasm call stack
+(`Maximum call stack size exceeded`), and the adversarial test was
+pinned at 1,000. Issue: https://github.com/danbri/factoidal/issues/673.
+
+The scan is now a loop (`XML.scanRefsLoop`, proved equal to
+`XML.scanRefs`). Measured 2026-10-10 with the rebuilt wasm module:
+100,000 sequential references in a text node or an attribute value parse
+under `node --stack-size=512`; before the repair, 5,000 overflowed there.
+`test/xmpp.test.mjs` now parses a body of 10,000 `&amp;` references and
+checks the rendered payload. The depth of NESTED entity expansion (an
+entity whose replacement text references another entity) still costs one
+frame per level, bounded by the number of declared entities. RFC 6120
+§11.1 excludes DTD subsets and every entity reference except the five
+predefined ones from an XML stream, so conforming XMPP input has no
+nested expansion.
+
+## Testing
+
+```bash
+cd npm/factoidal
+node --test test/xmpp.test.mjs
+```
+
+Skips wholesale when the Lean wasm assets aren't resolvable, and skips
+per-op when the resolved bundle predates a given op (see
+`xmppOpsAvailable`) — the same pattern `test/l4-core.test.js` uses.