@factoidal/core 0.11.0 → 0.13.0

package/lws/server.mjs

@@ -22,6 +22,29 @@
 //   lwsStep  [handle, requestJson]   -> { ok, response }
 //   lwsClose [handle]                -> { ok }
 //   lwsFetched [handle, documentJson] -> { ok, fetch? | installed? | refused? }
+//   lwsDelivered [handle, reportJson] -> { ok, changed, retry? }
+//   lwsFlush [handle, { now? }]       -> { ok, changed, deliveries?, wake? }
+//
+// RATE LIMIT AND INBOX CHECK (https://github.com/danbri/factoidal/issues/915)
+// An answer may carry `wake: SECONDS`: deliveries wait for a rate-limit
+// window. This host calls `lwsFlush` after that many seconds and makes the
+// deliveries it names. `lwsStep` may answer `{ ok, resolve: { host } }`:
+// this host resolves the name (`dns.lookup`, every address) and steps the
+// request again with `resolved: [{ host, addresses }]`. The listen address,
+// port and the machine's interface addresses go to `lwsOpen` as `listen`.
+// Whether an inbox is this server, and how many deliveries go when, are
+// decided by `L4Factoidal/LWS/InboxGuard.lean` and `Notifications.lean`.
+//
+// NOTIFICATIONS (https://github.com/danbri/factoidal/issues/890)
+// `lwsStep` may carry `deliveries: [{ id, url, headers, body }]`: HTTP
+// requests the engine decided to make (LWS notifications, webhook suite).
+// This host POSTs each one exactly as given, after the response to the
+// request that caused it, and reports the status it got (0 when no answer
+// came) to `lwsDelivered`. When the engine answers `retry`, this host
+// waits the number of seconds it names and makes that request. Which
+// changes are announced, to whom, with which body and signature, whether
+// to retry and when a subscription ends are all decided by
+// `formal/lean4/L4Factoidal/LWS/Notifications.lean`.
 //
 // `lwsStep` may answer `{ ok, fetch: { issuer, url } }` instead of a
 // response: the engine needs an authorization server's metadata or key
@@ -38,7 +61,7 @@
 // region, with no encoding and no decoding here. This host never reads a
 // body as text.
 //
-// The config members are `baseIri`, `now`, `owner`, `agent`, `rootAcl`,
+// The config members are `baseIri`, `now`, `owner`, `agent`, `pageSize`, `rootAcl`,
 // `auth` and `insecureNoAuth`, all optional. `target` is a path within the storage, so this host passes
 // Node's `request.url` through unchanged and never composes an absolute
 // IRI: `baseIri` tells the engine what the root maps to.
@@ -61,10 +84,27 @@
 // produced and this code keeps working unchanged.
 
 import { loadEngine } from '../bin/engine.mjs'
-import { readSnapshot, persistIfChanged } from './persist.mjs'
+import { readSnapshot, persistIfChanged, storageKeyText, storageModeText, loadFiles, persistFilesIfChanged, readStepBodies } from './persist.mjs'
 
 /** The ops this host needs from the engine. */
-export const LWS_OPS = ['lwsOpen', 'lwsStep', 'lwsClose', 'lwsSnapshot', 'lwsFetched']
+export const LWS_OPS = ['lwsOpen', 'lwsStep', 'lwsClose', 'lwsSnapshot', 'lwsFetched', 'lwsDelivered',
+  'lwsLoad', 'lwsPersist', 'lwsRead']
+
+/** How long this host waits for the answer to one delivery request before
+ *  it reports "no answer" (status 0) to the engine. */
+const DELIVERY_TIMEOUT_MS = 10000
+
+/** Random bytes for the engine, base64: `entropy` at open (32 bytes, the
+ *  seed of the signing key of a new storage and of the identifiers of one
+ *  run) and `random` with each request (16 bytes, the name of a search
+ *  results resource when the engine makes one). Pure I/O. */
+function entropyBase64 (size = 32) {
+  const bytes = new Uint8Array(size)
+  globalThis.crypto.getRandomValues(bytes)
+  let text = ''
+  for (const b of bytes) text += String.fromCharCode(b)
+  return btoa(text)
+}
 
 /** The most documents one request may make this host fetch before the
  *  engine has answered it. The engine names at most two per issuer (its
@@ -72,6 +112,32 @@ export const LWS_OPS = ['lwsOpen', 'lwsStep', 'lwsClose', 'lwsSnapshot', 'lwsFet
  *  from looping the host. */
 const MAX_FETCHES = 8
 
+/** The most host names one request may make this host resolve. The engine
+ *  names at most one (the inbox of a subscription request). */
+const MAX_RESOLVES = 4
+
+/** Resolve one host name the engine named. Pure I/O: every address
+ *  `dns.lookup` gives, or none when it fails; the engine decides. */
+async function resolveHost (host) {
+  try {
+    const dns = await import('node:dns')
+    const found = await dns.promises.lookup(host, { all: true, verbatim: true })
+    return found.map((a) => a.address)
+  } catch (_error) {
+    return []
+  }
+}
+
+/** The addresses of this machine's network interfaces. Pure I/O. */
+async function interfaceAddresses () {
+  try {
+    const os = await import('node:os')
+    return Object.values(os.networkInterfaces()).flat().filter(Boolean).map((i) => i.address)
+  } catch (_error) {
+    return []
+  }
+}
+
 /**
  * Fetch one document the engine named, for `lwsFetched`. Pure I/O: the
  * engine decides what the bytes mean. A network failure is reported as
@@ -169,7 +235,7 @@ function contentBytes (request) {
  * Open one LWS storage on the engine and return the handle plus the two
  * calls a host makes against it.
  *
- * The config members are the ABI's own: `baseIri`, `now`, `owner`,
+ * The config members are the ABI's own: `baseIri`, `now`, `owner`, `pageSize`,
  * `agent`. Nothing else is sent.
  *
  * @param {object} options
@@ -177,7 +243,17 @@ function contentBytes (request) {
  * @param {number} [options.now] the engine clock, seconds since the epoch
  * @param {string} [options.owner] the WebID the storage advertises
  * @param {string} [options.agent] the WebID of the requesting agent
+ * @param {number} [options.pageSize] the container pagination threshold and page size (default 100; 0 turns pagination off)
+ * @param {boolean} [options.paginateAlways] present every container listing as paginated (default false)
+ * @param {boolean|string} [options.moves] which resources move: true or "all", "data", false or "off" (default all)
+ * @param {boolean} [options.linksetPut] does a linkset accept PUT (default true)
  * @param {boolean} [options.realClock] stamp each request with this host's clock
+ * @param {string|null} [options.storageKey] the storage key text (64 hex digits or base64url);
+ *   default the environment variable FACTOIDAL_LWS_KEY; `null` reads no key
+ * @param {string} [options.storage] where DIR keeps the storage: "snapshot" (default, one
+ *   file) or "files" (one file per body and a journal, issue 918); passed unread
+ * @param {boolean|string[]} [options.cors] CORS (issue 923): false for none, a list of
+ *   origins to answer; default: echo any valid request Origin. Passed unread
  * @param {object} [options.engine] an engine handle to reuse
  * @returns {Promise<{step: Function, close: Function, handle: string, engine: object}>}
  */
@@ -188,6 +264,9 @@ export async function openLwsStorage (options = {}) {
   if (typeof options.now === 'number') config.now = options.now
   if (typeof options.owner === 'string') config.owner = options.owner
   if (typeof options.agent === 'string') config.agent = options.agent
+  // The pagination threshold and page size of container listings; the
+  // engine decides what it means (L4Factoidal/LWS/Pagination.lean).
+  if (typeof options.pageSize === 'number') config.pageSize = options.pageSize
   // Authorization (L4Factoidal/LWS/Authorization.lean). `auth` is passed
   // through unread; `insecureNoAuth` is the explicit development mode.
   // With neither, the engine enforces authorization with no trusted
@@ -195,10 +274,70 @@ export async function openLwsStorage (options = {}) {
   if (options.auth !== undefined && options.auth !== null) config.auth = options.auth
   if (options.insecureNoAuth === true) config.insecureNoAuth = true
   if (typeof options.rootAcl === 'string') config.rootAcl = options.rootAcl
+  // Notifications are on unless the caller turns them off; the engine
+  // decides everything else about them.
+  if (options.notifications === false) config.notifications = false
+  // The Type Index and Type Search services are on unless the caller turns
+  // them off (L4Factoidal/LWS/Search.lean).
+  if (options.typeIndex === false) config.typeIndex = false
+  // Moves (a change of the up link, L4Factoidal/LWS/Move.lean) are on
+  // unless the caller turns them off.
+  if (options.moves === false) config.moves = false
+  // `moves` may also be "all", "data" or "off", passed unread; the engine
+  // decides what each means and refuses another value (Wasm/Ops/Lws.lean).
+  if (typeof options.moves === 'string') config.moves = options.moves
+  // A linkset resource accepts PUT unless the caller turns it off
+  // (L4Factoidal/LWS/Operations.lean, linksetStep).
+  if (options.linksetPut === false) config.linksetPut = false
+  // Every container listing paginated, also one that fits on one page
+  // (L4Factoidal/LWS/Pagination.lean, paginatedWith). Off unless turned on.
+  if (options.paginateAlways === true) config.paginateAlways = true
+  // Access requests and grants are on unless the caller turns them off
+  // (L4Factoidal/LWS/AccessGrants.lean).
+  if (options.accessGrants === false) config.accessGrants = false
+  // ANTICIPATORY AND UNSTABLE (https://github.com/danbri/factoidal/issues/922):
+  // a QUERY answer of the type search names a results resource with
+  // Content-Location. Off unless the caller turns it on
+  // (L4Factoidal/LWS/SearchResults.lean decides everything about it).
+  if (options.searchResults === true) config.searchResults = true
+  if (typeof options.searchResultsTtl === 'number') config.searchResultsTtl = options.searchResultsTtl
+  if (typeof options.searchResultsMax === 'number') config.searchResultsMax = options.searchResultsMax
+  // Merkle hashes (https://github.com/danbri/factoidal/issues/919): off
+  // unless the caller turns them on (L4Factoidal/LWS/MerkleHttp.lean decides
+  // every field and member they add).
+  if (options.merkle === true) config.merkle = true
+  // Webhook delivery rate (issue 915), the redirect table bound (issue 914)
+  // and the storage controller's inbox (issue 920): passed unread; the
+  // engine checks and decides.
+  if (typeof options.deliveryLimit === 'number') config.deliveryLimit = options.deliveryLimit
+  if (typeof options.deliveryWindow === 'number') config.deliveryWindow = options.deliveryWindow
+  if (typeof options.redirectsMax === 'number') config.redirectsMax = options.redirectsMax
+  if (typeof options.controllerInbox === 'string') config.controllerInbox = options.controllerInbox
+  // CORS (https://github.com/danbri/factoidal/issues/923): `cors` is false or
+  // a list of origins, passed unread; absent, the engine echoes the request
+  // Origin. L4Factoidal/LWS/Cors.lean decides every CORS field.
+  if (options.cors !== undefined) config.cors = options.cors
+  // What this host bound: data for the engine's inbox check (issue 915).
+  if (options.listen !== undefined && options.listen !== null) config.listen = options.listen
+  config.entropy = entropyBase64()
+  // Encryption at rest (issue 921): the key text from `storageKey` or the
+  // environment variable FACTOIDAL_LWS_KEY, passed unread. The engine checks
+  // it, refuses a file that does not match it, and seals every snapshot.
+  const storageKey = storageKeyText(options.storageKey)
+  if (storageKey !== undefined) config.storageKey = storageKey
+  // Where the storage is kept (https://github.com/danbri/factoidal/issues/918):
+  // `storage` is passed unread; the engine refuses a value it does not know.
+  // With "files" the engine names the files to read (`load`) and gives the
+  // plan of every write (`lwsPersist`), and this host reads no snapshot.
+  // With a DIR and no `storage` option, the environment variable
+  // FACTOIDAL_LWS_STORAGE is the value; without a DIR there is nothing to
+  // keep, and only an explicit option is passed.
+  const dir = typeof options.dir === 'string' ? options.dir : null
+  const storage = dir !== null ? storageModeText(options.storage) : options.storage
+  if (typeof storage === 'string') config.storage = storage
   // `dir` names the storage directory. The snapshot found there, if any,
   // is the storage to continue from; the engine checks it.
-  const dir = typeof options.dir === 'string' ? options.dir : null
-  if (dir !== null) {
+  if (dir !== null && storage !== 'files') {
     const snapshot = readSnapshot(dir)
     if (snapshot !== null) config.snapshot = snapshot
   }
@@ -217,6 +356,95 @@ export async function openLwsStorage (options = {}) {
   if (handle === null) {
     throw new LwsHostError('lwsOpen answered with no handle')
   }
+  // A storage kept in files: read the files the engine names until it is
+  // ready, and perform the plan of the start. Without a DIR, there are no
+  // files: the engine is told none is present.
+  const filesMode = opened !== null && typeof opened === 'object' &&
+    opened.load !== undefined && opened.load !== null
+  if (filesMode) {
+    if (dir === null) throw new LwsHostError('a storage kept in files needs a storage directory (dir)')
+    loadFiles(engine, handle, opened, dir)
+  }
+  /** After a call whose envelope may say `changed`: write what the engine
+   *  gives (the snapshot, or the plan of a storage kept in files). */
+  const persist = (envelope) => {
+    if (dir === null) return
+    if (filesMode) persistFilesIfChanged(engine, handle, envelope, dir)
+    else persistIfChanged(engine, 'lwsSnapshot', handle, envelope, dir)
+  }
+
+  // Deliveries in flight, so a caller (a test, a shutdown) can wait.
+  const inFlight = new Set()
+  let closed = false
+  // The one timer that calls `lwsFlush` when the engine named a `wake`.
+  let flushTimer = null
+  let flushAt = Infinity
+  const clockNow = () => (options.realClock === true ? { now: Math.floor(Date.now() / 1000) } : {})
+
+  /** Make one delivery request the engine named, report the status, and
+   *  follow the engine's answer. */
+  function deliver (delivery) {
+    const job = (async () => {
+      let status = 0
+      try {
+        const answer = await fetch(delivery.url, {
+          method: 'POST',
+          headers: Array.isArray(delivery.headers) ? delivery.headers : [],
+          body: delivery.body,
+          redirect: 'manual',
+          signal: AbortSignal.timeout(DELIVERY_TIMEOUT_MS)
+        })
+        status = answer.status
+        await answer.arrayBuffer()
+      } catch (_error) {
+        status = 0
+      }
+      if (closed) return
+      const told = engine.call('lwsDelivered', [handle, JSON.stringify({
+        id: delivery.id, status, ...clockNow()
+      })])
+      persist(told)
+      const retry = told !== null && typeof told === 'object' ? told.retry : null
+      if (retry !== undefined && retry !== null && typeof retry.after === 'number') {
+        await new Promise((resolve) => {
+          const timer = setTimeout(resolve, retry.after * 1000)
+          if (timer && typeof timer.unref === 'function') timer.unref()
+        })
+        if (!closed) deliver(retry.delivery)
+      }
+    })()
+    inFlight.add(job)
+    // A failure here is a failure of this host or of the engine call, not a
+    // protocol answer; it is reported and the delivery is dropped.
+    job.catch((error) => { console.error(`lws-serve: delivery: ${error && error.message ? error.message : error}`) })
+      .finally(() => inFlight.delete(job))
+  }
+
+  /** Call `lwsFlush` after `seconds`, unless an earlier call is due. */
+  function scheduleFlush (seconds) {
+    if (closed || typeof seconds !== 'number') return
+    const at = Date.now() + seconds * 1000
+    if (flushTimer !== null && flushAt <= at) return
+    if (flushTimer !== null) clearTimeout(flushTimer)
+    flushAt = at
+    flushTimer = setTimeout(() => {
+      flushTimer = null
+      flushAt = Infinity
+      if (closed) return
+      try {
+        const told = engine.call('lwsFlush', [handle, JSON.stringify(clockNow())])
+        persist(told)
+        if (told !== null && typeof told === 'object') {
+          if (Array.isArray(told.deliveries)) for (const delivery of told.deliveries) deliver(delivery)
+          if (typeof told.wake === 'number') scheduleFlush(told.wake)
+        }
+      } catch (error) {
+        console.error(`lws-serve: flush: ${error && error.message ? error.message : error}`)
+      }
+    }, seconds * 1000)
+    if (flushTimer && typeof flushTimer.unref === 'function') flushTimer.unref()
+  }
+
   return {
     engine,
     handle,
@@ -229,12 +457,46 @@ export async function openLwsStorage (options = {}) {
     async step (request) {
       const content = contentBytes(request)
       const { body: _body, ...fields } = request
+      // Random bytes with every request; the engine uses them only to name
+      // a search results resource (issue 922), and only when that is on.
+      const stamped = { ...fields, random: entropyBase64(16) }
       const record = options.realClock === true
-        ? { ...fields, now: Math.floor(Date.now() / 1000) }
-        : fields
-      const callStep = () => engine.callBlobIO('lwsStep', [handle, JSON.stringify(record)], content)
+        ? { ...stamped, now: Math.floor(Date.now() / 1000) }
+        : stamped
+      // The engine may ask for the addresses of a host name first
+      // (`resolve`); this host resolves it and steps the request again.
+      let resolves = 0
+      // A storage kept in files (https://github.com/danbri/factoidal/issues/918,
+      // stage 2) may answer `read: [NAME, ...]` first: the body files the step
+      // needs. This host reads exactly those files, passes them to `lwsRead`
+      // (the engine checks them), and steps the same request again.
+      const callStep = () => {
+        let got = engine.callBlobIO('lwsStep', [handle, JSON.stringify(record)], content)
+        let reads = 0
+        while (got.envelope !== null && typeof got.envelope === 'object' && Array.isArray(got.envelope.read)) {
+          reads += 1
+          if (reads > 2 || dir === null) {
+            throw new LwsHostError('the engine asked for body files this host cannot read')
+          }
+          readStepBodies(engine, handle, got.envelope.read, dir)
+          got = engine.callBlobIO('lwsStep', [handle, JSON.stringify(record)], content)
+        }
+        return got
+      }
       let answered = callStep()
       let envelope = answered.envelope
+      while (envelope !== null && typeof envelope === 'object' &&
+             envelope.resolve !== undefined && envelope.resolve !== null) {
+        resolves += 1
+        if (resolves > MAX_RESOLVES) {
+          throw new LwsHostError('the engine asked for more host names than one request may resolve')
+        }
+        const host = String(envelope.resolve.host)
+        const addresses = await resolveHost(host)
+        record.resolved = [...(Array.isArray(record.resolved) ? record.resolved : []), { host, addresses }]
+        answered = callStep()
+        envelope = answered.envelope
+      }
       // The engine may answer "fetch this document first" (an issuer's
       // metadata or key set). The host fetches exactly what it is told,
       // hands the bytes back, and steps the request again.
@@ -258,10 +520,23 @@ export async function openLwsStorage (options = {}) {
         envelope = answered.envelope
       }
       const answer = responseOf(envelope, answered.bytes)
-      if (dir !== null) persistIfChanged(engine, 'lwsSnapshot', handle, envelope, dir)
+      persist(envelope)
+      if (Array.isArray(envelope.deliveries)) {
+        // After this step's answer is ready; the engine's state already
+        // records them as pending.
+        for (const delivery of envelope.deliveries) setTimeout(() => deliver(delivery), 0)
+      }
+      if (typeof envelope.wake === 'number') scheduleFlush(envelope.wake)
       return answer
     },
+    /** Resolves when no delivery request is in flight. */
+    async settled () {
+      await new Promise((resolve) => setTimeout(resolve, 0))
+      while (inFlight.size > 0) await Promise.allSettled([...inFlight])
+    },
     close () {
+      closed = true
+      if (flushTimer !== null) clearTimeout(flushTimer)
       try { engine.call('lwsClose', [handle]) } catch (_error) { /* already gone */ }
     }
   }
@@ -359,6 +634,14 @@ export async function createLwsServer (options = {}) {
       }
       settings = { ...settings, baseIri: value }
     },
+    /** Pass what the socket bound (address, port, interface addresses),
+     *  before the first request. Data for the engine's inbox check. */
+    setListen (value) {
+      if (opening !== null) {
+        throw new LwsHostError('the storage is already open; set listen first')
+      }
+      settings = { ...settings, listen: value }
+    },
     close () {
       return new Promise((resolve) => {
         server.close(() => {
@@ -406,6 +689,7 @@ export async function listen (options = {}) {
   const bound = created.server.address()
   const origin = `http://${host}:${bound.port}`
   if (typeof options.baseIri !== 'string') created.setBaseIri(`${origin}/`)
+  created.setListen({ address: bound.address, port: bound.port, addresses: await interfaceAddresses() })
   try {
     await created.storage()
   } catch (error) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@factoidal/core",
-  "version": "0.11.0",
+  "version": "0.13.0",
   "description": "RDF parsing and writing (Turtle, TriG, N-Triples, N-Quads, RDF/XML, JSON-LD), SPARQL 1.1 and 1.2, RDFS and OWL reasoning, SHACL and ShEx validation, CSVW and RML mapping, RDFC-1.0, and SPARQL, LWS and Solid servers, for Node, Deno and browsers.",
   "keywords": [
     "sparql",
@@ -69,6 +69,7 @@
     "./store-host": "./store-host/index.mjs",
     "./store-host/paths": "./store-host/paths.mjs",
     "./xmpp": "./xmpp/index.mjs",
+    "./xmpp/server": "./xmpp/server.mjs",
     "./store": {
       "default": "./bin/store.mjs"
     },
@@ -87,6 +88,12 @@
     "./lws": {
       "default": "./lws/server.mjs"
     },
+    "./lws/client": {
+      "default": "./lws/client.mjs"
+    },
+    "./authz": {
+      "default": "./authz/server.mjs"
+    },
     "./solid/server": {
       "default": "./solid/server/index.mjs"
     },
@@ -116,6 +123,7 @@
     "lib/",
     "bin/",
     "lws/",
+    "authz/",
     "solid/",
     "store-host/",
     "xmpp/",
@@ -135,6 +143,7 @@
     "build": "bash ../../formal/lean4/Wasm/build-wasm.sh",
     "test": "node --test 'test/*.test.js' 'test/*.test.mjs'",
     "test:speed": "node ../../tests/perf/npm_speed_gate.mjs",
+    "test:costs": "node ../../tests/perf/npm_costs.mjs",
     "prepublishOnly": "npm test"
   },
   "devDependencies": {

package/rdfjs.mjs

@@ -15,6 +15,8 @@
 // ES module (0.9.0): the same file serves Node (index.mjs, and rdfjs.js for
 // require()) and the browser (browser.js), with no build step.
 
+import { FactoidalError } from './lib/errors.mjs';
+
 // RDF/JS Data Model defaults of DataFactory.literal.
 const XSD_STRING = 'http://www.w3.org/2001/XMLSchema#string'; // purity-lint: allow RDF/JS DataFactory default
 const RDF_LANGSTRING =
@@ -330,10 +332,10 @@ function _setTextEngine(engine) { textEngine = engine; }
 
 function needEngine(who) {
   if (!textEngine) {
-    throw new Error(
+    throw new FactoidalError(
       `${who}: N-Quads text is read and written by the engine, which is not ` +
       'loaded yet. Await any @factoidal/core call first (for example ' +
-      '`await ready()`), or use parse() and Dataset.toString().');
+      '`await ready()`), or use parse() and Dataset.toString().', { code: 'op.unknown' });
   }
   return textEngine;
 }

package/solid/server/index.mjs

@@ -56,7 +56,7 @@
 // stay distinct.
 
 import { loadEngine } from '../../bin/engine.mjs'
-import { readSnapshot, persistIfChanged } from '../../lws/persist.mjs'
+import { readSnapshot, persistIfChanged, storageKeyText } from '../../lws/persist.mjs'
 
 /** The ops this host needs from the engine. */
 export const SOLID_SERVER_OPS = ['solidOpen', 'solidStep', 'solidClose', 'solidSnapshot',
@@ -169,6 +169,8 @@ function contentBytes (request) {
  * @param {string} [options.owner] the WebID the storage advertises
  * @param {string} [options.agent] the WebID of the requesting agent
  * @param {boolean} [options.realClock] stamp each request with this host's clock
+ * @param {string|null} [options.storageKey] the storage key text (64 hex digits or base64url);
+ *   default the environment variable FACTOIDAL_LWS_KEY; `null` reads no key
  * @param {object} [options.engine] an engine handle to reuse
  * @returns {Promise<{step: Function, close: Function, handle: string,
  *                    engine: object}>}
@@ -186,6 +188,10 @@ export async function openSolidStorage (options = {}) {
   if (options.insecureNoAuth === true) config.insecureNoAuth = true
   if (typeof options.enforceWac === 'boolean') config.enforceWac = options.enforceWac
   if (typeof options.rootAcl === 'string') config.rootAcl = options.rootAcl
+  // Encryption at rest (issue 921), as the LWS host: the key text from
+  // `storageKey` or FACTOIDAL_LWS_KEY, passed unread; the engine decides.
+  const storageKey = storageKeyText(options.storageKey)
+  if (storageKey !== undefined) config.storageKey = storageKey
   // `dir` names the storage directory. The snapshot found there, if any,
   // is the storage to continue from; the engine checks it.
   const dir = typeof options.dir === 'string' ? options.dir : null

package/xmpp/README.md

@@ -59,6 +59,85 @@ carries these ops at all, so a caller (or a test) can skip gracefully
 against an older committed bundle instead of getting a confusing
 "unknown op" error.
 
+## The server: `@factoidal/core/xmpp/server` and `factoidal xmpp-serve`
+
+```sh
+mkdir -p state && printf 'juliet:r0m30\nromeo:juli3t\n' > state/accounts
+npx factoidal xmpp-serve state --domain localhost            # plaintext on 5222
+npx factoidal xmpp-serve state --cert fullchain.pem --key privkey.pem   # XEP-0368 direct TLS on 5223
+```
+
+```js
+import { listen } from '@factoidal/core/xmpp/server'
+const running = await listen({ dir: 'state', domain: 'localhost', port: 0 })
+console.log(running.origin)        // xmpp://127.0.0.1:<port>
+await running.close()
+```
+
+`server.mjs` is a socket, a UTF-8 decoder and a routing table. It runs
+one engine session per connection and answers every byte through the
+engine's `xmppSessionFeed` operation; `L4Factoidal.XMPP.Server.step`
+(Lean, `formal/lean4/L4Factoidal/XMPP/Server.lean`) makes every
+protocol decision and the host performs what comes back. It is the same
+library the native carrier (`deploy/fly/xmpp/`, one `l4xmpp-serve`
+process per connection behind socat) serves; the wasm session ops are
+`formal/lean4/Wasm/Ops/XmppSession.lean`.
+
+| op | arguments | answer |
+|---|---|---|
+| `xmppOpen` | `[configJson]` | `{ ok, handle, domain, accounts }` |
+| `xmppSessionOpen` | `[server, optionsJson]` | `{ ok, handle }` — `streamId` and `nonce` are required and come from the host's random source |
+| `xmppSessionFeed` | `[session, text]` | `{ ok, send: [...], route: [{to, text}], presence: [...], rostersSaved: [...], close, stage, bare, full, bufferedBytes }` |
+| `xmppSessionDeliver` | `[session, text]` | `{ ok, delivered, send }` — `delivered:false` for an unbound session; nothing is lost |
+| `xmppSessionClose` | `[session]` | `{ ok, send }` |
+| `xmppRosterGet` / `xmppRosterSet` | `[server, bare]` / `[server, bare, encoded]` | the roster in the engine's own on-disk encoding |
+| `xmppClose` | `[server]` | `{ ok, sessions }` |
+
+The state directory is laid out as the native carrier's, so the two
+hosts can share one: `DIR/accounts` (one `user:password` per line),
+`DIR/roster/<bare jid>.tsv` (persisted on every `rostersSaved`, loaded
+on start), and `DIR/spool/<bare jid>/<file>` — a stanza written there
+by any process is delivered to that account's bound session and
+unlinked. That last one is how a program that is not an XMPP client (a
+web application, say) hands a user a message: write a file.
+
+Roster snapshots are written in order through temporary-file replacement.
+`await running.close()` waits for queued writes and rejects if persistence
+failed. This protects against overlapping writes within one host; it does
+not coordinate concurrent processes or promise recovery after a power loss.
+The spool file is removed when the engine accepts delivery, before any
+remote acknowledgement. Stanzas queued for offline accounts in this Node
+host are held in memory and disappear on process exit.
+
+`maxBuffered` (default 1 MiB) limits the engine's residual UTF-8 input after
+each feed, reported by `bufferedBytes`. A decoder can separately hold up to
+three bytes of an incomplete UTF-8 sequence. This is not a bound on complete
+stanza size, output queues, offline queues, connection count or idle time.
+The host requires this field; an older engine without it closes the affected
+connection with an error. Rebuild the host and engine together.
+
+What it speaks: RFC 6120 stream negotiation, stream errors, SASL PLAIN
+and SCRAM-SHA-256, resource binding; RFC 6121 roster, presence
+subscription and broadcast, message delivery with the server-stamped
+`from`; XEP-0030 `disco#info`; XEP-0368 direct TLS with `--cert`/`--key`.
+Not: STARTTLS, server-to-server federation, MUC/MIX/GC3 rooms, PRECIS.
+Known limitation shared with the native harness: one SCRAM salt per
+server (`saltB64` in the config), not one per account.
+
+The shared engine currently reduces routed destinations to bare JIDs and
+the Node host delivers to every bound resource. It does not implement full
+resource-selection/priority semantics. Binding alone permits delivery; this
+is not a complete presence-availability policy. The plaintext test listener
+also offers SASL PLAIN and has no transport confidentiality. Supplying only
+one of `cert`/`key` is rejected; supplying both selects direct TLS.
+
+Tests: `node --test test/xmpp-serve.test.mjs` drives the host over real
+TCP sockets, including an independent SCRAM-SHA-256 computed with
+`node:crypto`; `tools/xmpp-interop.sh wasm` drives it with
+`@xmpp/client`, a third-party implementation.
+`node --test test/xmpp-host.test.mjs` separately checks IO cleanup and
+persistence with a mock engine; passing it makes no protocol-conformance claim.
+
 ## What this does not cover yet, on purpose
 
 - **Stanza construction from JSON.** `parseStanza` has no inverse: building
@@ -66,10 +145,9 @@ against an older committed bundle instead of getting a confusing
   sibling-node XML constructor the Lean side doesn't have yet.
 - **GC3 itself.** Still an unfinished, early-stage upstream XSF spec —
   see the design record.
-- **SASL/SCRAM computation.** The message *format* exists on the Lean
-  side (`L4Factoidal.XMPP.Sasl`); the actual hash/HMAC computation
-  doesn't yet (no HACL* HMAC-SHA-256 binding — see the design record's
-  crypto section).
+- **SASL/SCRAM computation as a pure function.** The server ops above
+  do SCRAM-SHA-256 (via `L4Factoidal.XMPP.Scram` and the HACL* SHA-256
+  binding); there is no standalone pure op for it in this module.
 - **MUC/MIX interop modules.**
 
 ## Upstream limit found via this module's tests (2026-09-07), repaired 2026-10-10