@factoidal/core 0.11.0 → 0.12.0

package/lws/persist.mjs

@@ -11,15 +11,83 @@
 //   DIR/lws-state.json      the snapshot, replaced atomically
 //
 // No request target ever becomes a file name here, so a request cannot
-// address a file outside DIR. The write is: a temporary file in DIR,
+// address a file outside DIR.
+//
+// ENCRYPTION AT REST (https://github.com/danbri/factoidal/issues/921)
+// This file also reads the environment variable FACTOIDAL_LWS_KEY and
+// draws 12 random bytes for every write. Both go to the engine unread: the
+// key text as the open configuration's `storageKey`, the bytes as the
+// `nonce` of the snapshot call. The engine decides everything else
+// (`L4Factoidal/LWS/SnapshotSeal.lean`): whether the key is valid, whether
+// the file in DIR is sealed, whether it authenticates, and the bytes of
+// the sealed document. This file never inspects the document it writes. The write is: a temporary file in DIR,
 // fsync, rename over the state file, fsync of DIR. A crash leaves the old
 // snapshot or the new one, never a torn one.
 
-import { closeSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, writeSync, existsSync } from 'node:fs'
-import { join, resolve } from 'node:path'
+import { closeSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, unlinkSync, writeSync, existsSync } from 'node:fs'
+import { dirname, join, resolve, sep } from 'node:path'
 
 export const STATE_FILE = 'lws-state.json'
 
+/** The environment variable that holds the storage key. */
+export const STORAGE_KEY_VARIABLE = 'FACTOIDAL_LWS_KEY'
+
+/**
+ * The text of the storage key: `explicit` when the caller gave a string,
+ * none when the caller gave `null`, and otherwise the environment variable
+ * (Node `process.env`, or `Deno.env` when only Deno's API is present). An
+ * empty value is no key. The engine checks the text.
+ */
+export function storageKeyText (explicit) {
+  if (typeof explicit === 'string') return explicit
+  if (explicit === null) return undefined
+  let value
+  try {
+    if (globalThis.process !== undefined && globalThis.process.env !== undefined) {
+      value = globalThis.process.env[STORAGE_KEY_VARIABLE]
+    } else if (globalThis.Deno !== undefined) {
+      value = globalThis.Deno.env.get(STORAGE_KEY_VARIABLE)
+    }
+  } catch (_error) {
+    value = undefined
+  }
+  return typeof value === 'string' && value !== '' ? value : undefined
+}
+
+/** The environment variable that sets where `lws-serve DIR` keeps the
+ *  storage when the caller names no `storage` (issue 918). */
+export const STORAGE_MODE_VARIABLE = 'FACTOIDAL_LWS_STORAGE'
+
+/**
+ * The text of the storage mode: `explicit` when the caller gave a string,
+ * and otherwise the environment variable, passed unread; the engine checks
+ * it. An empty value is none (the engine's default, the snapshot).
+ */
+export function storageModeText (explicit) {
+  if (typeof explicit === 'string') return explicit
+  let value
+  try {
+    if (globalThis.process !== undefined && globalThis.process.env !== undefined) {
+      value = globalThis.process.env[STORAGE_MODE_VARIABLE]
+    } else if (globalThis.Deno !== undefined) {
+      value = globalThis.Deno.env.get(STORAGE_MODE_VARIABLE)
+    }
+  } catch (_error) {
+    value = undefined
+  }
+  return typeof value === 'string' && value !== '' ? value : undefined
+}
+
+/** 12 bytes from the platform's random source, base64: the nonce of one
+ *  snapshot write. Pure I/O. */
+function nonceBase64 () {
+  const bytes = new Uint8Array(12)
+  globalThis.crypto.getRandomValues(bytes)
+  let text = ''
+  for (const b of bytes) text += String.fromCharCode(b)
+  return btoa(text)
+}
+
 /** The absolute path of the state file of a storage directory. */
 export function stateFileOf (dir) {
   return join(resolve(dir), STATE_FILE)
@@ -37,7 +105,7 @@ export function readSnapshot (dir) {
   try {
     return JSON.parse(text)
   } catch (error) {
-    throw new Error(`${file} is not JSON: ${error.message}`)
+    throw Object.assign(new Error(`${file} is not JSON: ${error.message}`), { stateFile: true })
   }
 }
 
@@ -65,14 +133,177 @@ export function writeSnapshot (dir, snapshot) {
 
 /**
  * After one step: when the engine says the storage changed, ask it for
- * the snapshot (`snapshotOp`) and write it.
+ * the snapshot (`snapshotOp`) and write it. The call carries a fresh random
+ * nonce; the engine uses it when the storage is sealed under a key.
  */
 export function persistIfChanged (engine, snapshotOp, handle, envelope, dir) {
   if (typeof dir !== 'string' || envelope === null || typeof envelope !== 'object') return
   if (envelope.changed !== true) return
-  const answer = engine.call(snapshotOp, [handle])
+  const answer = engine.call(snapshotOp, [handle, JSON.stringify({ nonce: nonceBase64() })])
   if (answer === null || typeof answer !== 'object' || answer.snapshot === undefined) {
     throw new Error(`${snapshotOp} answered with no snapshot`)
   }
   writeSnapshot(dir, answer.snapshot)
 }
+
+// ---------------------------------------------------------------------------
+// A STORAGE KEPT IN FILES (https://github.com/danbri/factoidal/issues/918)
+//
+// With the open configuration `storage: "files"` the engine decides the
+// directory layout and every byte (formal/lean4/L4Factoidal/LWS/FileLayout.lean,
+// design record docs/designissues/2026-10-22-lws-storage-on-disk.md). This
+// part of the file only performs what the engine names:
+//
+//   * reads: the engine names files (paths relative to DIR); each is read
+//     whole, or reported absent;
+//   * a plan of file operations, in order, with their bytes in one region:
+//       write  {path, temp, offset, length}  parent directory, temp file,
+//                                            fsync, rename, fsync directory
+//       append {path, offset, length}        append, fsync
+//       remove {path}                        remove; a missing file is no error
+//       rename {from, to}                    rename, fsync directory; a missing
+//                                            file is no error
+//
+// A name that leaves DIR is refused as a failure of this host; the engine
+// never names one.
+
+/** The absolute path of a name the engine gave, inside DIR. */
+function insideDir (root, name) {
+  if (typeof name !== 'string' || name === '') throw new Error('lws files: the engine named no file')
+  const abs = resolve(root, name)
+  if (!abs.startsWith(root + sep)) throw new Error(`lws files: ${name} is outside the storage directory`)
+  return abs
+}
+
+/** fsync a directory; some platforms cannot, and the rename stands. */
+function fsyncDir (dir) {
+  try {
+    const fd = openSync(dir, 'r')
+    try { fsyncSync(fd) } finally { closeSync(fd) }
+  } catch (_error) {
+    // Some platforms cannot fsync a directory.
+  }
+}
+
+/** `seed` for one engine call: 32 random bytes, base64. Pure I/O. */
+function seedBase64 () {
+  const bytes = new Uint8Array(32)
+  globalThis.crypto.getRandomValues(bytes)
+  let text = ''
+  for (const b of bytes) text += String.fromCharCode(b)
+  return btoa(text)
+}
+
+/**
+ * Read the files the engine named. Returns the `files` member of the load
+ * document and the bytes of the present files, one after the other.
+ */
+export function readNamedFiles (dir, names) {
+  const root = resolve(dir)
+  const files = []
+  const parts = []
+  let total = 0
+  for (const name of names) {
+    const abs = insideDir(root, name)
+    if (!existsSync(abs)) {
+      files.push({ path: name, present: false })
+      continue
+    }
+    const bytes = readFileSync(abs)
+    files.push({ path: name, present: true, length: bytes.length })
+    parts.push(bytes)
+    total += bytes.length
+  }
+  const region = new Uint8Array(total)
+  let at = 0
+  for (const part of parts) { region.set(part, at); at += part.length }
+  return { files, region }
+}
+
+/** Perform a plan of file operations exactly as the engine gave it. */
+export function performPlan (dir, plan, bytes) {
+  const root = resolve(dir)
+  mkdirSync(root, { recursive: true })
+  const slice = (op) => bytes.subarray(op.offset, op.offset + op.length)
+  for (const op of Array.isArray(plan) ? plan : []) {
+    switch (op.op) {
+      case 'write': {
+        const path = insideDir(root, op.path)
+        const temp = insideDir(root, op.temp)
+        mkdirSync(dirname(path), { recursive: true })
+        const fd = openSync(temp, 'w', 0o600)
+        try {
+          writeSync(fd, slice(op))
+          fsyncSync(fd)
+        } finally {
+          closeSync(fd)
+        }
+        renameSync(temp, path)
+        fsyncDir(dirname(path))
+        break
+      }
+      case 'append': {
+        const path = insideDir(root, op.path)
+        mkdirSync(dirname(path), { recursive: true })
+        const fd = openSync(path, 'a', 0o600)
+        try {
+          writeSync(fd, slice(op))
+          fsyncSync(fd)
+        } finally {
+          closeSync(fd)
+        }
+        break
+      }
+      case 'remove': {
+        try { unlinkSync(insideDir(root, op.path)) } catch (error) {
+          if (error.code !== 'ENOENT') throw error
+        }
+        break
+      }
+      case 'rename': {
+        const to = insideDir(root, op.to)
+        try { renameSync(insideDir(root, op.from), to) } catch (error) {
+          if (error.code !== 'ENOENT') throw error
+        }
+        fsyncDir(dirname(to))
+        break
+      }
+      default:
+        throw new Error(`lws files: the engine named an unknown file operation ${String(op.op)}`)
+    }
+  }
+}
+
+/**
+ * Open a storage kept in files: read what `lwsOpen` named, give it to
+ * `lwsLoad`, read what that names, until the engine is ready; then perform
+ * the plan of the start. An engine refusal is thrown as the engine's error.
+ */
+export function loadFiles (engine, handle, opened, dir) {
+  let want = opened.load
+  for (let round = 0; round < 8; round += 1) {
+    const { files, region } = readNamedFiles(dir, Array.isArray(want.read) ? want.read : [])
+    const answered = engine.callBlobIO('lwsLoad', [handle, JSON.stringify({ files, seed: seedBase64() })], region)
+    const envelope = answered.envelope
+    if (envelope.ready === true) {
+      performPlan(dir, envelope.plan, answered.bytes)
+      return envelope
+    }
+    if (envelope.load === undefined || envelope.load === null) {
+      throw new Error('lwsLoad answered neither load nor ready')
+    }
+    want = envelope.load
+  }
+  throw new Error('lwsLoad asked for more reads than one start may make')
+}
+
+/**
+ * After one step of a storage kept in files: when the engine says the
+ * storage changed, ask it for the write plan (`lwsPersist`) and perform it.
+ */
+export function persistFilesIfChanged (engine, handle, envelope, dir) {
+  if (typeof dir !== 'string' || envelope === null || typeof envelope !== 'object') return
+  if (envelope.changed !== true) return
+  const answered = engine.callBlobIO('lwsPersist', [handle, JSON.stringify({ seed: seedBase64() })], new Uint8Array(0))
+  performPlan(dir, answered.envelope.plan, answered.bytes)
+}

package/lws/server.mjs

@@ -22,6 +22,29 @@
 //   lwsStep  [handle, requestJson]   -> { ok, response }
 //   lwsClose [handle]                -> { ok }
 //   lwsFetched [handle, documentJson] -> { ok, fetch? | installed? | refused? }
+//   lwsDelivered [handle, reportJson] -> { ok, changed, retry? }
+//   lwsFlush [handle, { now? }]       -> { ok, changed, deliveries?, wake? }
+//
+// RATE LIMIT AND INBOX CHECK (https://github.com/danbri/factoidal/issues/915)
+// An answer may carry `wake: SECONDS`: deliveries wait for a rate-limit
+// window. This host calls `lwsFlush` after that many seconds and makes the
+// deliveries it names. `lwsStep` may answer `{ ok, resolve: { host } }`:
+// this host resolves the name (`dns.lookup`, every address) and steps the
+// request again with `resolved: [{ host, addresses }]`. The listen address,
+// port and the machine's interface addresses go to `lwsOpen` as `listen`.
+// Whether an inbox is this server, and how many deliveries go when, are
+// decided by `L4Factoidal/LWS/InboxGuard.lean` and `Notifications.lean`.
+//
+// NOTIFICATIONS (https://github.com/danbri/factoidal/issues/890)
+// `lwsStep` may carry `deliveries: [{ id, url, headers, body }]`: HTTP
+// requests the engine decided to make (LWS notifications, webhook suite).
+// This host POSTs each one exactly as given, after the response to the
+// request that caused it, and reports the status it got (0 when no answer
+// came) to `lwsDelivered`. When the engine answers `retry`, this host
+// waits the number of seconds it names and makes that request. Which
+// changes are announced, to whom, with which body and signature, whether
+// to retry and when a subscription ends are all decided by
+// `formal/lean4/L4Factoidal/LWS/Notifications.lean`.
 //
 // `lwsStep` may answer `{ ok, fetch: { issuer, url } }` instead of a
 // response: the engine needs an authorization server's metadata or key
@@ -38,7 +61,7 @@
 // region, with no encoding and no decoding here. This host never reads a
 // body as text.
 //
-// The config members are `baseIri`, `now`, `owner`, `agent`, `rootAcl`,
+// The config members are `baseIri`, `now`, `owner`, `agent`, `pageSize`, `rootAcl`,
 // `auth` and `insecureNoAuth`, all optional. `target` is a path within the storage, so this host passes
 // Node's `request.url` through unchanged and never composes an absolute
 // IRI: `baseIri` tells the engine what the root maps to.
@@ -61,10 +84,27 @@
 // produced and this code keeps working unchanged.
 
 import { loadEngine } from '../bin/engine.mjs'
-import { readSnapshot, persistIfChanged } from './persist.mjs'
+import { readSnapshot, persistIfChanged, storageKeyText, storageModeText, loadFiles, persistFilesIfChanged } from './persist.mjs'
 
 /** The ops this host needs from the engine. */
-export const LWS_OPS = ['lwsOpen', 'lwsStep', 'lwsClose', 'lwsSnapshot', 'lwsFetched']
+export const LWS_OPS = ['lwsOpen', 'lwsStep', 'lwsClose', 'lwsSnapshot', 'lwsFetched', 'lwsDelivered',
+  'lwsLoad', 'lwsPersist']
+
+/** How long this host waits for the answer to one delivery request before
+ *  it reports "no answer" (status 0) to the engine. */
+const DELIVERY_TIMEOUT_MS = 10000
+
+/** Random bytes for the engine, base64: `entropy` at open (32 bytes, the
+ *  seed of the signing key of a new storage and of the identifiers of one
+ *  run) and `random` with each request (16 bytes, the name of a search
+ *  results resource when the engine makes one). Pure I/O. */
+function entropyBase64 (size = 32) {
+  const bytes = new Uint8Array(size)
+  globalThis.crypto.getRandomValues(bytes)
+  let text = ''
+  for (const b of bytes) text += String.fromCharCode(b)
+  return btoa(text)
+}
 
 /** The most documents one request may make this host fetch before the
  *  engine has answered it. The engine names at most two per issuer (its
@@ -72,6 +112,32 @@ export const LWS_OPS = ['lwsOpen', 'lwsStep', 'lwsClose', 'lwsSnapshot', 'lwsFet
  *  from looping the host. */
 const MAX_FETCHES = 8
 
+/** The most host names one request may make this host resolve. The engine
+ *  names at most one (the inbox of a subscription request). */
+const MAX_RESOLVES = 4
+
+/** Resolve one host name the engine named. Pure I/O: every address
+ *  `dns.lookup` gives, or none when it fails; the engine decides. */
+async function resolveHost (host) {
+  try {
+    const dns = await import('node:dns')
+    const found = await dns.promises.lookup(host, { all: true, verbatim: true })
+    return found.map((a) => a.address)
+  } catch (_error) {
+    return []
+  }
+}
+
+/** The addresses of this machine's network interfaces. Pure I/O. */
+async function interfaceAddresses () {
+  try {
+    const os = await import('node:os')
+    return Object.values(os.networkInterfaces()).flat().filter(Boolean).map((i) => i.address)
+  } catch (_error) {
+    return []
+  }
+}
+
 /**
  * Fetch one document the engine named, for `lwsFetched`. Pure I/O: the
  * engine decides what the bytes mean. A network failure is reported as
@@ -169,7 +235,7 @@ function contentBytes (request) {
  * Open one LWS storage on the engine and return the handle plus the two
  * calls a host makes against it.
  *
- * The config members are the ABI's own: `baseIri`, `now`, `owner`,
+ * The config members are the ABI's own: `baseIri`, `now`, `owner`, `pageSize`,
  * `agent`. Nothing else is sent.
  *
  * @param {object} options
@@ -177,7 +243,15 @@ function contentBytes (request) {
  * @param {number} [options.now] the engine clock, seconds since the epoch
  * @param {string} [options.owner] the WebID the storage advertises
  * @param {string} [options.agent] the WebID of the requesting agent
+ * @param {number} [options.pageSize] the container pagination threshold and page size (default 100; 0 turns pagination off)
+ * @param {boolean} [options.paginateAlways] present every container listing as paginated (default false)
+ * @param {boolean|string} [options.moves] which resources move: true or "all", "data", false or "off" (default all)
+ * @param {boolean} [options.linksetPut] does a linkset accept PUT (default true)
  * @param {boolean} [options.realClock] stamp each request with this host's clock
+ * @param {string|null} [options.storageKey] the storage key text (64 hex digits or base64url);
+ *   default the environment variable FACTOIDAL_LWS_KEY; `null` reads no key
+ * @param {string} [options.storage] where DIR keeps the storage: "snapshot" (default, one
+ *   file) or "files" (one file per body and a journal, issue 918); passed unread
  * @param {object} [options.engine] an engine handle to reuse
  * @returns {Promise<{step: Function, close: Function, handle: string, engine: object}>}
  */
@@ -188,6 +262,9 @@ export async function openLwsStorage (options = {}) {
   if (typeof options.now === 'number') config.now = options.now
   if (typeof options.owner === 'string') config.owner = options.owner
   if (typeof options.agent === 'string') config.agent = options.agent
+  // The pagination threshold and page size of container listings; the
+  // engine decides what it means (L4Factoidal/LWS/Pagination.lean).
+  if (typeof options.pageSize === 'number') config.pageSize = options.pageSize
   // Authorization (L4Factoidal/LWS/Authorization.lean). `auth` is passed
   // through unread; `insecureNoAuth` is the explicit development mode.
   // With neither, the engine enforces authorization with no trusted
@@ -195,10 +272,66 @@ export async function openLwsStorage (options = {}) {
   if (options.auth !== undefined && options.auth !== null) config.auth = options.auth
   if (options.insecureNoAuth === true) config.insecureNoAuth = true
   if (typeof options.rootAcl === 'string') config.rootAcl = options.rootAcl
+  // Notifications are on unless the caller turns them off; the engine
+  // decides everything else about them.
+  if (options.notifications === false) config.notifications = false
+  // The Type Index and Type Search services are on unless the caller turns
+  // them off (L4Factoidal/LWS/Search.lean).
+  if (options.typeIndex === false) config.typeIndex = false
+  // Moves (a change of the up link, L4Factoidal/LWS/Move.lean) are on
+  // unless the caller turns them off.
+  if (options.moves === false) config.moves = false
+  // `moves` may also be "all", "data" or "off", passed unread; the engine
+  // decides what each means and refuses another value (Wasm/Ops/Lws.lean).
+  if (typeof options.moves === 'string') config.moves = options.moves
+  // A linkset resource accepts PUT unless the caller turns it off
+  // (L4Factoidal/LWS/Operations.lean, linksetStep).
+  if (options.linksetPut === false) config.linksetPut = false
+  // Every container listing paginated, also one that fits on one page
+  // (L4Factoidal/LWS/Pagination.lean, paginatedWith). Off unless turned on.
+  if (options.paginateAlways === true) config.paginateAlways = true
+  // Access requests and grants are on unless the caller turns them off
+  // (L4Factoidal/LWS/AccessGrants.lean).
+  if (options.accessGrants === false) config.accessGrants = false
+  // ANTICIPATORY AND UNSTABLE (https://github.com/danbri/factoidal/issues/922):
+  // a QUERY answer of the type search names a results resource with
+  // Content-Location. Off unless the caller turns it on
+  // (L4Factoidal/LWS/SearchResults.lean decides everything about it).
+  if (options.searchResults === true) config.searchResults = true
+  if (typeof options.searchResultsTtl === 'number') config.searchResultsTtl = options.searchResultsTtl
+  if (typeof options.searchResultsMax === 'number') config.searchResultsMax = options.searchResultsMax
+  // Merkle hashes (https://github.com/danbri/factoidal/issues/919): off
+  // unless the caller turns them on (L4Factoidal/LWS/MerkleHttp.lean decides
+  // every field and member they add).
+  if (options.merkle === true) config.merkle = true
+  // Webhook delivery rate (issue 915), the redirect table bound (issue 914)
+  // and the storage controller's inbox (issue 920): passed unread; the
+  // engine checks and decides.
+  if (typeof options.deliveryLimit === 'number') config.deliveryLimit = options.deliveryLimit
+  if (typeof options.deliveryWindow === 'number') config.deliveryWindow = options.deliveryWindow
+  if (typeof options.redirectsMax === 'number') config.redirectsMax = options.redirectsMax
+  if (typeof options.controllerInbox === 'string') config.controllerInbox = options.controllerInbox
+  // What this host bound: data for the engine's inbox check (issue 915).
+  if (options.listen !== undefined && options.listen !== null) config.listen = options.listen
+  config.entropy = entropyBase64()
+  // Encryption at rest (issue 921): the key text from `storageKey` or the
+  // environment variable FACTOIDAL_LWS_KEY, passed unread. The engine checks
+  // it, refuses a file that does not match it, and seals every snapshot.
+  const storageKey = storageKeyText(options.storageKey)
+  if (storageKey !== undefined) config.storageKey = storageKey
+  // Where the storage is kept (https://github.com/danbri/factoidal/issues/918):
+  // `storage` is passed unread; the engine refuses a value it does not know.
+  // With "files" the engine names the files to read (`load`) and gives the
+  // plan of every write (`lwsPersist`), and this host reads no snapshot.
+  // With a DIR and no `storage` option, the environment variable
+  // FACTOIDAL_LWS_STORAGE is the value; without a DIR there is nothing to
+  // keep, and only an explicit option is passed.
+  const dir = typeof options.dir === 'string' ? options.dir : null
+  const storage = dir !== null ? storageModeText(options.storage) : options.storage
+  if (typeof storage === 'string') config.storage = storage
   // `dir` names the storage directory. The snapshot found there, if any,
   // is the storage to continue from; the engine checks it.
-  const dir = typeof options.dir === 'string' ? options.dir : null
-  if (dir !== null) {
+  if (dir !== null && storage !== 'files') {
     const snapshot = readSnapshot(dir)
     if (snapshot !== null) config.snapshot = snapshot
   }
@@ -217,6 +350,95 @@ export async function openLwsStorage (options = {}) {
   if (handle === null) {
     throw new LwsHostError('lwsOpen answered with no handle')
   }
+  // A storage kept in files: read the files the engine names until it is
+  // ready, and perform the plan of the start. Without a DIR, there are no
+  // files: the engine is told none is present.
+  const filesMode = opened !== null && typeof opened === 'object' &&
+    opened.load !== undefined && opened.load !== null
+  if (filesMode) {
+    if (dir === null) throw new LwsHostError('a storage kept in files needs a storage directory (dir)')
+    loadFiles(engine, handle, opened, dir)
+  }
+  /** After a call whose envelope may say `changed`: write what the engine
+   *  gives (the snapshot, or the plan of a storage kept in files). */
+  const persist = (envelope) => {
+    if (dir === null) return
+    if (filesMode) persistFilesIfChanged(engine, handle, envelope, dir)
+    else persistIfChanged(engine, 'lwsSnapshot', handle, envelope, dir)
+  }
+
+  // Deliveries in flight, so a caller (a test, a shutdown) can wait.
+  const inFlight = new Set()
+  let closed = false
+  // The one timer that calls `lwsFlush` when the engine named a `wake`.
+  let flushTimer = null
+  let flushAt = Infinity
+  const clockNow = () => (options.realClock === true ? { now: Math.floor(Date.now() / 1000) } : {})
+
+  /** Make one delivery request the engine named, report the status, and
+   *  follow the engine's answer. */
+  function deliver (delivery) {
+    const job = (async () => {
+      let status = 0
+      try {
+        const answer = await fetch(delivery.url, {
+          method: 'POST',
+          headers: Array.isArray(delivery.headers) ? delivery.headers : [],
+          body: delivery.body,
+          redirect: 'manual',
+          signal: AbortSignal.timeout(DELIVERY_TIMEOUT_MS)
+        })
+        status = answer.status
+        await answer.arrayBuffer()
+      } catch (_error) {
+        status = 0
+      }
+      if (closed) return
+      const told = engine.call('lwsDelivered', [handle, JSON.stringify({
+        id: delivery.id, status, ...clockNow()
+      })])
+      persist(told)
+      const retry = told !== null && typeof told === 'object' ? told.retry : null
+      if (retry !== undefined && retry !== null && typeof retry.after === 'number') {
+        await new Promise((resolve) => {
+          const timer = setTimeout(resolve, retry.after * 1000)
+          if (timer && typeof timer.unref === 'function') timer.unref()
+        })
+        if (!closed) deliver(retry.delivery)
+      }
+    })()
+    inFlight.add(job)
+    // A failure here is a failure of this host or of the engine call, not a
+    // protocol answer; it is reported and the delivery is dropped.
+    job.catch((error) => { console.error(`lws-serve: delivery: ${error && error.message ? error.message : error}`) })
+      .finally(() => inFlight.delete(job))
+  }
+
+  /** Call `lwsFlush` after `seconds`, unless an earlier call is due. */
+  function scheduleFlush (seconds) {
+    if (closed || typeof seconds !== 'number') return
+    const at = Date.now() + seconds * 1000
+    if (flushTimer !== null && flushAt <= at) return
+    if (flushTimer !== null) clearTimeout(flushTimer)
+    flushAt = at
+    flushTimer = setTimeout(() => {
+      flushTimer = null
+      flushAt = Infinity
+      if (closed) return
+      try {
+        const told = engine.call('lwsFlush', [handle, JSON.stringify(clockNow())])
+        persist(told)
+        if (told !== null && typeof told === 'object') {
+          if (Array.isArray(told.deliveries)) for (const delivery of told.deliveries) deliver(delivery)
+          if (typeof told.wake === 'number') scheduleFlush(told.wake)
+        }
+      } catch (error) {
+        console.error(`lws-serve: flush: ${error && error.message ? error.message : error}`)
+      }
+    }, seconds * 1000)
+    if (flushTimer && typeof flushTimer.unref === 'function') flushTimer.unref()
+  }
+
   return {
     engine,
     handle,
@@ -229,12 +451,30 @@ export async function openLwsStorage (options = {}) {
     async step (request) {
       const content = contentBytes(request)
       const { body: _body, ...fields } = request
+      // Random bytes with every request; the engine uses them only to name
+      // a search results resource (issue 922), and only when that is on.
+      const stamped = { ...fields, random: entropyBase64(16) }
       const record = options.realClock === true
-        ? { ...fields, now: Math.floor(Date.now() / 1000) }
-        : fields
+        ? { ...stamped, now: Math.floor(Date.now() / 1000) }
+        : stamped
+      // The engine may ask for the addresses of a host name first
+      // (`resolve`); this host resolves it and steps the request again.
+      let resolves = 0
       const callStep = () => engine.callBlobIO('lwsStep', [handle, JSON.stringify(record)], content)
       let answered = callStep()
       let envelope = answered.envelope
+      while (envelope !== null && typeof envelope === 'object' &&
+             envelope.resolve !== undefined && envelope.resolve !== null) {
+        resolves += 1
+        if (resolves > MAX_RESOLVES) {
+          throw new LwsHostError('the engine asked for more host names than one request may resolve')
+        }
+        const host = String(envelope.resolve.host)
+        const addresses = await resolveHost(host)
+        record.resolved = [...(Array.isArray(record.resolved) ? record.resolved : []), { host, addresses }]
+        answered = callStep()
+        envelope = answered.envelope
+      }
       // The engine may answer "fetch this document first" (an issuer's
       // metadata or key set). The host fetches exactly what it is told,
       // hands the bytes back, and steps the request again.
@@ -258,10 +498,23 @@ export async function openLwsStorage (options = {}) {
         envelope = answered.envelope
       }
       const answer = responseOf(envelope, answered.bytes)
-      if (dir !== null) persistIfChanged(engine, 'lwsSnapshot', handle, envelope, dir)
+      persist(envelope)
+      if (Array.isArray(envelope.deliveries)) {
+        // After this step's answer is ready; the engine's state already
+        // records them as pending.
+        for (const delivery of envelope.deliveries) setTimeout(() => deliver(delivery), 0)
+      }
+      if (typeof envelope.wake === 'number') scheduleFlush(envelope.wake)
       return answer
     },
+    /** Resolves when no delivery request is in flight. */
+    async settled () {
+      await new Promise((resolve) => setTimeout(resolve, 0))
+      while (inFlight.size > 0) await Promise.allSettled([...inFlight])
+    },
     close () {
+      closed = true
+      if (flushTimer !== null) clearTimeout(flushTimer)
       try { engine.call('lwsClose', [handle]) } catch (_error) { /* already gone */ }
     }
   }
@@ -359,6 +612,14 @@ export async function createLwsServer (options = {}) {
       }
       settings = { ...settings, baseIri: value }
     },
+    /** Pass what the socket bound (address, port, interface addresses),
+     *  before the first request. Data for the engine's inbox check. */
+    setListen (value) {
+      if (opening !== null) {
+        throw new LwsHostError('the storage is already open; set listen first')
+      }
+      settings = { ...settings, listen: value }
+    },
     close () {
       return new Promise((resolve) => {
         server.close(() => {
@@ -406,6 +667,7 @@ export async function listen (options = {}) {
   const bound = created.server.address()
   const origin = `http://${host}:${bound.port}`
   if (typeof options.baseIri !== 'string') created.setBaseIri(`${origin}/`)
+  created.setListen({ address: bound.address, port: bound.port, addresses: await interfaceAddresses() })
   try {
     await created.storage()
   } catch (error) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@factoidal/core",
-  "version": "0.11.0",
+  "version": "0.12.0",
   "description": "RDF parsing and writing (Turtle, TriG, N-Triples, N-Quads, RDF/XML, JSON-LD), SPARQL 1.1 and 1.2, RDFS and OWL reasoning, SHACL and ShEx validation, CSVW and RML mapping, RDFC-1.0, and SPARQL, LWS and Solid servers, for Node, Deno and browsers.",
   "keywords": [
     "sparql",
@@ -135,6 +135,7 @@
     "build": "bash ../../formal/lean4/Wasm/build-wasm.sh",
     "test": "node --test 'test/*.test.js' 'test/*.test.mjs'",
     "test:speed": "node ../../tests/perf/npm_speed_gate.mjs",
+    "test:costs": "node ../../tests/perf/npm_costs.mjs",
     "prepublishOnly": "npm test"
   },
   "devDependencies": {