@factoidal/core 0.19.0 → 0.21.0

package/lws/persist.mjs

@@ -9,6 +9,8 @@
 // This file decides only where the one file lives and how it is written:
 //
 //   DIR/lws-state.json      the snapshot, replaced atomically
+//   DIR/lws.lock            the lock of the one writing process (named by
+//                           the engine, issue 1135; see the end of this file)
 //
 // No request target ever becomes a file name here, so a request cannot
 // address a file outside DIR.
@@ -26,6 +28,7 @@
 
 import { closeSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, unlinkSync, writeSync, existsSync } from 'node:fs'
 import { dirname, join, resolve, sep } from 'node:path'
+import { hostname } from 'node:os'
 
 export const STATE_FILE = 'lws-state.json'
 
@@ -323,3 +326,169 @@ export function persistFilesIfChanged (engine, handle, envelope, dir) {
   const answered = engine.callBlobIO('lwsPersist', [handle, JSON.stringify({ seed: seedBase64() })], new Uint8Array(0))
   performPlan(dir, answered.envelope.plan, answered.bytes)
 }
+
+// ---------------------------------------------------------------------------
+// ONE WRITING PROCESS PER STORAGE DIRECTORY
+// (https://github.com/danbri/factoidal/issues/1135)
+//
+// The engine decides the lock file name, its contents and what a start does
+// (`lwsDirLock`, formal/lean4/L4Factoidal/LWS/DirLock.lean). This host does
+// three things only: the exclusive create of the file it names (flag `wx`,
+// O_EXCL), the liveness probe of a process id it names (`kill(pid, 0)`), and
+// the removal of the file at a clean exit. Each result goes back to the
+// engine as data.
+
+/** Random text of this process, drawn once: it tells a lock this process
+ *  wrote from the lock of an earlier process with the same process id. */
+let processToken = null
+function processTokenText () {
+  if (processToken === null) {
+    const bytes = new Uint8Array(18)
+    globalThis.crypto.getRandomValues(bytes)
+    let text = ''
+    for (const b of bytes) text += String.fromCharCode(b)
+    processToken = btoa(text)
+  }
+  return processToken
+}
+
+/** The lock files this process holds: absolute path -> contents. */
+const heldLocks = new Map()
+let exitHookSet = false
+
+/** Remove a lock file this process holds, when it still holds this
+ *  process's contents. Pure I/O. */
+function removeHeldLock (file) {
+  if (!heldLocks.has(file)) return
+  const contents = heldLocks.get(file)
+  heldLocks.delete(file)
+  try {
+    if (readFileSync(file, 'utf8') === contents) unlinkSync(file)
+  } catch (_error) {
+    // Already gone, or not readable: nothing to remove.
+  }
+}
+
+/** At process exit (normal end, `process.exit`, and after a SIGINT or
+ *  SIGTERM handler ends the process), remove every lock still held. */
+function ensureExitHook () {
+  if (exitHookSet || globalThis.process === undefined || typeof globalThis.process.on !== 'function') return
+  exitHookSet = true
+  globalThis.process.on('exit', () => {
+    for (const file of [...heldLocks.keys()]) removeHeldLock(file)
+  })
+}
+
+/** Is process `pid` alive on this host? Pure I/O: `kill(pid, 0)`; EPERM
+ *  means it exists under another user. */
+function processAlive (pid) {
+  try {
+    globalThis.process.kill(pid, 0)
+    return true
+  } catch (error) {
+    return error !== null && typeof error === 'object' && error.code === 'EPERM'
+  }
+}
+
+/** Create `file` with O_EXCL and write `contents`; false when it exists. */
+function createExclusive (file, contents) {
+  let fd
+  try {
+    fd = openSync(file, 'wx', 0o600)
+  } catch (error) {
+    if (error.code === 'EEXIST') return false
+    throw error
+  }
+  try {
+    writeSync(fd, contents)
+    fsyncSync(fd)
+  } finally {
+    closeSync(fd)
+  }
+  return true
+}
+
+/** An error that stops a start because another process holds the storage
+ *  directory. `envelope` carries the engine's message, as for every other
+ *  refusal of a start. */
+export class StorageLockedError extends Error {
+  constructor (message) {
+    super(message)
+    this.name = 'StorageLockedError'
+    this.envelope = { ok: false, error: message }
+  }
+}
+
+/**
+ * Take the lock of storage directory `dir` for `command` (`lws-serve`,
+ * `solid-serve`, `webdav-serve`). Returns `{ file, warning, release }`:
+ * `warning` is the engine's text when it took over a stale lock, else null;
+ * `release()` removes the lock. Throws `StorageLockedError` with the
+ * engine's message when the engine refuses.
+ */
+export function acquireDirLock (engine, dir, command) {
+  const root = resolve(dir)
+  mkdirSync(root, { recursive: true })
+  const base = {
+    dir: root,
+    pid: globalThis.process.pid,
+    command,
+    started: new Date().toISOString(),
+    host: hostname(),
+    token: processTokenText()
+  }
+  let request = { ...base }
+  // Each round is one engine answer; a create that loses to another start
+  // reads the file again and asks again.
+  for (let round = 0; round < 8; round += 1) {
+    const told = engine.call('lwsDirLock', [JSON.stringify(request)])
+    if (told === null || typeof told !== 'object' || told.ok === false) {
+      throw Object.assign(new Error(`lwsDirLock: ${told && told.error ? told.error : 'no answer'}`), { envelope: told })
+    }
+    switch (told.action) {
+      case 'create':
+      case 'replace': {
+        const file = insideDir(root, told.file)
+        if (told.action === 'replace') {
+          try { unlinkSync(file) } catch (error) { if (error.code !== 'ENOENT') throw error }
+        }
+        if (createExclusive(file, told.contents)) {
+          heldLocks.set(file, told.contents)
+          ensureExitHook()
+          return {
+            file,
+            warning: told.action === 'replace' ? told.warning : null,
+            release: () => removeHeldLock(file)
+          }
+        }
+        let present = null
+        try { present = readFileSync(file, 'utf8') } catch (error) { if (error.code !== 'ENOENT') throw error }
+        request = present === null ? { ...base } : { ...base, present }
+        break
+      }
+      case 'probe':
+        request = { ...request, alive: processAlive(Number(told.pid)) }
+        break
+      case 'refuse':
+        throw new StorageLockedError(told.message)
+      default:
+        throw new Error(`lwsDirLock answered an unknown action ${String(told.action)}`)
+    }
+  }
+  throw new Error('lwsDirLock: the lock file changed on every attempt')
+}
+
+/**
+ * The start notice about encryption at rest of the storage in `dir`
+ * (`lwsAtRestNotice`): `{ sealed, keyId?, notice }`. The key text is passed
+ * unread; the engine checks it.
+ */
+export function atRestNotice (engine, dir, storageKey) {
+  const request = { dir: resolve(dir) }
+  if (typeof storageKey === 'string') request.storageKey = storageKey
+  const told = engine.call('lwsAtRestNotice', [JSON.stringify(request)])
+  if (told === null || typeof told !== 'object' || told.ok === false) {
+    throw Object.assign(new Error(`lwsAtRestNotice: ${told && told.error ? told.error : 'no answer'}`), { envelope: told })
+  }
+  return told
+}

package/lws/server.mjs

@@ -84,11 +84,11 @@
 // produced and this code keeps working unchanged.
 
 import { loadEngine } from '../bin/engine.mjs'
-import { readSnapshot, persistIfChanged, storageKeyText, storageModeText, loadFiles, persistFilesIfChanged, readStepBodies } from './persist.mjs'
+import { readSnapshot, persistIfChanged, storageKeyText, storageModeText, loadFiles, persistFilesIfChanged, readStepBodies, acquireDirLock, atRestNotice } from './persist.mjs'
 
 /** The ops this host needs from the engine. */
 export const LWS_OPS = ['lwsOpen', 'lwsStep', 'lwsClose', 'lwsSnapshot', 'lwsFetched', 'lwsDelivered',
-  'lwsLoad', 'lwsPersist', 'lwsRead']
+  'lwsLoad', 'lwsPersist', 'lwsRead', 'lwsDirLock', 'lwsAtRestNotice']
 
 /** How long this host waits for the answer to one delivery request before
  *  it reports "no answer" (status 0) to the engine. */
@@ -254,8 +254,11 @@ function contentBytes (request) {
  *   file) or "files" (one file per body and a journal, issue 918); passed unread
  * @param {boolean|string[]} [options.cors] CORS (issue 923): false for none, a list of
  *   origins to answer; default: echo any valid request Origin. Passed unread
+ * @param {string} [options.lockCommand] the command named in the lock file of DIR
+ *   (default "lws-serve"; https://github.com/danbri/factoidal/issues/1135)
  * @param {object} [options.engine] an engine handle to reuse
- * @returns {Promise<{step: Function, close: Function, handle: string, engine: object}>}
+ * @returns {Promise<{step: Function, close: Function, handle: string, engine: object,
+ *                    lockFile: string|null, notices: string[]}>}
  */
 export async function openLwsStorage (options = {}) {
   const engine = options.engine !== undefined ? options.engine : await loadEngine()
@@ -315,8 +318,14 @@ export async function openLwsStorage (options = {}) {
   if (typeof options.controllerInbox === 'string') config.controllerInbox = options.controllerInbox
   // CORS (https://github.com/danbri/factoidal/issues/923): `cors` is false or
   // a list of origins, passed unread; absent, the engine echoes the request
-  // Origin. L4Factoidal/LWS/Cors.lean decides every CORS field.
+  // Origin under authorization and sends no CORS field with insecureNoAuth
+  // (issue 1124). L4Factoidal/LWS/Cors.lean decides every CORS field.
   if (options.cors !== undefined) config.cors = options.cors
+  // The Host and Origin guard of a storage with insecureNoAuth
+  // (https://github.com/danbri/factoidal/issues/1124): extra host names and
+  // origins, passed unread; L4Factoidal/HTTP/LocalGuard.lean decides.
+  if (Array.isArray(options.allowHosts)) config.allowHosts = options.allowHosts
+  if (Array.isArray(options.allowOrigins)) config.allowOrigins = options.allowOrigins
   // What this host bound: data for the engine's inbox check (issue 915).
   if (options.listen !== undefined && options.listen !== null) config.listen = options.listen
   config.entropy = entropyBase64()
@@ -335,36 +344,69 @@ export async function openLwsStorage (options = {}) {
   const dir = typeof options.dir === 'string' ? options.dir : null
   const storage = dir !== null ? storageModeText(options.storage) : options.storage
   if (typeof storage === 'string') config.storage = storage
-  // `dir` names the storage directory. The snapshot found there, if any,
-  // is the storage to continue from; the engine checks it.
-  if (dir !== null && storage !== 'files') {
-    const snapshot = readSnapshot(dir)
-    if (snapshot !== null) config.snapshot = snapshot
+  // One writing process per storage directory
+  // (https://github.com/danbri/factoidal/issues/1135): the lock is taken
+  // before anything in DIR is read, and released by `close` and at process
+  // exit. The engine decides the file, its contents and every refusal
+  // (L4Factoidal/LWS/DirLock.lean); a refusal is thrown with its message.
+  let lock = null
+  if (dir !== null) {
+    try {
+      lock = acquireDirLock(engine, dir, typeof options.lockCommand === 'string' ? options.lockCommand : 'lws-serve')
+    } catch (error) {
+      if (isUnknownOp(error)) {
+        throw new LwsHostError(
+          'the loaded l4factoidal WebAssembly module does not carry lwsDirLock',
+          { unknownOp: true })
+      }
+      throw error
+    }
   }
   let opened
+  let handle
+  let filesMode
+  let notices = []
   try {
-    opened = engine.call('lwsOpen', [JSON.stringify(config)])
-  } catch (error) {
-    if (isUnknownOp(error)) {
-      throw new LwsHostError(
-        'the loaded l4factoidal WebAssembly module does not carry lwsOpen',
-        { unknownOp: true })
+    // `dir` names the storage directory. The snapshot found there, if any,
+    // is the storage to continue from; the engine checks it.
+    if (dir !== null && storage !== 'files') {
+      const snapshot = readSnapshot(dir)
+      if (snapshot !== null) config.snapshot = snapshot
     }
+    try {
+      opened = engine.call('lwsOpen', [JSON.stringify(config)])
+    } catch (error) {
+      if (isUnknownOp(error)) {
+        throw new LwsHostError(
+          'the loaded l4factoidal WebAssembly module does not carry lwsOpen',
+          { unknownOp: true })
+      }
+      throw error
+    }
+    handle = handleOf(opened)
+    if (handle === null) {
+      throw new LwsHostError('lwsOpen answered with no handle')
+    }
+    // A storage kept in files: read the files the engine names until it is
+    // ready, and perform the plan of the start. Without a DIR, there are no
+    // files: the engine is told none is present.
+    filesMode = opened !== null && typeof opened === 'object' &&
+      opened.load !== undefined && opened.load !== null
+    if (filesMode) {
+      if (dir === null) throw new LwsHostError('a storage kept in files needs a storage directory (dir)')
+      loadFiles(engine, handle, opened, dir)
+    }
+    // What a server prints at start, after the experimental code notice:
+    // a take-over of a stale lock, and whether the storage on disk is
+    // sealed (encryption at rest; the engine words both).
+    if (dir !== null) {
+      if (lock.warning !== null) notices.push(`WARNING: ${lock.warning}`)
+      notices.push(atRestNotice(engine, dir, storageKey).notice)
+    }
+  } catch (error) {
+    if (lock !== null) lock.release()
     throw error
   }
-  const handle = handleOf(opened)
-  if (handle === null) {
-    throw new LwsHostError('lwsOpen answered with no handle')
-  }
-  // A storage kept in files: read the files the engine names until it is
-  // ready, and perform the plan of the start. Without a DIR, there are no
-  // files: the engine is told none is present.
-  const filesMode = opened !== null && typeof opened === 'object' &&
-    opened.load !== undefined && opened.load !== null
-  if (filesMode) {
-    if (dir === null) throw new LwsHostError('a storage kept in files needs a storage directory (dir)')
-    loadFiles(engine, handle, opened, dir)
-  }
   /** After a call whose envelope may say `changed`: write what the engine
    *  gives (the snapshot, or the plan of a storage kept in files). */
   const persist = (envelope) => {
@@ -448,6 +490,10 @@ export async function openLwsStorage (options = {}) {
   return {
     engine,
     handle,
+    /** The lock file of the storage directory, or null with no DIR. */
+    lockFile: lock !== null ? lock.file : null,
+    /** The start notices, for the server to print on standard error. */
+    notices,
     /**
      * One protocol step.
      * @param {{method: string, target: string,
@@ -538,6 +584,7 @@ export async function openLwsStorage (options = {}) {
       closed = true
       if (flushTimer !== null) clearTimeout(flushTimer)
       try { engine.call('lwsClose', [handle]) } catch (_error) { /* already gone */ }
+      if (lock !== null) lock.release()
     }
   }
 }
@@ -593,6 +640,12 @@ export async function createLwsServer (options = {}) {
    *  (L4Factoidal/Authz/Mount.lean). */
   function storage () {
     if (opening === null) {
+      // What the socket bound, when the caller listened itself and did not
+      // say (data for the engine's Host check, issue 1124).
+      const bound = server.address()
+      if (settings.listen === undefined && bound !== null && typeof bound === 'object') {
+        settings = { ...settings, listen: { address: bound.address, port: bound.port, addresses: [] } }
+      }
       opening = (async () => {
         let storageSettings = settings
         if (settings.authz !== undefined && settings.authz !== null) {
@@ -720,7 +773,7 @@ export async function listen (options = {}) {
   const bound = created.server.address()
   const origin = `http://${host}:${bound.port}`
   if (typeof options.baseIri !== 'string') created.setBaseIri(`${origin}/`)
-  created.setListen({ address: bound.address, port: bound.port, addresses: await interfaceAddresses() })
+  created.setListen({ host, address: bound.address, port: bound.port, addresses: await interfaceAddresses() })
   try {
     await created.storage()
   } catch (error) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@factoidal/core",
-  "version": "0.19.0",
+  "version": "0.21.0",
   "description": "EXPERIMENTAL UNREVIEWED CODE: do not run public services on it without careful independent expert review. RDF parsing and writing (Turtle, TriG, N-Triples, N-Quads, RDF/XML, JSON-LD), SPARQL 1.1 and 1.2, RDFS and OWL reasoning, SHACL and ShEx validation, CSVW and RML mapping, RDFC-1.0, and SPARQL, LWS and Solid servers, for Node, Deno and browsers.",
   "keywords": [
     "sparql",
@@ -70,6 +70,7 @@
     "./store-host/paths": "./store-host/paths.mjs",
     "./xmpp": "./xmpp/index.mjs",
     "./xmpp/server": "./xmpp/server.mjs",
+    "./cipher-host": "./lib/cipher-host.mjs",
     "./store": {
       "default": "./bin/store.mjs"
     },

package/solid/server/index.mjs

@@ -55,8 +55,9 @@
 // loader, by the design record's rule that the two conformance classes
 // stay distinct.
 
+import { existsSync, readdirSync } from 'node:fs'
 import { loadEngine } from '../../bin/engine.mjs'
-import { readSnapshot, persistIfChanged, storageKeyText } from '../../lws/persist.mjs'
+import { readSnapshot, persistIfChanged, storageKeyText, acquireDirLock, atRestNotice } from '../../lws/persist.mjs'
 
 /** The ops this host needs from the engine. */
 export const SOLID_SERVER_OPS = ['solidOpen', 'solidStep', 'solidClose', 'solidSnapshot',
@@ -187,6 +188,11 @@ export async function openSolidStorage (options = {}) {
   if (options.auth !== undefined && options.auth !== null) config.auth = options.auth
   if (options.insecureNoAuth === true) config.insecureNoAuth = true
   if (typeof options.enforceWac === 'boolean') config.enforceWac = options.enforceWac
+  // The Host and Origin guard with insecureNoAuth (issue 1124): what the
+  // socket bound and the operator's names and origins, passed unread.
+  if (options.listen !== undefined && options.listen !== null) config.listen = options.listen
+  if (Array.isArray(options.allowHosts)) config.allowHosts = options.allowHosts
+  if (Array.isArray(options.allowOrigins)) config.allowOrigins = options.allowOrigins
   if (typeof options.rootAcl === 'string') config.rootAcl = options.rootAcl
   // Encryption at rest (issue 921), as the LWS host: the key text from
   // `storageKey` or FACTOIDAL_LWS_KEY, passed unread; the engine decides.
@@ -195,28 +201,65 @@ export async function openSolidStorage (options = {}) {
   // `dir` names the storage directory. The snapshot found there, if any,
   // is the storage to continue from; the engine checks it.
   const dir = typeof options.dir === 'string' ? options.dir : null
+  // One writing process per storage directory
+  // (https://github.com/danbri/factoidal/issues/1135), as the LWS host: the
+  // engine decides; the lock is taken before DIR is read.
+  let lock = null
   if (dir !== null) {
-    const snapshot = readSnapshot(dir)
-    if (snapshot !== null) config.snapshot = snapshot
+    try {
+      lock = acquireDirLock(engine, dir, 'solid-serve')
+    } catch (error) {
+      if (isUnknownOp(error)) {
+        throw new SolidServerHostError(
+          'the loaded l4factoidal WebAssembly module does not carry lwsDirLock',
+          { unknownOp: true })
+      }
+      throw error
+    }
   }
   let opened
+  let handle
+  const notices = []
   try {
-    opened = engine.call('solidOpen', [JSON.stringify(config)])
-  } catch (error) {
-    if (isUnknownOp(error)) {
-      throw new SolidServerHostError(
-        'the loaded l4factoidal WebAssembly module does not carry solidOpen',
-        { unknownOp: true })
+    if (dir !== null) {
+      const snapshot = readSnapshot(dir)
+      if (snapshot !== null) config.snapshot = snapshot
+      // The names in DIR, unread: the engine refuses a storage that LWS
+      // keeps there (`solidRefusesLws`, Wasm/Ops/Solid.lean); only lws-serve
+      // serves an LWS storage.
+      config.dirFiles = existsSync(dir) ? readdirSync(dir) : []
+    }
+    try {
+      opened = engine.call('solidOpen', [JSON.stringify(config)])
+    } catch (error) {
+      if (isUnknownOp(error)) {
+        throw new SolidServerHostError(
+          'the loaded l4factoidal WebAssembly module does not carry solidOpen',
+          { unknownOp: true })
+      }
+      throw error
+    }
+    handle = handleOf(opened)
+    if (handle === null) {
+      throw new SolidServerHostError('solidOpen answered with no handle')
     }
+    // The start notices: a take-over of a stale lock, and whether the
+    // storage on disk is sealed (encryption at rest).
+    if (dir !== null) {
+      if (lock.warning !== null) notices.push(`WARNING: ${lock.warning}`)
+      notices.push(atRestNotice(engine, dir, storageKey).notice)
+    }
+  } catch (error) {
+    if (lock !== null) lock.release()
     throw error
   }
-  const handle = handleOf(opened)
-  if (handle === null) {
-    throw new SolidServerHostError('solidOpen answered with no handle')
-  }
   return {
     engine,
     handle,
+    /** The lock file of the storage directory, or null with no DIR. */
+    lockFile: lock !== null ? lock.file : null,
+    /** The start notices, for the server to print on standard error. */
+    notices,
     storageIri: typeof opened.storage === 'string' ? opened.storage : null,
     /**
      * One protocol step.
@@ -258,6 +301,7 @@ export async function openSolidStorage (options = {}) {
     },
     close () {
       try { engine.call('solidClose', [handle]) } catch (_error) { /* already gone */ }
+      if (lock !== null) lock.release()
     }
   }
 }
@@ -302,6 +346,10 @@ export async function createSolidServer (options = {}) {
 
   function storage () {
     if (opening === null) {
+      const bound = server.address()
+      if (settings.listen === undefined && bound !== null && typeof bound === 'object') {
+        settings = { ...settings, listen: { address: bound.address, port: bound.port, addresses: [] } }
+      }
       opening = openSolidStorage(settings).then((value) => { opened = value; return value })
     }
     return opening
@@ -345,6 +393,13 @@ export async function createSolidServer (options = {}) {
       }
       settings = { ...settings, baseIri: value }
     },
+    /** Pass what the socket bound, before the first request (issue 1124). */
+    setListen (value) {
+      if (opening !== null) {
+        throw new SolidServerHostError('the storage is already open; set listen first')
+      }
+      settings = { ...settings, listen: value }
+    },
     close () {
       return new Promise((resolve) => {
         server.close(() => {
@@ -380,6 +435,16 @@ function writeHeaders (response, pairs) {
  * @returns {Promise<{server: object, storage: Function, port: number,
  *                    origin: string, close: Function}>}
  */
+/** The addresses of this machine's interfaces. Pure I/O. */
+async function interfaceAddresses () {
+  try {
+    const os = await import('node:os')
+    return Object.values(os.networkInterfaces()).flat().filter(Boolean).map((i) => i.address)
+  } catch (_error) {
+    return []
+  }
+}
+
 export async function listen (options = {}) {
   const port = typeof options.port === 'number' ? options.port : 0
   const host = typeof options.host === 'string' ? options.host : '127.0.0.1'
@@ -391,6 +456,7 @@ export async function listen (options = {}) {
   const bound = created.server.address()
   const origin = `http://${host}:${bound.port}`
   if (typeof options.baseIri !== 'string') created.setBaseIri(`${origin}/`)
+  created.setListen({ host, address: bound.address, port: bound.port, addresses: await interfaceAddresses() })
   try {
     await created.storage()
   } catch (error) {