@factoidal/core 0.29.0 → 0.31.0

package/lws/server.mjs

@@ -250,8 +250,9 @@ function contentBytes (request) {
  * @param {boolean} [options.realClock] stamp each request with this host's clock
  * @param {string|null} [options.storageKey] the storage key text (64 hex digits or base64url);
  *   default the environment variable FACTOIDAL_LWS_KEY; `null` reads no key
- * @param {string} [options.storage] where DIR keeps the storage: "snapshot" (default, one
- *   file) or "files" (one file per body and a journal, issue 918); passed unread
+ * @param {string} [options.storage] where DIR keeps the storage: "files" (default, one
+ *   file per body and a journal, issue 918) or "snapshot" (one file written whole after
+ *   each change); passed unread
  * @param {boolean|string[]} [options.cors] CORS (issue 923): false for none, a list of
  *   origins to answer; default: echo any valid request Origin. Passed unread
  * @param {string} [options.lockCommand] the command named in the lock file of DIR
@@ -270,12 +271,20 @@ export async function openLwsStorage (options = {}) {
   // The pagination threshold and page size of container listings; the
   // engine decides what it means (L4Factoidal/LWS/Pagination.lean).
   if (typeof options.pageSize === 'number') config.pageSize = options.pageSize
+  // The largest content of one write; the engine answers 413 above it
+  // (L4Factoidal/LWS/Operations.lean, step_tooLarge; issue 1240).
+  if (typeof options.maxBody === 'number') config.maxBody = options.maxBody
   // Authorization (L4Factoidal/LWS/Authorization.lean). `auth` is passed
   // through unread; `insecureNoAuth` is the explicit development mode.
   // With neither, the engine enforces authorization with no trusted
   // issuer.
   if (options.auth !== undefined && options.auth !== null) config.auth = options.auth
   if (options.insecureNoAuth === true) config.insecureNoAuth = true
+  // A component principal (issue 1238): a storage a component opens
+  // in-process. No token is read; every request is this agent's, and WAC
+  // decides it (Authorization.decide_principal_agents). Never served over
+  // HTTP: createLwsServer refuses it.
+  if (typeof options.principal === 'string') config.principal = options.principal
   if (typeof options.rootAcl === 'string') config.rootAcl = options.rootAcl
   // Notifications are on unless the caller turns them off; the engine
   // decides everything else about them.
@@ -339,10 +348,13 @@ export async function openLwsStorage (options = {}) {
   // With "files" the engine names the files to read (`load`) and gives the
   // plan of every write (`lwsPersist`), and this host reads no snapshot.
   // With a DIR and no `storage` option, the environment variable
-  // FACTOIDAL_LWS_STORAGE is the value; without a DIR there is nothing to
+  // FACTOIDAL_LWS_STORAGE is the value, and without it "files" (2026-11-03):
+  // a write then costs what it changes, not the size of the storage. A DIR
+  // that holds a snapshot is migrated at the first start (lws-state.json is
+  // renamed lws-state.json.migrated). Without a DIR there is nothing to
   // keep, and only an explicit option is passed.
   const dir = typeof options.dir === 'string' ? options.dir : null
-  const storage = dir !== null ? storageModeText(options.storage) : options.storage
+  const storage = dir !== null ? (storageModeText(options.storage) ?? 'files') : options.storage
   if (typeof storage === 'string') config.storage = storage
   // One writing process per storage directory
   // (https://github.com/danbri/factoidal/issues/1135): the lock is taken
@@ -589,13 +601,37 @@ export async function openLwsStorage (options = {}) {
   }
 }
 
-/** Read a Node request body as its bytes. */
-function readBody (request) {
+/** The engine's default `maxBody` (L4Factoidal/LWS/Operations.lean). */
+export const DEFAULT_MAX_BODY = 16 * 1024 * 1024
+
+/**
+ * Read a Node request body as its bytes, at most `limit + 1` of them. Past
+ * the limit it keeps the first `limit + 1` bytes, stops reading and says so
+ * (`over`), so an oversized upload never reaches memory whole (issue 1240).
+ * The engine decides the answer: content over its `maxBody` is refused with
+ * 413 and changes nothing (`step_tooLarge`, L4Factoidal/LWS/Operations.lean).
+ */
+function readBody (request, limit = DEFAULT_MAX_BODY) {
   return new Promise((resolve, reject) => {
     const chunks = []
-    request.on('data', (chunk) => chunks.push(chunk))
+    let size = 0
+    let over = false
+    const done = () => resolve({ bytes: new Uint8Array(Buffer.concat(chunks)), over })
+    request.on('data', (chunk) => {
+      if (over) return
+      if (size + chunk.length > limit) {
+        chunks.push(chunk.subarray(0, limit + 1 - size))
+        size = limit + 1
+        over = true
+        done()
+        request.resume()
+        return
+      }
+      size += chunk.length
+      chunks.push(chunk)
+    })
     request.on('error', reject)
-    request.on('end', () => resolve(new Uint8Array(Buffer.concat(chunks))))
+    request.on('end', () => { if (!over) done() })
   })
 }
 
@@ -623,6 +659,11 @@ function headerPairs (raw) {
  *                    setBaseIri: Function}>}
  */
 export async function createLwsServer (options = {}) {
+  if (options.principal !== undefined) {
+    // Served over HTTP, a principal storage would let every client act as
+    // the component. It is for in-process use only (issue 1238).
+    throw new LwsHostError('a storage with a component principal is in-process only; it is not served over HTTP')
+  }
   const http = await import('node:http')
   const onError = typeof options.onError === 'function'
     ? options.onError
@@ -675,8 +716,13 @@ export async function createLwsServer (options = {}) {
         method: request.method,
         target: request.url,
         headers: headerPairs(request.rawHeaders),
-        body: await readBody(request)
+        body: new Uint8Array(0)
       }
+      const read = await readBody(request, typeof settings.maxBody === 'number' ? settings.maxBody : DEFAULT_MAX_BODY)
+      record.body = read.bytes
+      // The rest of an oversized body is not read: the connection closes
+      // after the answer. The answer is the engine's.
+      if (read.over) response.shouldKeepAlive = false
     } catch (error) {
       onError(error)
       response.writeHead(500, { 'content-type': 'text/plain' })

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@factoidal/core",
-  "version": "0.29.0",
-  "description": "EXPERIMENTAL UNREVIEWED CODE: do not run public services on it without careful independent expert review. RDF parsing and writing (Turtle, TriG, N-Triples, N-Quads, RDF/XML, JSON-LD), SPARQL 1.1 and 1.2, RDFS and OWL reasoning, SHACL and ShEx validation, CSVW and RML mapping, RDFC-1.0, and SPARQL, LWS and Solid servers, for Node, Deno and browsers.",
+  "version": "0.31.0",
+  "description": "EXPERIMENTAL UNDER-REVIEWED CODE: do not run public services on it without careful independent expert review. RDF parsing and writing (Turtle, TriG, N-Triples, N-Quads, RDF/XML, JSON-LD), SPARQL 1.1 and 1.2, RDFS and OWL reasoning, SHACL and ShEx validation, CSVW and RML mapping, RDFC-1.0, and SPARQL, LWS and Solid servers, for Node, Deno and browsers.",
   "keywords": [
     "sparql",
     "rdf",
@@ -70,6 +70,7 @@
     "./store-host/paths": "./store-host/paths.mjs",
     "./xmpp": "./xmpp/index.mjs",
     "./xmpp/server": "./xmpp/server.mjs",
+    "./xmpp/inpage": "./xmpp/inpage.mjs",
     "./cipher-host": "./lib/cipher-host.mjs",
     "./store": {
       "default": "./bin/store.mjs"
@@ -92,9 +93,15 @@
     "./webdav": {
       "default": "./webdav/server.mjs"
     },
+    "./lws/inpage": {
+      "default": "./lws/inpage.mjs"
+    },
     "./lws/client": {
       "default": "./lws/client.mjs"
     },
+    "./lws/dataset": {
+      "default": "./lws/dataset.mjs"
+    },
     "./authz": {
       "default": "./authz/server.mjs"
     },

package/xmpp/README.md

@@ -65,8 +65,12 @@ against an older committed bundle instead of getting a confusing
 mkdir -p state && printf 'juliet:r0m30\nromeo:juli3t\n' > state/accounts
 npx factoidal xmpp-serve state --domain localhost            # plaintext on 5222
 npx factoidal xmpp-serve state --cert fullchain.pem --key privkey.pem   # XEP-0368 direct TLS on 5223
+npx factoidal xmpp-serve state --cert fullchain.pem --key privkey.pem --starttls   # STARTTLS required on 5222
 ```
 
+Without `--cert` and `--key` the listener has no TLS and offers PLAIN, so
+the command binds it to a loopback address only.
+
 ```js
 import { listen } from '@factoidal/core/xmpp/server'
 const running = await listen({ dir: 'state', domain: 'localhost', port: 0 })
@@ -90,6 +94,7 @@ process per connection behind socat) serves; the wasm session ops are
 | `xmppSessionFeed` | `[session, text]` | `{ ok, send: [...], route: [{to, text}], presence: [...], rostersSaved: [...], close, stage, bare, full, bufferedBytes }` |
 | `xmppSessionDeliver` | `[session, text]` | `{ ok, delivered, send }` — `delivered:false` for an unbound session; nothing is lost |
 | `xmppSessionClose` | `[session]` | `{ ok, send }` |
+| `xmppSessionTlsUp` | `[session]` | `{ ok, stage }` — after a feed answered `startTls: true` and the host finished the TLS handshake; refused otherwise |
 | `xmppRosterGet` / `xmppRosterSet` | `[server, bare]` / `[server, bare, encoded]` | the roster in the engine's own on-disk encoding |
 | `xmppClose` | `[server]` | `{ ok, sessions }` |
 
@@ -120,7 +125,10 @@ What it speaks: RFC 6120 stream negotiation, stream errors, SASL PLAIN
 and SCRAM-SHA-256, resource binding; RFC 6121 roster, presence
 subscription and broadcast, message delivery with the server-stamped
 `from`; XEP-0030 `disco#info`; XEP-0368 direct TLS with `--cert`/`--key`.
-Not: STARTTLS, server-to-server federation, MUC/MIX/GC3 rooms, PRECIS.
+RFC 6120 section 5 STARTTLS with `startTls: true` (op `xmppSessionTlsUp`;
+the engine decides, the host does the handshake; see the theorems in
+`formal/lean4/L4Factoidal/XMPP/Server.lean`).
+Not: server-to-server federation, MUC/MIX/GC3 rooms, PRECIS.
 Known limitation shared with the native harness: one SCRAM salt per
 server (`saltB64` in the config), not one per account.
 

package/xmpp/inpage.mjs

@@ -0,0 +1,197 @@
+// factoidal/xmpp/inpage.mjs — the Lean XMPP server with no network: it
+// runs in the page (or a worker, or Node) beside the code that talks to it,
+// and each connection is a pair of in-order message channels.
+//
+//   import { loadEngine } from '@factoidal/core/browser'
+//   import { openInPageXmppServer, inPageWebSocket } from '@factoidal/core/xmpp/inpage'
+//   const server = await openInPageXmppServer({ engine: await loadEngine(),
+//     domain: 'foafos.local', mucDomain: 'rooms.foafos.local',
+//     accounts: [{ username: 'shell', password }] })
+//   const link = server.connect()            // { send(message), onmessage, close() }
+//   const WS = inPageWebSocket(server)       // a WebSocket-shaped class for
+//   new WS('xmpp-inpage:', 'xmpp')           //   Strophe.js and other RFC 7395 clients
+//
+// A message is one RFC 7395 message (https://www.rfc-editor.org/rfc/rfc7395):
+// `<open/>` and `<close/>` in the framing namespace, then one complete
+// element each. The engine's message carrier (`xmppMsgOpen`, `xmppMsgFeed`,
+// `xmppMsgDeliver`, `xmppMsgClose`; Wasm/Ops/XmppSession.lean) makes every
+// protocol decision; this file only moves messages and performs the routes
+// (router.mjs, shared with the Node host). No Node or browser API beyond
+// `crypto.getRandomValues` and `queueMicrotask`.
+//
+// There is no socket, so there is no Origin to check: whoever holds the
+// server object can connect, and the accounts' passwords are what the
+// engine checks (SCRAM-SHA-256). The host page decides which code gets it.
+
+import { createXmppRouter } from './router.mjs'
+
+/** Unpredictable values for the stream id and the SCRAM server nonce
+ *  (RFC 6120 section 4.7.3, RFC 5802 section 5.1); the engine asks the host. */
+function randomToken (bytes = 18) {
+  const b = new Uint8Array(bytes)
+  globalThis.crypto.getRandomValues(b)
+  let s = ''
+  for (const x of b) s += String.fromCharCode(x)
+  return btoa(s).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '')
+}
+
+/**
+ * @param {object} options
+ * @param {object} options.engine a loaded engine handle (`loadEngine()`)
+ * @param {string} [options.domain] default `localhost`
+ * @param {Array<{username: string, password: string}>} [options.accounts]
+ * @param {string} [options.mucDomain] XEP-0045 rooms (L4Factoidal/XMPP/Muc.lean)
+ * @param {string} [options.mixDomain] XEP-0369 channels (L4Factoidal/XMPP/Mix.lean)
+ * @param {string} [options.gc3Domain] GC3 prototype groups (L4Factoidal/XMPP/Gc3.lean)
+ * @param {Function} [options.onError]
+ */
+export async function openInPageXmppServer (options = {}) {
+  const engine = options.engine
+  if (!engine || typeof engine.call !== 'function') throw new Error('openInPageXmppServer: options.engine is required')
+  const onError = typeof options.onError === 'function' ? options.onError : (e) => console.error(`xmpp-inpage: ${e.message}`)
+  const config = {}
+  for (const k of ['domain', 'mucDomain', 'mixDomain', 'gc3Domain', 'saltB64']) {
+    if (typeof options[k] === 'string') config[k] = options[k]
+  }
+  if (Array.isArray(options.accounts)) config.accounts = options.accounts
+  if (typeof options.iterationCount === 'number') config.iterationCount = options.iterationCount
+  const opened = engine.call('xmppOpen', [JSON.stringify(config)])
+  if (typeof opened.handle !== 'string') throw new Error('xmppOpen answered with no handle')
+  const handle = opened.handle
+  const router = createXmppRouter({ onError })
+  let closed = false
+
+  function connect () {
+    if (closed) throw new Error('the in-page XMPP server is closed')
+    const got = engine.call('xmppMsgOpen', [handle, JSON.stringify({ streamId: randomToken(), nonce: randomToken() })])
+    if (typeof got.handle !== 'string') throw new Error('xmppMsgOpen answered with no handle (an engine without the message carrier?)')
+    const session = got.handle
+    let open = true
+    // Messages to the client go out in order, never inside an engine call.
+    const emit = (msgs) => {
+      for (const m of msgs) {
+        queueMicrotask(() => {
+          if (typeof link.onmessage === 'function') {
+            try { link.onmessage(m) } catch (error) { onError(error) }
+          }
+        })
+      }
+    }
+    const end = () => {
+      if (!open) return
+      open = false
+      router.connections.delete(c)
+      let answer
+      try { answer = engine.call('xmppMsgClose', [session]) } catch (error) { onError(error); answer = { send: [], routeFull: [] } }
+      for (const r of answer.routeFull || []) router.routeFull(null, r.to, r.text, null)
+      emit(answer.send || [])
+      queueMicrotask(() => { if (typeof link.onclose === 'function') link.onclose() })
+    }
+    const c = {
+      bare: null,
+      full: null,
+      deliver (text) {
+        if (!open) return false
+        const answer = engine.call('xmppMsgDeliver', [session, text])
+        if (answer.delivered === true) { emit(answer.send); return true }
+        return false
+      }
+    }
+    router.connections.add(c)
+    const link = {
+      onmessage: null,
+      onclose: null,
+      get open () { return open },
+      /** One whole RFC 7395 message from the client. */
+      send (message) {
+        if (!open) throw new Error('the in-page XMPP connection is closed')
+        let answer
+        try { answer = engine.call('xmppMsgFeed', [session, String(message)]) } catch (error) { onError(error); end(); return }
+        emit(answer.send || [])
+        router.apply(c, answer)
+        if (answer.close === true) end()
+      },
+      close: end
+    }
+    return link
+  }
+
+  return {
+    engine,
+    handle,
+    domain: typeof opened.domain === 'string' ? opened.domain : (config.domain || 'localhost'),
+    mucDomain: config.mucDomain || null,
+    connect,
+    /** The live connections' `{bare, full}` pairs. */
+    sessions () { return Array.from(router.connections, (c) => ({ bare: c.bare, full: c.full })) },
+    close () {
+      if (closed) return
+      closed = true
+      for (const c of Array.from(router.connections)) router.connections.delete(c)
+      try { engine.call('xmppClose', [handle]) } catch (_error) { /* already gone */ }
+    }
+  }
+}
+
+/**
+ * A WebSocket-shaped class whose instances are connections to `server`, so
+ * an RFC 7395 client written for `WebSocket` (Strophe.js, the foafos XmppWs
+ * client) runs unchanged. The URL is not used; the subprotocol is `xmpp`.
+ */
+export function inPageWebSocket (server) {
+  return class InPageWebSocket {
+    static CONNECTING = 0
+    static OPEN = 1
+    static CLOSING = 2
+    static CLOSED = 3
+    constructor (url, protocols) {
+      this.url = String(url)
+      this.protocol = 'xmpp'
+      this.readyState = 0
+      this.binaryType = 'blob'
+      this.bufferedAmount = 0
+      this.extensions = ''
+      this.onopen = this.onmessage = this.onclose = this.onerror = null
+      this._listeners = { open: [], message: [], close: [], error: [] }
+      const wanted = Array.isArray(protocols) ? protocols : (protocols === undefined ? [] : [protocols])
+      queueMicrotask(() => {
+        if (wanted.length > 0 && !wanted.includes('xmpp')) { this._fail(); return }
+        try { this._link = server.connect() } catch (_error) { this._fail(); return }
+        this._link.onmessage = (data) => this._fire('message', { type: 'message', data, target: this })
+        this._link.onclose = () => this._closed(1000)
+        this.readyState = 1
+        this._fire('open', { type: 'open', target: this })
+      })
+    }
+    addEventListener (type, fn) { this._listeners[type]?.push(fn) }
+    removeEventListener (type, fn) {
+      const l = this._listeners[type]
+      if (l) { const i = l.indexOf(fn); if (i >= 0) l.splice(i, 1) }
+    }
+    dispatchEvent (event) { this._fire(event.type, event); return true }
+    send (data) {
+      if (this.readyState !== 1) throw new Error('InvalidStateError: the connection is not open')
+      this._link.send(String(data))
+    }
+    close (code = 1000) {
+      if (this.readyState >= 2) return
+      this.readyState = 2
+      this._link?.close()
+      if (!this._link) this._closed(code)
+    }
+    _fail () {
+      this._fire('error', { type: 'error', target: this })
+      this._closed(1006)
+    }
+    _closed (code) {
+      if (this.readyState === 3) return
+      this.readyState = 3
+      this._fire('close', { type: 'close', code, reason: '', wasClean: code === 1000, target: this })
+    }
+    _fire (type, event) {
+      const prop = this['on' + type]
+      if (typeof prop === 'function') prop.call(this, event)
+      for (const fn of [...(this._listeners[type] || [])]) fn.call(this, event)
+    }
+  }
+}

package/xmpp/router.mjs

@@ -0,0 +1,96 @@
+// factoidal/xmpp/router.mjs — the routing table of an XMPP host, with no
+// transport: the Node host (`server.mjs`, TCP, TLS, WebSocket) and the
+// in-page host (`inpage.mjs`, whole messages over a MessagePort or a
+// function call) share it. No Node or browser API is used here.
+//
+// A connection is any object with `bare`, `full` (the last values the
+// engine reported, or null) and `deliver(text) → boolean` (hand it a stanza
+// another session routed to it; false when its session refused). The engine
+// decides where every stanza goes (`route`, `routeFull` of a feed answer);
+// this file only finds the connections and calls `deliver`.
+
+/**
+ * @param {object} [options]
+ * @param {Function} [options.onError]
+ * @param {number} [options.pendingLimit] stanzas kept per bare JID for the
+ *   race where a session closed between the engine's decision and the
+ *   delivery (default 100)
+ */
+export function createXmppRouter (options = {}) {
+  const onError = typeof options.onError === 'function' ? options.onError : () => {}
+  const pendingLimit = typeof options.pendingLimit === 'number' ? options.pendingLimit : 100
+  /** Every live connection. */
+  const connections = new Set()
+  /** bare JID → stanzas routed to it while no session was bound. */
+  const pending = new Map()
+
+  function boundTo (bare) {
+    const found = []
+    for (const c of connections) if (c.full !== null && c.bare === bare) found.push(c)
+    return found
+  }
+
+  function hold (bare, text) {
+    if (!pending.has(bare)) pending.set(bare, [])
+    const q = pending.get(bare)
+    if (q.length < pendingLimit) q.push(text)
+    else onError(new Error(`dropped a stanza for ${bare}: the host fallback queue is full`))
+  }
+
+  /**
+   * Deliver one stanza to every session bound to `bare`. The engine holds
+   * stanzas for accounts with no bound session (L4Factoidal/XMPP/Held.lean,
+   * bounded; issue 1235), so a route reaches here only for a bound account.
+   * `pending` covers the race where that session closed between the
+   * engine's decision and this delivery; it is bounded too.
+   */
+  function routeStanza (bare, text) {
+    const targets = boundTo(bare)
+    if (targets.length === 0) { hold(bare, text); return }
+    for (const t of targets) {
+      let delivered = false
+      try { delivered = t.deliver(text) } catch (error) { onError(error); continue }
+      if (!delivered) hold(bare, text)
+    }
+  }
+
+  /**
+   * `Out.routeFull`: deliver to the ONE session bound to `full`, now. With no
+   * such session (or one that refuses), deliver the engine's bounce, if it
+   * gave one, back to the sending session. Never held (RFC 6120 section
+   * 10.5.3.2; https://github.com/danbri/factoidal/issues/1234).
+   */
+  function routeFull (sender, full, text, bounce) {
+    for (const t of connections) {
+      if (t.full !== full) continue
+      try { if (t.deliver(text)) return } catch (error) { onError(error) }
+    }
+    if (typeof bounce === 'string' && sender !== null) {
+      try { sender.deliver(bounce) } catch (error) { onError(error) }
+    }
+  }
+
+  /** A session just bound: hand it what was waiting. */
+  function flushPending (c) {
+    const queue = pending.get(c.bare)
+    if (!queue || queue.length === 0) return
+    pending.delete(c.bare)
+    for (const text of queue) routeStanza(c.bare, text)
+  }
+
+  /**
+   * The routing part of one feed answer: the session's JIDs as the engine
+   * reports them, then `route` and `routeFull`, then what waited for a
+   * session that has just bound.
+   */
+  function apply (c, answer) {
+    const wasBound = c.full !== null
+    c.bare = typeof answer.bare === 'string' ? answer.bare : null
+    c.full = typeof answer.full === 'string' ? answer.full : null
+    for (const r of answer.route || []) routeStanza(r.to, r.text)
+    for (const r of answer.routeFull || []) routeFull(c, r.to, r.text, r.bounce)
+    if (!wasBound && c.full !== null) flushPending(c)
+  }
+
+  return { connections, pending, boundTo, routeStanza, routeFull, flushPending, apply }
+}