@factoidal/core 0.2.0 → 0.4.0

package/store-host/index.mjs

@@ -0,0 +1,247 @@
+// The store host: the byte-moving half of the persisted Shardborough store
+// when the engine runs as WebAssembly instead of a native binary.
+// https://github.com/danbri/factoidal/issues/641
+//
+// WHAT THIS FILE IS ALLOWED TO DO
+// Open a file by name, move its bytes, sync, rename, list a directory.
+// Nothing else. It never parses a manifest, checks a digest, decodes a
+// block, or decides which artifact answers a query. Those are format
+// decisions and they live in the Lean source (iron rule 7). A reviewer
+// who finds a magic number, a field offset or a hash in this directory
+// has found a rule violation.
+//
+// WHAT IT CORRESPONDS TO
+// `formal/lean4/Harness/PosixRangeIO.lean` declares three `@[extern]`
+// primitives, realised in `formal/lean4/ffi/block_pread.c`:
+//
+//   l4_block_pread                    -> readRange
+//   l4_delta_log_append_sync_at_size  -> appendSyncAtSize
+//   l4_atomic_replace_file_sync       -> atomicReplace
+//
+// `readWhole`, `openCollection` and `listGeneration` have no extern of
+// their own; the native tools use Lean's own `IO.FS` for them.
+//
+// WHERE THE HOST DIFFERS FROM THE C EXTERNS
+//
+// 1. No advisory lock on the append. `l4_delta_log_append_sync_at_size`
+//    holds `flock(fd, LOCK_EX)` across the size check, the write loop and
+//    the fsync, so exactly one of two concurrent writers appends and the
+//    loser gets `false` and retries. Neither Node nor Deno has `flock` in
+//    its standard library, and this module takes no dependency, so the
+//    sequence here is fstat, then append, then fsync, with no lock. Two
+//    writers that observe the same size therefore both append, and the
+//    delta log gets two batches where the protocol expects one. Use this
+//    function only where a single writer is guaranteed. This is the one
+//    divergence that can corrupt a store; see issue 641 stage 4.
+//
+// 2. Failures raise instead of returning false. Every C extern collapses
+//    an open, write or fsync failure into an empty result or `false`. A
+//    `false` there is therefore ambiguous: it can mean the size did not
+//    match, or that the disk is full. Here `false` from
+//    `appendSyncAtSize` means only that the file size was not the
+//    expected one; an I/O failure throws a `StoreHostError`. A caller
+//    that wants the extern's exact shape catches and maps to false.
+//
+// 3. `readRange` raises on a short read. The extern returns an empty
+//    array on any failure and `Harness.PosixRangeIO.readRange?` turns a
+//    wrong size into `none`. Here a short read throws with code
+//    SHORT_READ, which carries the same refusal and says why.
+//
+// 4. `atomicReplace` returning false does not mean nothing changed. The C
+//    version returns false when the parent-directory fsync fails, after
+//    the rename has already succeeded. This module reports it the same
+//    way: false means the new bytes may be in place but their directory
+//    entry is not known to be durable.
+//
+// 5. Deno has no positional read. `readRange` opens its own handle, seeks
+//    and reads. The handle is private to the call, so there is no cursor
+//    to race, but it is a seek plus a read and not one `pread`.
+//
+// 6. Offsets and lengths are JavaScript numbers. The externs take
+//    `UInt64`. Anything at or above 2^53 is rejected with BAD_ARGUMENT
+//    rather than silently rounded.
+//
+// 7. Directory fsync is not portable. `atomicReplace` opens the parent
+//    directory and fsyncs it, which works on Linux and macOS under both
+//    runtimes. Windows refuses to open a directory as a file; the call
+//    returns false there and the rename still happened (point 4).
+
+import { StoreHostError, requireBytes, requireCount, requirePath } from './errors.mjs'
+import { joinPath, requireChildName } from './paths.mjs'
+
+const isDeno = typeof globalThis.Deno !== 'undefined' &&
+  typeof globalThis.Deno.openSync === 'function'
+
+const impl = isDeno ? await import('./deno.mjs') : await import('./node.mjs')
+
+/** 'node' or 'deno' — which implementation load-time selection chose. */
+export const runtime = impl.runtime
+
+export { StoreHostError }
+
+/**
+ * Read a whole file.
+ * @param {string} path
+ * @returns {Uint8Array} a fresh array of exactly the file's bytes
+ */
+export function readWhole (path) {
+  requirePath(path, 'path')
+  return impl.readWhole(path)
+}
+
+/**
+ * Read exactly `length` bytes starting at `offset`, without touching any
+ * shared file cursor. The counterpart of `l4_block_pread`.
+ * @param {string} path
+ * @param {number} offset
+ * @param {number} length
+ * @returns {Uint8Array} exactly `length` bytes
+ * @throws {StoreHostError} code SHORT_READ when the file has fewer bytes
+ */
+export function readRange (path, offset, length) {
+  requirePath(path, 'path')
+  requireCount(offset, 'offset')
+  requireCount(length, 'length')
+  return impl.readRange(path, offset, length)
+}
+
+/**
+ * Read at most `length` bytes starting at `offset`. A short read is not an
+ * error: fewer bytes than asked for means the file ended there, and zero
+ * bytes means the offset is at or past the end. This is the shape a
+ * streaming reader needs; `readRange` is the exact-length shape the store
+ * uses when the manifest already committed a length.
+ * @param {string} path
+ * @param {number} offset
+ * @param {number} length
+ * @returns {Uint8Array} between 0 and `length` bytes
+ */
+export function readChunk (path, offset, length) {
+  requirePath(path, 'path')
+  requireCount(offset, 'offset')
+  requireCount(length, 'length')
+  return impl.readChunk(path, offset, length)
+}
+
+/**
+ * Create `path`, write `bytes` and fsync. Refuses an existing file with
+ * code FILE_EXISTS: a generation directory is immutable once written, so
+ * a silent overwrite would hide a name collision rather than report it.
+ * @param {string} path
+ * @param {Uint8Array} bytes
+ */
+export function writeNew (path, bytes) {
+  requirePath(path, 'path')
+  impl.writeNew(path, bytes)
+}
+
+/**
+ * Create a directory and every missing parent.
+ * @param {string} path
+ */
+export function makeDirectory (path) {
+  requirePath(path, 'path')
+  impl.makeDirectory(path)
+}
+
+/**
+ * Append `bytes` only if the file currently has exactly `expectedSize`
+ * bytes, then fsync it. The counterpart of
+ * `l4_delta_log_append_sync_at_size`, minus its advisory lock
+ * (divergence 1 above). Creates the file when absent, in which case the
+ * only size that matches is 0.
+ * @param {string} path
+ * @param {Uint8Array} bytes
+ * @param {number} expectedSize
+ * @returns {boolean} true when the append happened and was synced;
+ *   false when the file's size was not `expectedSize`
+ */
+export function appendSyncAtSize (path, bytes, expectedSize) {
+  requirePath(path, 'path')
+  requireBytes(bytes, 'bytes')
+  requireCount(expectedSize, 'expectedSize')
+  return impl.appendSyncAtSize(path, bytes, expectedSize)
+}
+
+/**
+ * Replace a file's whole contents so that a reader sees either the old
+ * bytes or the new bytes and never a mixture: write a temporary file in
+ * the same directory, fsync it, rename it over the target, fsync the
+ * directory. The counterpart of `l4_atomic_replace_file_sync`.
+ * @param {string} path
+ * @param {Uint8Array} bytes
+ * @returns {boolean} true when the replacement is durable; false when the
+ *   rename succeeded but the directory fsync did not (divergence 4)
+ */
+export function atomicReplace (path, bytes) {
+  requirePath(path, 'path')
+  requireBytes(bytes, 'bytes')
+  return impl.atomicReplace(path, bytes)
+}
+
+/**
+ * List the regular files of one directory with their sizes, sorted by
+ * name. Subdirectories and symbolic links to directories are left out.
+ * @param {string} directory
+ * @returns {{name: string, size: number}[]}
+ */
+export function listGeneration (directory) {
+  requirePath(directory, 'directory')
+  return impl.listGeneration(directory)
+}
+
+// The manifest file names a generation directory can carry, in the order
+// `Harness.ShardMerklePread.readManifest` tries them. This module reads
+// whichever exists and returns its bytes untouched; it does not look
+// inside either one.
+const MANIFEST_NAMES = ['manifest.sbm2', 'manifest.sbm1']
+
+/**
+ * Open an activated collection: read `CURRENT`, and return the generation
+ * name it holds together with the raw manifest bytes of that generation.
+ *
+ * The returned `manifest` is bytes. Deciding what those bytes mean —
+ * which wire version, which artifacts, which digests — is the engine's
+ * job, not this module's.
+ *
+ * @param {string} root the collection root that holds CURRENT
+ * @returns {{root: string, generation: string, generationDir: string,
+ *            manifestName: string, manifest: Uint8Array}}
+ * @throws {StoreHostError} code NO_CURRENT when the root has no CURRENT,
+ *   NO_MANIFEST when the generation carries none of the manifest names
+ */
+export function openCollection (root) {
+  requirePath(root, 'root')
+  const pointerPath = joinPath(root, 'CURRENT')
+  let pointerBytes
+  try {
+    pointerBytes = impl.readWhole(pointerPath)
+  } catch (cause) {
+    throw new StoreHostError(
+      'NO_CURRENT',
+      `${root} has no readable CURRENT pointer`,
+      { path: pointerPath, cause }
+    )
+  }
+  // CURRENT holds a UTF-8 child-generation name (spec section 6.4). Trailing
+  // ASCII whitespace is tolerated so a pointer written by hand still opens.
+  const generation = requireChildName(
+    new TextDecoder('utf-8', { fatal: true }).decode(pointerBytes).replace(/[\r\n\t ]+$/, ''),
+    'CURRENT'
+  )
+  const generationDir = joinPath(root, generation)
+  for (const manifestName of MANIFEST_NAMES) {
+    try {
+      const manifest = impl.readWhole(joinPath(generationDir, manifestName))
+      return { root, generation, generationDir, manifestName, manifest }
+    } catch (error) {
+      if (error instanceof StoreHostError && error.code === 'OPEN_FAILED') continue
+      throw error
+    }
+  }
+  throw new StoreHostError(
+    'NO_MANIFEST',
+    `${generationDir} has none of ${MANIFEST_NAMES.join(', ')}`,
+    { path: generationDir }
+  )
+}

package/store-host/node.mjs

@@ -0,0 +1,273 @@
+// Node implementation of the four host primitives the Lean persisted store
+// needs (Harness/PosixRangeIO.lean). See ./index.mjs for the contract and
+// for the places where Node's semantics differ from the C externs.
+//
+// Nothing here parses, verifies or interprets a byte. It opens files,
+// moves bytes, and syncs.
+
+import {
+  closeSync, fstatSync, fsyncSync, mkdirSync, openSync, readSync, readdirSync,
+  renameSync, statSync, unlinkSync, writeSync
+} from 'node:fs'
+
+import { StoreHostError } from './errors.mjs'
+import { baseName, dirName, joinPath } from './paths.mjs'
+
+export const runtime = 'node'
+
+function wrap (code, message, path, cause) {
+  return new StoreHostError(code, `${message}: ${String(cause && cause.message ? cause.message : cause)}`, { path, cause })
+}
+
+function isInterrupt (error) {
+  return error && (error.code === 'EINTR' || error.code === 'EAGAIN')
+}
+
+function openRead (path) {
+  try {
+    return openSync(path, 'r')
+  } catch (cause) {
+    throw wrap('OPEN_FAILED', `cannot open ${path} for reading`, path, cause)
+  }
+}
+
+/** Read `length` bytes at `offset` into `out` at `outOffset`. Returns how many. */
+function preadInto (fd, out, outOffset, length, offset, path) {
+  let done = 0
+  while (done < length) {
+    let read
+    try {
+      read = readSync(fd, out, outOffset + done, length - done, offset + done)
+    } catch (cause) {
+      if (isInterrupt(cause)) continue
+      throw wrap('READ_FAILED', `read failed on ${path}`, path, cause)
+    }
+    if (read === 0) break
+    done += read
+  }
+  return done
+}
+
+export function readWhole (path) {
+  const fd = openRead(path)
+  try {
+    const size = fstatSync(fd).size
+    if (!Number.isSafeInteger(size)) {
+      throw new StoreHostError('FILE_TOO_LARGE', `${path} is larger than 2^53 - 1 bytes`, { path })
+    }
+    const out = new Uint8Array(size)
+    const done = preadInto(fd, out, 0, size, 0, path)
+    if (done !== size) {
+      throw new StoreHostError('SHORT_READ', `${path} shrank during the read (${done} of ${size} bytes)`, { path })
+    }
+    return out
+  } finally {
+    closeSync(fd)
+  }
+}
+
+export function readRange (path, offset, length) {
+  if (length === 0) return new Uint8Array(0)
+  const fd = openRead(path)
+  try {
+    const out = new Uint8Array(length)
+    const done = preadInto(fd, out, 0, length, offset, path)
+    if (done !== length) {
+      throw new StoreHostError(
+        'SHORT_READ',
+        `${path} returned ${done} of ${length} bytes at offset ${offset}`,
+        { path }
+      )
+    }
+    return out
+  } finally {
+    closeSync(fd)
+  }
+}
+
+/**
+ * Read at most `length` bytes at `offset`. Unlike `readRange` a short read
+ * is NOT an error: it is how the reader learns it reached the end of the
+ * file. The packer streams a file it did not stat first, so it needs this
+ * shape rather than the exact-length one.
+ */
+export function readChunk (path, offset, length) {
+  if (length === 0) return new Uint8Array(0)
+  const fd = openRead(path)
+  try {
+    const out = new Uint8Array(length)
+    const done = preadInto(fd, out, 0, length, offset, path)
+    return done === length ? out : out.subarray(0, done)
+  } finally {
+    closeSync(fd)
+  }
+}
+
+/**
+ * Create `path` and write `bytes`. Refuses to replace an existing file:
+ * a generation directory is immutable once written, and a packer that
+ * silently overwrote an artifact would hide a name collision.
+ */
+export function writeNew (path, bytes) {
+  let fd
+  try {
+    // 'wx' is O_WRONLY | O_CREAT | O_EXCL.
+    fd = openSync(path, 'wx')
+  } catch (cause) {
+    if (cause && cause.code === 'EEXIST') {
+      throw new StoreHostError('FILE_EXISTS', `${path} already exists`, { path })
+    }
+    throw wrap('OPEN_FAILED', `cannot create ${path}`, path, cause)
+  }
+  try {
+    let done = 0
+    while (done < bytes.length) {
+      let written
+      try {
+        written = writeSync(fd, bytes, done, bytes.length - done, null)
+      } catch (cause) {
+        if (isInterrupt(cause)) continue
+        throw wrap('WRITE_FAILED', `write failed on ${path}`, path, cause)
+      }
+      done += written
+    }
+    fsyncSync(fd)
+  } finally {
+    closeSync(fd)
+  }
+}
+
+/** Create a directory and every missing parent. */
+export function makeDirectory (path) {
+  try {
+    mkdirSync(path, { recursive: true })
+  } catch (cause) {
+    throw wrap('MKDIR_FAILED', `cannot create ${path}`, path, cause)
+  }
+}
+
+export function appendSyncAtSize (path, bytes, expectedSize) {
+  let fd
+  try {
+    // 'a' is O_WRONLY | O_CREAT | O_APPEND, matching the C extern's open.
+    fd = openSync(path, 'a')
+  } catch (cause) {
+    throw wrap('OPEN_FAILED', `cannot open ${path} for append`, path, cause)
+  }
+  try {
+    const size = fstatSync(fd).size
+    if (size !== expectedSize) return false
+    let done = 0
+    while (done < bytes.length) {
+      let written
+      try {
+        // position null keeps the O_APPEND placement the C extern relies on.
+        written = writeSync(fd, bytes, done, bytes.length - done, null)
+      } catch (cause) {
+        if (isInterrupt(cause)) continue
+        throw wrap('WRITE_FAILED', `append failed on ${path}`, path, cause)
+      }
+      done += written
+    }
+    try {
+      fsyncSync(fd)
+    } catch (cause) {
+      throw wrap('FSYNC_FAILED', `fsync failed on ${path}`, path, cause)
+    }
+    return true
+  } finally {
+    closeSync(fd)
+  }
+}
+
+function temporaryName (path) {
+  const suffix = Math.floor(Math.random() * 0xffffff).toString(16).padStart(6, '0')
+  return joinPath(dirName(path), baseName(path) + '.tmp.' + suffix)
+}
+
+function fsyncDirectory (directory) {
+  let fd
+  try {
+    fd = openSync(directory, 'r')
+  } catch (cause) {
+    throw wrap('DIR_OPEN_FAILED', `cannot open ${directory} to sync it`, directory, cause)
+  }
+  try {
+    fsyncSync(fd)
+  } finally {
+    closeSync(fd)
+  }
+}
+
+export function atomicReplace (path, bytes) {
+  const directory = dirName(path)
+  let temporary = null
+  let fd = null
+  try {
+    for (let attempt = 0; attempt < 8 && fd === null; attempt += 1) {
+      temporary = temporaryName(path)
+      try {
+        // 'wx' is O_WRONLY | O_CREAT | O_EXCL, the exclusive create that
+        // makes the name ours the way mkstemp does in the C extern.
+        fd = openSync(temporary, 'wx')
+      } catch (cause) {
+        if (cause && cause.code === 'EEXIST') continue
+        throw wrap('OPEN_FAILED', `cannot create ${temporary}`, temporary, cause)
+      }
+    }
+    if (fd === null) {
+      throw new StoreHostError('TEMP_NAME_EXHAUSTED', `no free temporary name beside ${path}`, { path })
+    }
+    let done = 0
+    while (done < bytes.length) {
+      let written
+      try {
+        written = writeSync(fd, bytes, done, bytes.length - done, null)
+      } catch (cause) {
+        if (isInterrupt(cause)) continue
+        throw wrap('WRITE_FAILED', `write failed on ${temporary}`, temporary, cause)
+      }
+      done += written
+    }
+    fsyncSync(fd)
+    closeSync(fd)
+    fd = null
+    renameSync(temporary, path)
+    temporary = null
+    try {
+      fsyncDirectory(directory)
+    } catch (_error) {
+      // The C extern also returns false here, with the replacement already
+      // done. Reported the same way; see ./index.mjs for what false means.
+      return false
+    }
+    return true
+  } catch (error) {
+    if (fd !== null) closeSync(fd)
+    if (temporary !== null) {
+      try { unlinkSync(temporary) } catch (_ignored) { /* the temporary may not exist */ }
+    }
+    throw error
+  }
+}
+
+export function listGeneration (directory) {
+  let names
+  try {
+    names = readdirSync(directory)
+  } catch (cause) {
+    throw wrap('DIR_READ_FAILED', `cannot list ${directory}`, directory, cause)
+  }
+  const out = []
+  for (const name of names.sort()) {
+    let info
+    try {
+      info = statSync(joinPath(directory, name))
+    } catch (_cause) {
+      continue // a name that vanished between readdir and stat
+    }
+    if (!info.isFile()) continue
+    out.push({ name, size: info.size })
+  }
+  return out
+}

package/store-host/paths.mjs

@@ -0,0 +1,77 @@
+// Path plumbing for the store host. No dependency on `node:path`, so the
+// same file loads under Deno without the Node compatibility layer.
+//
+// These functions join and split path strings. They make no decision about
+// what a Shardborough generation contains.
+
+import { StoreHostError } from './errors.mjs'
+
+const SEPARATORS = ['/', '\\']
+
+function isSeparator (character) {
+  return SEPARATORS.indexOf(character) >= 0
+}
+
+/** Join with '/'. Every runtime this module targets accepts '/'. */
+export function joinPath (base, child) {
+  if (base.length === 0) return child
+  const last = base[base.length - 1]
+  return isSeparator(last) ? base + child : base + '/' + child
+}
+
+/** The directory part of a path, or '.' when the path has no separator. */
+export function dirName (path) {
+  let end = path.length
+  while (end > 1 && isSeparator(path[end - 1])) end -= 1
+  let index = end - 1
+  while (index >= 0 && !isSeparator(path[index])) index -= 1
+  if (index < 0) return '.'
+  if (index === 0) return path[0]
+  return path.slice(0, index)
+}
+
+/** The final component of a path. */
+export function baseName (path) {
+  let end = path.length
+  while (end > 1 && isSeparator(path[end - 1])) end -= 1
+  let index = end - 1
+  while (index >= 0 && !isSeparator(path[index])) index -= 1
+  return path.slice(index + 1, end)
+}
+
+/**
+ * A single child name that may be appended to a directory path.
+ *
+ * `CURRENT` holds a generation name written by the Lean activation path;
+ * this module still refuses a value that would leave the collection root
+ * when joined. That is a filesystem-safety guard on a string the host is
+ * about to turn into a path, not a check of the pointer's format.
+ */
+export function requireChildName (name, label) {
+  if (typeof name !== 'string' || name.length === 0) {
+    throw new StoreHostError('BAD_CHILD_NAME', `${label} is empty`)
+  }
+  if (name === '.' || name === '..') {
+    throw new StoreHostError('BAD_CHILD_NAME', `${label} is "${name}"`)
+  }
+  for (const character of name) {
+    if (isSeparator(character) || character === '\u0000') {
+      throw new StoreHostError(
+        'BAD_CHILD_NAME',
+        `${label} contains a path separator or NUL byte`
+      )
+    }
+  }
+  return name
+}
+
+/** Convert a `file:` URL string to a filesystem path. */
+export function fileUrlToPath (url) {
+  const text = typeof url === 'string' ? url : String(url)
+  if (!text.startsWith('file://')) return text
+  let path = decodeURIComponent(text.slice('file://'.length))
+  const host = path.indexOf('/')
+  if (host > 0) path = path.slice(host)
+  if (/^\/[A-Za-z]:/.test(path)) path = path.slice(1)
+  return path
+}