@factoidal/core 0.1.0 → 0.3.0

package/store-host/deno.mjs

@@ -0,0 +1,199 @@
+// Deno implementation of the same four host primitives. Uses only Deno
+// globals, so it never loads Deno's Node compatibility layer.
+//
+// Deno 2 removed the resource-id fsync entry points (`Deno.fsyncSync`);
+// the file-handle method `FsFile.syncSync()` is the one that exists.
+
+import { StoreHostError } from './errors.mjs'
+import { baseName, dirName, joinPath } from './paths.mjs'
+
+export const runtime = 'deno'
+
+function wrap (code, message, path, cause) {
+  return new StoreHostError(code, `${message}: ${String(cause && cause.message ? cause.message : cause)}`, { path, cause })
+}
+
+function openRead (path) {
+  try {
+    return Deno.openSync(path, { read: true })
+  } catch (cause) {
+    throw wrap('OPEN_FAILED', `cannot open ${path} for reading`, path, cause)
+  }
+}
+
+/**
+ * Deno has no positional read. Each call here opens its own handle, so the
+ * seek belongs to that handle alone and no other reader can move it. That
+ * is what `pread` buys the C extern; it is not the same instruction.
+ */
+function readInto (file, out, length, offset, path) {
+  if (offset > 0) file.seekSync(offset, Deno.SeekMode.Start)
+  let done = 0
+  while (done < length) {
+    let read
+    try {
+      read = file.readSync(out.subarray(done, length))
+    } catch (cause) {
+      throw wrap('READ_FAILED', `read failed on ${path}`, path, cause)
+    }
+    if (read === null || read === 0) break
+    done += read
+  }
+  return done
+}
+
+export function readWhole (path) {
+  const file = openRead(path)
+  try {
+    const size = file.statSync().size
+    if (!Number.isSafeInteger(size)) {
+      throw new StoreHostError('FILE_TOO_LARGE', `${path} is larger than 2^53 - 1 bytes`, { path })
+    }
+    const out = new Uint8Array(size)
+    const done = readInto(file, out, size, 0, path)
+    if (done !== size) {
+      throw new StoreHostError('SHORT_READ', `${path} shrank during the read (${done} of ${size} bytes)`, { path })
+    }
+    return out
+  } finally {
+    file.close()
+  }
+}
+
+export function readRange (path, offset, length) {
+  if (length === 0) return new Uint8Array(0)
+  const file = openRead(path)
+  try {
+    const out = new Uint8Array(length)
+    const done = readInto(file, out, length, offset, path)
+    if (done !== length) {
+      throw new StoreHostError(
+        'SHORT_READ',
+        `${path} returned ${done} of ${length} bytes at offset ${offset}`,
+        { path }
+      )
+    }
+    return out
+  } finally {
+    file.close()
+  }
+}
+
+export function appendSyncAtSize (path, bytes, expectedSize) {
+  let file
+  try {
+    file = Deno.openSync(path, { write: true, create: true, append: true })
+  } catch (cause) {
+    throw wrap('OPEN_FAILED', `cannot open ${path} for append`, path, cause)
+  }
+  try {
+    const size = file.statSync().size
+    if (size !== expectedSize) return false
+    let done = 0
+    while (done < bytes.length) {
+      let written
+      try {
+        written = file.writeSync(bytes.subarray(done))
+      } catch (cause) {
+        throw wrap('WRITE_FAILED', `append failed on ${path}`, path, cause)
+      }
+      done += written
+    }
+    try {
+      file.syncSync()
+    } catch (cause) {
+      throw wrap('FSYNC_FAILED', `fsync failed on ${path}`, path, cause)
+    }
+    return true
+  } finally {
+    file.close()
+  }
+}
+
+function temporaryName (path) {
+  const suffix = Math.floor(Math.random() * 0xffffff).toString(16).padStart(6, '0')
+  return joinPath(dirName(path), baseName(path) + '.tmp.' + suffix)
+}
+
+function fsyncDirectory (directory) {
+  let handle
+  try {
+    handle = Deno.openSync(directory, { read: true })
+  } catch (cause) {
+    throw wrap('DIR_OPEN_FAILED', `cannot open ${directory} to sync it`, directory, cause)
+  }
+  try {
+    handle.syncSync()
+  } finally {
+    handle.close()
+  }
+}
+
+export function atomicReplace (path, bytes) {
+  const directory = dirName(path)
+  let temporary = null
+  let file = null
+  try {
+    for (let attempt = 0; attempt < 8 && file === null; attempt += 1) {
+      temporary = temporaryName(path)
+      try {
+        file = Deno.openSync(temporary, { write: true, createNew: true })
+      } catch (cause) {
+        if (cause instanceof Deno.errors.AlreadyExists) continue
+        throw wrap('OPEN_FAILED', `cannot create ${temporary}`, temporary, cause)
+      }
+    }
+    if (file === null) {
+      throw new StoreHostError('TEMP_NAME_EXHAUSTED', `no free temporary name beside ${path}`, { path })
+    }
+    let done = 0
+    while (done < bytes.length) {
+      let written
+      try {
+        written = file.writeSync(bytes.subarray(done))
+      } catch (cause) {
+        throw wrap('WRITE_FAILED', `write failed on ${temporary}`, temporary, cause)
+      }
+      done += written
+    }
+    file.syncSync()
+    file.close()
+    file = null
+    Deno.renameSync(temporary, path)
+    temporary = null
+    try {
+      fsyncDirectory(directory)
+    } catch (_error) {
+      return false
+    }
+    return true
+  } catch (error) {
+    if (file !== null) file.close()
+    if (temporary !== null) {
+      try { Deno.removeSync(temporary) } catch (_ignored) { /* the temporary may not exist */ }
+    }
+    throw error
+  }
+}
+
+export function listGeneration (directory) {
+  const out = []
+  let entries
+  try {
+    entries = [...Deno.readDirSync(directory)]
+  } catch (cause) {
+    throw wrap('DIR_READ_FAILED', `cannot list ${directory}`, directory, cause)
+  }
+  entries.sort((left, right) => (left.name < right.name ? -1 : left.name > right.name ? 1 : 0))
+  for (const entry of entries) {
+    if (!entry.isFile) continue
+    let info
+    try {
+      info = Deno.statSync(joinPath(directory, entry.name))
+    } catch (_cause) {
+      continue
+    }
+    out.push({ name: entry.name, size: info.size })
+  }
+  return out
+}

package/store-host/errors.mjs

@@ -0,0 +1,55 @@
+// Error type shared by the Node and Deno host-I/O implementations.
+//
+// The store host moves bytes. It never reports a FORMAT decision, so no
+// error code here names a manifest, a digest, a block or a generation
+// layout: those decisions belong to the Lean source
+// (https://github.com/danbri/factoidal/issues/641, iron rule 7).
+
+export class StoreHostError extends Error {
+  /**
+   * @param {string} code   stable machine-readable code
+   * @param {string} message
+   * @param {{cause?: unknown, path?: string}} [detail]
+   */
+  constructor (code, message, detail = {}) {
+    super(message, detail.cause === undefined ? undefined : { cause: detail.cause })
+    this.name = 'StoreHostError'
+    this.code = code
+    if (detail.path !== undefined) this.path = detail.path
+  }
+}
+
+/** Reject an argument that is not a usable filesystem path. */
+export function requirePath (value, label) {
+  if (typeof value !== 'string' || value.length === 0) {
+    throw new StoreHostError('BAD_ARGUMENT', `${label} must be a non-empty string`)
+  }
+  if (value.indexOf('\u0000') >= 0) {
+    throw new StoreHostError('BAD_ARGUMENT', `${label} must not contain a NUL byte`)
+  }
+  return value
+}
+
+/**
+ * Reject an offset or length that a 64-bit host accepts and JavaScript
+ * cannot represent exactly. The C extern refuses `offset > INT64_MAX`;
+ * a JavaScript number stops being exact at 2^53 - 1, so that is the
+ * limit this module enforces.
+ */
+export function requireCount (value, label) {
+  if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) {
+    throw new StoreHostError(
+      'BAD_ARGUMENT',
+      `${label} must be a non-negative integer below 2^53`
+    )
+  }
+  return value
+}
+
+/** Reject a payload that is not raw bytes. */
+export function requireBytes (value, label) {
+  if (!(value instanceof Uint8Array)) {
+    throw new StoreHostError('BAD_ARGUMENT', `${label} must be a Uint8Array`)
+  }
+  return value
+}

package/store-host/index.mjs

@@ -0,0 +1,208 @@
+// The store host: the byte-moving half of the persisted Shardborough store
+// when the engine runs as WebAssembly instead of a native binary.
+// https://github.com/danbri/factoidal/issues/641
+//
+// WHAT THIS FILE IS ALLOWED TO DO
+// Open a file by name, move its bytes, sync, rename, list a directory.
+// Nothing else. It never parses a manifest, checks a digest, decodes a
+// block, or decides which artifact answers a query. Those are format
+// decisions and they live in the Lean source (iron rule 7). A reviewer
+// who finds a magic number, a field offset or a hash in this directory
+// has found a rule violation.
+//
+// WHAT IT CORRESPONDS TO
+// `formal/lean4/Harness/PosixRangeIO.lean` declares three `@[extern]`
+// primitives, realised in `formal/lean4/ffi/block_pread.c`:
+//
+//   l4_block_pread                    -> readRange
+//   l4_delta_log_append_sync_at_size  -> appendSyncAtSize
+//   l4_atomic_replace_file_sync       -> atomicReplace
+//
+// `readWhole`, `openCollection` and `listGeneration` have no extern of
+// their own; the native tools use Lean's own `IO.FS` for them.
+//
+// WHERE THE HOST DIFFERS FROM THE C EXTERNS
+//
+// 1. No advisory lock on the append. `l4_delta_log_append_sync_at_size`
+//    holds `flock(fd, LOCK_EX)` across the size check, the write loop and
+//    the fsync, so exactly one of two concurrent writers appends and the
+//    loser gets `false` and retries. Neither Node nor Deno has `flock` in
+//    its standard library, and this module takes no dependency, so the
+//    sequence here is fstat, then append, then fsync, with no lock. Two
+//    writers that observe the same size therefore both append, and the
+//    delta log gets two batches where the protocol expects one. Use this
+//    function only where a single writer is guaranteed. This is the one
+//    divergence that can corrupt a store; see issue 641 stage 4.
+//
+// 2. Failures raise instead of returning false. Every C extern collapses
+//    an open, write or fsync failure into an empty result or `false`. A
+//    `false` there is therefore ambiguous: it can mean the size did not
+//    match, or that the disk is full. Here `false` from
+//    `appendSyncAtSize` means only that the file size was not the
+//    expected one; an I/O failure throws a `StoreHostError`. A caller
+//    that wants the extern's exact shape catches and maps to false.
+//
+// 3. `readRange` raises on a short read. The extern returns an empty
+//    array on any failure and `Harness.PosixRangeIO.readRange?` turns a
+//    wrong size into `none`. Here a short read throws with code
+//    SHORT_READ, which carries the same refusal and says why.
+//
+// 4. `atomicReplace` returning false does not mean nothing changed. The C
+//    version returns false when the parent-directory fsync fails, after
+//    the rename has already succeeded. This module reports it the same
+//    way: false means the new bytes may be in place but their directory
+//    entry is not known to be durable.
+//
+// 5. Deno has no positional read. `readRange` opens its own handle, seeks
+//    and reads. The handle is private to the call, so there is no cursor
+//    to race, but it is a seek plus a read and not one `pread`.
+//
+// 6. Offsets and lengths are JavaScript numbers. The externs take
+//    `UInt64`. Anything at or above 2^53 is rejected with BAD_ARGUMENT
+//    rather than silently rounded.
+//
+// 7. Directory fsync is not portable. `atomicReplace` opens the parent
+//    directory and fsyncs it, which works on Linux and macOS under both
+//    runtimes. Windows refuses to open a directory as a file; the call
+//    returns false there and the rename still happened (point 4).
+
+import { StoreHostError, requireBytes, requireCount, requirePath } from './errors.mjs'
+import { joinPath, requireChildName } from './paths.mjs'
+
+const isDeno = typeof globalThis.Deno !== 'undefined' &&
+  typeof globalThis.Deno.openSync === 'function'
+
+const impl = isDeno ? await import('./deno.mjs') : await import('./node.mjs')
+
+/** 'node' or 'deno' — which implementation load-time selection chose. */
+export const runtime = impl.runtime
+
+export { StoreHostError }
+
+/**
+ * Read a whole file.
+ * @param {string} path
+ * @returns {Uint8Array} a fresh array of exactly the file's bytes
+ */
+export function readWhole (path) {
+  requirePath(path, 'path')
+  return impl.readWhole(path)
+}
+
+/**
+ * Read exactly `length` bytes starting at `offset`, without touching any
+ * shared file cursor. The counterpart of `l4_block_pread`.
+ * @param {string} path
+ * @param {number} offset
+ * @param {number} length
+ * @returns {Uint8Array} exactly `length` bytes
+ * @throws {StoreHostError} code SHORT_READ when the file has fewer bytes
+ */
+export function readRange (path, offset, length) {
+  requirePath(path, 'path')
+  requireCount(offset, 'offset')
+  requireCount(length, 'length')
+  return impl.readRange(path, offset, length)
+}
+
+/**
+ * Append `bytes` only if the file currently has exactly `expectedSize`
+ * bytes, then fsync it. The counterpart of
+ * `l4_delta_log_append_sync_at_size`, minus its advisory lock
+ * (divergence 1 above). Creates the file when absent, in which case the
+ * only size that matches is 0.
+ * @param {string} path
+ * @param {Uint8Array} bytes
+ * @param {number} expectedSize
+ * @returns {boolean} true when the append happened and was synced;
+ *   false when the file's size was not `expectedSize`
+ */
+export function appendSyncAtSize (path, bytes, expectedSize) {
+  requirePath(path, 'path')
+  requireBytes(bytes, 'bytes')
+  requireCount(expectedSize, 'expectedSize')
+  return impl.appendSyncAtSize(path, bytes, expectedSize)
+}
+
+/**
+ * Replace a file's whole contents so that a reader sees either the old
+ * bytes or the new bytes and never a mixture: write a temporary file in
+ * the same directory, fsync it, rename it over the target, fsync the
+ * directory. The counterpart of `l4_atomic_replace_file_sync`.
+ * @param {string} path
+ * @param {Uint8Array} bytes
+ * @returns {boolean} true when the replacement is durable; false when the
+ *   rename succeeded but the directory fsync did not (divergence 4)
+ */
+export function atomicReplace (path, bytes) {
+  requirePath(path, 'path')
+  requireBytes(bytes, 'bytes')
+  return impl.atomicReplace(path, bytes)
+}
+
+/**
+ * List the regular files of one directory with their sizes, sorted by
+ * name. Subdirectories and symbolic links to directories are left out.
+ * @param {string} directory
+ * @returns {{name: string, size: number}[]}
+ */
+export function listGeneration (directory) {
+  requirePath(directory, 'directory')
+  return impl.listGeneration(directory)
+}
+
+// The manifest file names a generation directory can carry, in the order
+// `Harness.ShardMerklePread.readManifest` tries them. This module reads
+// whichever exists and returns its bytes untouched; it does not look
+// inside either one.
+const MANIFEST_NAMES = ['manifest.sbm2', 'manifest.sbm1']
+
+/**
+ * Open an activated collection: read `CURRENT`, and return the generation
+ * name it holds together with the raw manifest bytes of that generation.
+ *
+ * The returned `manifest` is bytes. Deciding what those bytes mean —
+ * which wire version, which artifacts, which digests — is the engine's
+ * job, not this module's.
+ *
+ * @param {string} root the collection root that holds CURRENT
+ * @returns {{root: string, generation: string, generationDir: string,
+ *            manifestName: string, manifest: Uint8Array}}
+ * @throws {StoreHostError} code NO_CURRENT when the root has no CURRENT,
+ *   NO_MANIFEST when the generation carries none of the manifest names
+ */
+export function openCollection (root) {
+  requirePath(root, 'root')
+  const pointerPath = joinPath(root, 'CURRENT')
+  let pointerBytes
+  try {
+    pointerBytes = impl.readWhole(pointerPath)
+  } catch (cause) {
+    throw new StoreHostError(
+      'NO_CURRENT',
+      `${root} has no readable CURRENT pointer`,
+      { path: pointerPath, cause }
+    )
+  }
+  // CURRENT holds a UTF-8 child-generation name (spec section 6.4). Trailing
+  // ASCII whitespace is tolerated so a pointer written by hand still opens.
+  const generation = requireChildName(
+    new TextDecoder('utf-8', { fatal: true }).decode(pointerBytes).replace(/[\r\n\t ]+$/, ''),
+    'CURRENT'
+  )
+  const generationDir = joinPath(root, generation)
+  for (const manifestName of MANIFEST_NAMES) {
+    try {
+      const manifest = impl.readWhole(joinPath(generationDir, manifestName))
+      return { root, generation, generationDir, manifestName, manifest }
+    } catch (error) {
+      if (error instanceof StoreHostError && error.code === 'OPEN_FAILED') continue
+      throw error
+    }
+  }
+  throw new StoreHostError(
+    'NO_MANIFEST',
+    `${generationDir} has none of ${MANIFEST_NAMES.join(', ')}`,
+    { path: generationDir }
+  )
+}

package/store-host/node.mjs

@@ -0,0 +1,212 @@
+// Node implementation of the four host primitives the Lean persisted store
+// needs (Harness/PosixRangeIO.lean). See ./index.mjs for the contract and
+// for the places where Node's semantics differ from the C externs.
+//
+// Nothing here parses, verifies or interprets a byte. It opens files,
+// moves bytes, and syncs.
+
+import {
+  closeSync, fstatSync, fsyncSync, openSync, readSync, readdirSync,
+  renameSync, statSync, unlinkSync, writeSync
+} from 'node:fs'
+
+import { StoreHostError } from './errors.mjs'
+import { baseName, dirName, joinPath } from './paths.mjs'
+
+export const runtime = 'node'
+
+function wrap (code, message, path, cause) {
+  return new StoreHostError(code, `${message}: ${String(cause && cause.message ? cause.message : cause)}`, { path, cause })
+}
+
+function isInterrupt (error) {
+  return error && (error.code === 'EINTR' || error.code === 'EAGAIN')
+}
+
+function openRead (path) {
+  try {
+    return openSync(path, 'r')
+  } catch (cause) {
+    throw wrap('OPEN_FAILED', `cannot open ${path} for reading`, path, cause)
+  }
+}
+
+/** Read `length` bytes at `offset` into `out` at `outOffset`. Returns how many. */
+function preadInto (fd, out, outOffset, length, offset, path) {
+  let done = 0
+  while (done < length) {
+    let read
+    try {
+      read = readSync(fd, out, outOffset + done, length - done, offset + done)
+    } catch (cause) {
+      if (isInterrupt(cause)) continue
+      throw wrap('READ_FAILED', `read failed on ${path}`, path, cause)
+    }
+    if (read === 0) break
+    done += read
+  }
+  return done
+}
+
+export function readWhole (path) {
+  const fd = openRead(path)
+  try {
+    const size = fstatSync(fd).size
+    if (!Number.isSafeInteger(size)) {
+      throw new StoreHostError('FILE_TOO_LARGE', `${path} is larger than 2^53 - 1 bytes`, { path })
+    }
+    const out = new Uint8Array(size)
+    const done = preadInto(fd, out, 0, size, 0, path)
+    if (done !== size) {
+      throw new StoreHostError('SHORT_READ', `${path} shrank during the read (${done} of ${size} bytes)`, { path })
+    }
+    return out
+  } finally {
+    closeSync(fd)
+  }
+}
+
+export function readRange (path, offset, length) {
+  if (length === 0) return new Uint8Array(0)
+  const fd = openRead(path)
+  try {
+    const out = new Uint8Array(length)
+    const done = preadInto(fd, out, 0, length, offset, path)
+    if (done !== length) {
+      throw new StoreHostError(
+        'SHORT_READ',
+        `${path} returned ${done} of ${length} bytes at offset ${offset}`,
+        { path }
+      )
+    }
+    return out
+  } finally {
+    closeSync(fd)
+  }
+}
+
+export function appendSyncAtSize (path, bytes, expectedSize) {
+  let fd
+  try {
+    // 'a' is O_WRONLY | O_CREAT | O_APPEND, matching the C extern's open.
+    fd = openSync(path, 'a')
+  } catch (cause) {
+    throw wrap('OPEN_FAILED', `cannot open ${path} for append`, path, cause)
+  }
+  try {
+    const size = fstatSync(fd).size
+    if (size !== expectedSize) return false
+    let done = 0
+    while (done < bytes.length) {
+      let written
+      try {
+        // position null keeps the O_APPEND placement the C extern relies on.
+        written = writeSync(fd, bytes, done, bytes.length - done, null)
+      } catch (cause) {
+        if (isInterrupt(cause)) continue
+        throw wrap('WRITE_FAILED', `append failed on ${path}`, path, cause)
+      }
+      done += written
+    }
+    try {
+      fsyncSync(fd)
+    } catch (cause) {
+      throw wrap('FSYNC_FAILED', `fsync failed on ${path}`, path, cause)
+    }
+    return true
+  } finally {
+    closeSync(fd)
+  }
+}
+
+function temporaryName (path) {
+  const suffix = Math.floor(Math.random() * 0xffffff).toString(16).padStart(6, '0')
+  return joinPath(dirName(path), baseName(path) + '.tmp.' + suffix)
+}
+
+function fsyncDirectory (directory) {
+  let fd
+  try {
+    fd = openSync(directory, 'r')
+  } catch (cause) {
+    throw wrap('DIR_OPEN_FAILED', `cannot open ${directory} to sync it`, directory, cause)
+  }
+  try {
+    fsyncSync(fd)
+  } finally {
+    closeSync(fd)
+  }
+}
+
+export function atomicReplace (path, bytes) {
+  const directory = dirName(path)
+  let temporary = null
+  let fd = null
+  try {
+    for (let attempt = 0; attempt < 8 && fd === null; attempt += 1) {
+      temporary = temporaryName(path)
+      try {
+        // 'wx' is O_WRONLY | O_CREAT | O_EXCL, the exclusive create that
+        // makes the name ours the way mkstemp does in the C extern.
+        fd = openSync(temporary, 'wx')
+      } catch (cause) {
+        if (cause && cause.code === 'EEXIST') continue
+        throw wrap('OPEN_FAILED', `cannot create ${temporary}`, temporary, cause)
+      }
+    }
+    if (fd === null) {
+      throw new StoreHostError('TEMP_NAME_EXHAUSTED', `no free temporary name beside ${path}`, { path })
+    }
+    let done = 0
+    while (done < bytes.length) {
+      let written
+      try {
+        written = writeSync(fd, bytes, done, bytes.length - done, null)
+      } catch (cause) {
+        if (isInterrupt(cause)) continue
+        throw wrap('WRITE_FAILED', `write failed on ${temporary}`, temporary, cause)
+      }
+      done += written
+    }
+    fsyncSync(fd)
+    closeSync(fd)
+    fd = null
+    renameSync(temporary, path)
+    temporary = null
+    try {
+      fsyncDirectory(directory)
+    } catch (_error) {
+      // The C extern also returns false here, with the replacement already
+      // done. Reported the same way; see ./index.mjs for what false means.
+      return false
+    }
+    return true
+  } catch (error) {
+    if (fd !== null) closeSync(fd)
+    if (temporary !== null) {
+      try { unlinkSync(temporary) } catch (_ignored) { /* the temporary may not exist */ }
+    }
+    throw error
+  }
+}
+
+export function listGeneration (directory) {
+  let names
+  try {
+    names = readdirSync(directory)
+  } catch (cause) {
+    throw wrap('DIR_READ_FAILED', `cannot list ${directory}`, directory, cause)
+  }
+  const out = []
+  for (const name of names.sort()) {
+    let info
+    try {
+      info = statSync(joinPath(directory, name))
+    } catch (_cause) {
+      continue // a name that vanished between readdir and stat
+    }
+    if (!info.isFile()) continue
+    out.push({ name, size: info.size })
+  }
+  return out
+}