@bjornpagen/bumbledb-log 0.18.0 → 0.19.0

package/README.md

@@ -1,7 +1,7 @@
 # @bjornpagen/bumbledb-log
 
 Braided object-store replication for [bumbledb](https://github.com/bjornpagen/bumbledb):
-a thin peer of `@bjornpagen/bumbledb` (peer `^0.18.0`). The package is
+a thin peer of `@bjornpagen/bumbledb` (peer `^0.19.0`). The package is
 three things:
 
 1. **The mirrored pure pair**, byte-exact against the Rust driver and
@@ -40,7 +40,7 @@ Async ⟺ network: `openReplica`, `refresh`, `waitFor`, `commit`,
 import { fsStore, openReplica, openWriter } from "@bjornpagen/bumbledb-log"
 
 export const replica = await openReplica({ store: s3(env), prefix: "prod/main", dir: "/tmp/store", theory: Ledger })
-export const writer = openWriter(replica)
+export const writer = await openWriter(replica)
 
 // route handler
 const out = await writer.commit((batch) => batch.insert(Booking, [row]))
@@ -78,7 +78,7 @@ const replica = await openReplica({
 	dir: `/data/primer/replicas/${scopeName}`, // per-process local dir — never shared
 	theory: Explanation
 })
-const writer = openWriter(replica)
+const writer = await openWriter(replica)
 
 // one pass = refresh, render, emit, lower, one commit
 await replica.refresh()
@@ -116,7 +116,8 @@ of the object file and its parent directory.
 Exported sentinel values on the SDK idiom, checked with `errors.is`,
 never by message strings: `ErrRefused` (typed per cause — batch shape,
 version, fingerprint, manifest shape, checkpoint braid-set drift),
-`ErrSpanningCommit`, `ErrGapDetected`, `ErrReplayDiverged`,
+`ErrManifestMissing` (a replica found no manifest; only the writer
+births a store), `ErrSpanningCommit`, `ErrGapDetected`, `ErrReplayDiverged`,
 `ErrChainMismatch` (cause `"prev" | "slot" |
 "timestamp"`), `ErrContention` (cause `hot-key` or `slot-race`),
 `ErrStore` (the vendor channel, present in every wrapped store

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bjornpagen/bumbledb-log",
-  "version": "0.18.0",
+  "version": "0.19.0",
   "description": "Braided object-store replication for bumbledb: the command codec, theory-derived braids, and replica/writer over five store verbs",
   "type": "module",
   "exports": {
@@ -33,10 +33,11 @@
   "homepage": "https://github.com/bjornpagen/bumbledb#readme",
   "dependencies": {
     "@aws-sdk/client-s3": "^3.1116.0",
-    "@superbuilders/errors": "^4.0.2"
+    "@superbuilders/errors": "^4.0.2",
+    "arkregex": "^0.0.8"
   },
   "peerDependencies": {
-    "@bjornpagen/bumbledb": "^0.18.0"
+    "@bjornpagen/bumbledb": "^0.19.0"
   },
   "devDependencies": {
     "@biomejs/biome": "2.5.4",

package/src/braids.ts

@@ -6,7 +6,9 @@
  */
 
 import type { Braid, SerialStatement, Theory } from "#descriptor.ts"
-import { braid, descriptorOf } from "#descriptor.ts"
+import { braid, braidHex, descriptorOf } from "#descriptor.ts"
+
+const U32_MAX = 0xffffffff
 
 /** The schema's own shard map: ordinary relation name → braid id. */
 function braidsOf(theory: Theory): ReadonlyMap<string, Braid> {
@@ -31,5 +33,18 @@ function serialAtStatementsOf(theory: Theory): readonly SerialStatement[] {
 	return descriptorOf(theory).serialAtStatements
 }
 
+/**
+ * Parses a wire u32 into a braid id: valid only when the relation it
+ * names is the smallest in its own component. An unknown, closed, or
+ * non-head id is not a braid — the caller refuses, it is not ignored.
+ */
+function parse(theory: Theory, raw: number): Braid | undefined {
+	if (!Number.isInteger(raw) || raw < 0 || raw > U32_MAX) {
+		return undefined
+	}
+	const name = braidHex(raw)
+	return descriptorOf(theory).braidMembers.has(name) ? name : undefined
+}
+
 export type { Braid }
-export { braid, braidsOf, serialAtStatementsOf }
+export { braid, braidsOf, parse, serialAtStatementsOf }

package/src/bytes.ts

@@ -1,9 +1,11 @@
 /**
  * Little-endian byte primitives shared by the codec and the
- * fingerprint mirror. Every multi-byte integer on the batch wire
- * is little-endian; the fingerprint's canonical literal encoding is the
- * engine's big-endian order-preserving form — both live here so no third
- * spelling can appear.
+ * fingerprint mirror, plus the hex grammar every document digest
+ * walks. Every multi-byte integer on the batch wire is little-endian;
+ * the fingerprint's canonical literal encoding is the engine's
+ * big-endian order-preserving form; a digest is 32 bytes, rendered as
+ * 64 lowercase hex characters. Integer order and hex width live here
+ * so no third spelling can appear.
  */
 
 import * as errors from "@superbuilders/errors"
@@ -13,6 +15,13 @@ const I64_MIN = -0x8000000000000000n
 const I64_MAX = 0x7fffffffffffffffn
 const I64_SIGN_BIT = 0x8000000000000000n
 
+const utf8Encoder = new TextEncoder()
+/** Fatal UTF-8. ignoreBOM is true: a leading U+FEFF is a character, not a stripped BOM. */
+const utf8StrictDecoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true })
+
+declare const digest32Brand: unique symbol
+type Digest32 = Uint8Array & { readonly [digest32Brand]: typeof digest32Brand }
+
 class ByteWriter {
 	private buf: Uint8Array
 	private len = 0
@@ -46,6 +55,10 @@ class ByteWriter {
 		this.len += raw.length
 	}
 
+	array32(value: Digest32): void {
+		this.bytes(value)
+	}
+
 	u16le(value: number): void {
 		this.grow(2)
 		this.buf[this.len] = value & 0xff
@@ -148,6 +161,10 @@ class ByteReader {
 		return new Uint8Array(this.take(count, what))
 	}
 
+	array32(what: string): Digest32 {
+		return digest32(this.take(32, what))
+	}
+
 	u16le(what: string): number {
 		const raw = this.take(2, what)
 		return (raw[0] ?? 0) | ((raw[1] ?? 0) << 8)
@@ -198,6 +215,16 @@ function bytesCompare(a: Uint8Array, b: Uint8Array): number {
 
 const HEX_DIGITS = "0123456789abcdef"
 
+function hexNibble(byte: number): number | undefined {
+	if (byte >= 0x30 && byte <= 0x39) {
+		return byte - 0x30
+	}
+	if (byte >= 0x61 && byte <= 0x66) {
+		return byte - 0x61 + 10
+	}
+	return undefined
+}
+
 function toHex(bytes: Uint8Array): string {
 	let out = ""
 	for (const byte of bytes) {
@@ -208,27 +235,68 @@ function toHex(bytes: Uint8Array): string {
 }
 
 function fromHex(hex: string): Uint8Array {
-	if (hex.length % 2 !== 0 || /[^0-9a-f]/.test(hex)) {
+	const raw = utf8Encoder.encode(hex)
+	if (raw.length % 2 !== 0) {
 		throw errors.new(`not lowercase hex: ${hex}`)
 	}
-	const out = new Uint8Array(hex.length / 2)
-	for (let i = 0; i < out.length; i++) {
-		out[i] = Number.parseInt(hex.slice(i * 2, i * 2 + 2), 16)
+	const out = new Uint8Array(raw.length / 2)
+	for (let i = 0, j = 0; i < raw.length; i += 2, j++) {
+		const hiByte = raw[i]
+		const loByte = raw[i + 1]
+		if (hiByte === undefined || loByte === undefined) {
+			throw errors.new(`not lowercase hex: ${hex}`)
+		}
+		const hi = hexNibble(hiByte)
+		const lo = hexNibble(loByte)
+		if (hi === undefined || lo === undefined) {
+			throw errors.new(`not lowercase hex: ${hex}`)
+		}
+		out[j] = (hi << 4) | lo
 	}
 	return out
 }
 
-const utf8Encoder = new TextEncoder()
-const utf8StrictDecoder = new TextDecoder("utf-8", { fatal: true })
+function digest32(bytes: Uint8Array): Digest32 {
+	if (bytes.length !== 32) {
+		throw errors.new(`digest is not 32 bytes: ${bytes.length}`)
+	}
+	const out = new Uint8Array(32)
+	out.set(bytes)
+	return out as Digest32
+}
+
+function digest32FromHex(hex: string): Digest32 {
+	return digest32(fromHex(hex))
+}
+
+function hex32(bytes: Digest32): string {
+	return toHex(bytes)
+}
+
+function saturatingAddU64(a: bigint, b: bigint): bigint {
+	const sum = a + b
+	return sum > U64_MAX ? U64_MAX : sum
+}
+
+function checkedAddU64(a: bigint, b: bigint): bigint | undefined {
+	const sum = a + b
+	return sum > U64_MAX ? undefined : sum
+}
 
+export type { Digest32 }
 export {
 	ByteReader,
 	ByteWriter,
 	bytesCompare,
 	bytesEqual,
+	checkedAddU64,
+	digest32,
+	digest32FromHex,
 	fromHex,
+	hex32,
 	I64_MAX,
 	I64_MIN,
+	saturatingAddU64,
 	toHex,
 	U64_MAX,
 	utf8Encoder,

package/src/chain.ts

@@ -1,22 +1,34 @@
 /**
- * The chain sidecar (50): the per-braid split and chain position in
- * `dir/chain.json`, written atomically (temp + rename, fsync). A floor
- * cache, never a truth the store reconciles against — recovery is the
- * catch-up loop, and the one wholeness check lives at the replica.
- * `pending` is the writer's one extra field: the encoded batch bytes a
- * local commit owes the log, present until its slot exists.
+ * The chain sidecar in `dir/chain`: a floor cache of chain
+ * position, written atomically (temp + rename, fsync). The chain is
+ * Settled or Pending — generation is the vector sum, plus one exactly
+ * when the value is Pending. The document is a binary v:3 record:
+ * version byte 3, counted roster of braid / g / prev / ts, pending
+ * tag. Wire pending is the batch bytes. The content address is blake3
+ * of those bytes. Every integer is little-endian.
  */
 
 import * as crypto from "node:crypto"
 import * as fs from "node:fs/promises"
 import * as path from "node:path"
 import * as errors from "@superbuilders/errors"
-import { utf8StrictDecoder } from "#bytes.ts"
+import { ByteReader, ByteWriter, saturatingAddU64, U64_MAX } from "#bytes.ts"
 import type { ChainEntry } from "#codec.ts"
 import type { Braid } from "#descriptor.ts"
-import { braid } from "#descriptor.ts"
+import { braidHex } from "#descriptor.ts"
+import { refuse } from "#errors.ts"
 import type { Generation } from "#keys.ts"
 import { generation } from "#keys.ts"
+import { Vector } from "#vector.ts"
+
+/** The sidecar's file name inside a replica directory. */
+const CHAIN_FILE = "chain"
+
+const VERSION = 3
+const SETTLED = 0
+const PENDING = 1
+/** u32 braid + u64 g + 32 prev + u64 ts. */
+const ENTRY_BYTES = 52n
 
 interface Pending {
 	readonly braid: Braid
@@ -24,69 +36,174 @@ interface Pending {
 	readonly bytes: Uint8Array
 }
 
-interface Sidecar {
-	readonly chain: ReadonlyMap<Braid, ChainEntry>
-	readonly pending: Pending | null
+type Chain =
+	| { readonly tag: "settled"; readonly entries: ReadonlyMap<Braid, ChainEntry> }
+	| { readonly tag: "pending"; readonly entries: ReadonlyMap<Braid, ChainEntry>; readonly batch: Pending }
+
+type SidecarRead =
+	| { readonly tag: "absent" }
+	| { readonly tag: "fault"; readonly io: Error }
+	| { readonly tag: "corrupt"; readonly parse: Error }
+	| { readonly tag: "read"; readonly chain: Chain }
+
+function codeOf(error: Error): string | undefined {
+	return (error as NodeJS.ErrnoException).code
+}
+
+function braidIdOf(id: Braid): number {
+	return Number.parseInt(id.slice(1), 16)
+}
+
+function vectorOf(entries: ReadonlyMap<Braid, { readonly g: bigint }>): Vector {
+	const counts = new Map<Braid, bigint>()
+	for (const [braid, entry] of entries) {
+		counts.set(braid, entry.g)
+	}
+	return Vector.from(counts)
+}
+
+function chainSum(chain: Chain): bigint {
+	const sum = vectorOf(chain.entries).sum()
+	return typeof sum === "bigint" ? sum : U64_MAX
 }
 
-function renderSidecar(sidecar: Sidecar): string {
-	const braids = [...sidecar.chain.keys()].sort()
-	const chain = braids
-		.map(function renderEntry(braid) {
-			const entry = sidecar.chain.get(braid)
-			if (entry === undefined) {
-				throw errors.new(`sidecar chain lost braid ${braid}`)
-			}
-			return `"${braid}":{"g":${entry.g},"prev":"${entry.prev}","ts":${entry.ts}}`
-		})
-		.join(",")
-	const pending =
-		sidecar.pending === null
-			? "null"
-			: `{"braid":"${sidecar.pending.braid}","gen":${sidecar.pending.gen},"bytes":"${Buffer.from(sidecar.pending.bytes).toString("base64")}"}`
-	return `{"v":2,"chain":{${chain}},"pending":${pending}}`
+function chainGeneration(chain: Chain): bigint {
+	const sum = chainSum(chain)
+	return chain.tag === "settled" ? sum : saturatingAddU64(sum, 1n)
 }
 
-function parseSidecar(text: string): Sidecar {
-	const parsed = JSON.parse(text) as {
-		v: number
-		chain: Record<string, { g: number; prev: string; ts: number }>
-		pending: { braid: string; gen: number; bytes: string } | null
-	}
-	if (parsed.v !== 2 || typeof parsed.chain !== "object" || parsed.chain === null) {
-		throw errors.new("sidecar is not a v2 chain file")
-	}
-	const chain = new Map<Braid, ChainEntry>()
-	for (const [name, entry] of Object.entries(parsed.chain)) {
-		chain.set(braid(name), { g: generation(BigInt(entry.g)), prev: entry.prev, ts: BigInt(entry.ts) })
-	}
-	const pending =
-		parsed.pending === null
-			? null
-			: {
-					braid: braid(parsed.pending.braid),
-					gen: generation(BigInt(parsed.pending.gen)),
-					bytes: new Uint8Array(Buffer.from(parsed.pending.bytes, "base64"))
-				}
-	return { chain, pending }
+/** A declared count the remaining bytes cannot open is Malformed
+ *  before the loop. */
+function refuseUnbacked(count: bigint, remaining: number, minItem: bigint, at: string): void {
+	if (count === 0n) {
+		return
+	}
+	if (minItem === 0n || BigInt(remaining) / minItem < count) {
+		refuse({ kind: "Malformed", at: remaining }, `declared ${at} ${count} outruns the remaining ${remaining} bytes`)
+	}
 }
 
-async function readSidecar(file: string): Promise<Sidecar | null> {
+function renderSidecar(chain: Chain): Uint8Array {
+	const out = new ByteWriter(64)
+	out.u8(VERSION)
+	const braids = [...chain.entries.keys()].sort()
+	if (braids.length > 0xffffffff) {
+		throw errors.new("sidecar chain count exceeds u32")
+	}
+	out.u32le(braids.length)
+	for (const id of braids) {
+		const entry = chain.entries.get(id)
+		if (entry === undefined) {
+			throw errors.new(`sidecar chain lost braid ${id}`)
+		}
+		out.u32le(braidIdOf(id))
+		out.u64le(entry.g)
+		out.bytes(entry.prev)
+		out.u64le(entry.ts)
+	}
+	if (chain.tag === "settled") {
+		out.u8(SETTLED)
+		return out.finish()
+	}
+	out.u8(PENDING)
+	out.u32le(braidIdOf(chain.batch.braid))
+	out.u64le(chain.batch.gen)
+	if (chain.batch.bytes.length > 0xffffffff) {
+		throw errors.new("sidecar pending exceeds u32 length")
+	}
+	out.u32le(chain.batch.bytes.length)
+	out.bytes(chain.batch.bytes)
+	return out.finish()
+}
+
+function parseSidecar(bytes: Uint8Array, known?: ReadonlySet<Braid>): Chain {
+	const reader = new ByteReader(bytes, {
+		fail(what: string): never {
+			refuse({ kind: "Malformed", at: bytes.length }, `sidecar truncated at ${what}`)
+		}
+	})
+	const at = function offset(): number {
+		return bytes.length - reader.remaining()
+	}
+	const version = reader.u8("version")
+	if (version !== VERSION) {
+		refuse({ kind: "Version", version }, `sidecar version ${version}, consumers refuse ≠ ${VERSION}`)
+	}
+	const count = BigInt(reader.u32le("chain count"))
+	refuseUnbacked(count, reader.remaining(), ENTRY_BYTES, "chain count")
+	const entries = new Map<Braid, ChainEntry>()
+	let last: Braid | undefined
+	for (let i = 0n; i < count; i++) {
+		const raw = reader.u32le("braid")
+		const name = braidHex(raw)
+		if (known !== undefined && !known.has(name)) {
+			refuse({ kind: "UnknownBraid", braid: raw }, `sidecar cites unknown braid ${name}`)
+		}
+		const g = reader.u64le("g")
+		const prev = reader.array32("prev")
+		const ts = reader.u64le("ts")
+		if (last !== undefined && last >= name) {
+			refuse({ kind: "Malformed", at: at() }, "sidecar chain is not strictly ascending")
+		}
+		entries.set(name, { g: generation(g), prev, ts })
+		last = name
+	}
+	if (typeof vectorOf(entries).sum() !== "bigint") {
+		refuse({ kind: "Overflow" }, "sidecar chain sum overflows u64")
+	}
+	const tag = reader.u8("pending")
+	if (tag === SETTLED) {
+		if (reader.remaining() !== 0) {
+			refuse({ kind: "Malformed", at: reader.remaining() }, `${reader.remaining()} trailing bytes after the sidecar`)
+		}
+		return { tag: "settled", entries }
+	}
+	if (tag !== PENDING) {
+		refuse({ kind: "Malformed", at: at() - 1 }, `sidecar pending tag ${tag}`)
+	}
+	const raw = reader.u32le("pending braid")
+	const name = braidHex(raw)
+	if (known !== undefined && !known.has(name)) {
+		refuse({ kind: "UnknownBraid", braid: raw }, `sidecar pending cites unknown braid ${name}`)
+	}
+	const slot = reader.u64le("pending generation")
+	const length = reader.u32le("pending length")
+	const body = reader.bytes(length, "pending bytes")
+	if (reader.remaining() !== 0) {
+		refuse({ kind: "Malformed", at: reader.remaining() }, `${reader.remaining()} trailing bytes after the sidecar`)
+	}
+	return {
+		tag: "pending",
+		entries,
+		batch: { braid: name, gen: generation(slot), bytes: body }
+	}
+}
+
+async function readSidecar(file: string, known?: ReadonlySet<Braid>): Promise<SidecarRead> {
 	const read = await errors.try(fs.readFile(file))
 	if (read.error) {
-		return null
+		if (codeOf(read.error) === "ENOENT") {
+			return { tag: "absent" }
+		}
+		return { tag: "fault", io: read.error }
+	}
+	const parsed = errors.trySync(function parse() {
+		return parseSidecar(read.data, known)
+	})
+	if (parsed.error) {
+		return { tag: "corrupt", parse: parsed.error }
 	}
-	return parseSidecar(utf8StrictDecoder.decode(read.data))
+	return { tag: "read", chain: parsed.data }
 }
 
-async function writeSidecar(file: string, sidecar: Sidecar): Promise<void> {
+async function writeSidecar(file: string, chain: Chain): Promise<void> {
 	const dir = path.dirname(file)
 	await fs.mkdir(dir, { recursive: true })
 	const temp = path.join(dir, `.chain-${process.pid}-${crypto.randomBytes(4).toString("hex")}`)
 	const handle = await fs.open(temp, "wx")
 	const written = await errors.try(
 		(async function writeAll() {
-			await handle.writeFile(renderSidecar(sidecar))
+			await handle.writeFile(renderSidecar(chain))
 			await handle.sync()
 		})()
 	)
@@ -104,5 +221,5 @@ async function writeSidecar(file: string, sidecar: Sidecar): Promise<void> {
 	}
 }
 
-export type { ChainEntry, Pending, Sidecar }
-export { readSidecar, renderSidecar, writeSidecar }
+export type { Chain, ChainEntry, Pending, SidecarRead }
+export { CHAIN_FILE, chainGeneration, chainSum, parseSidecar, readSidecar, renderSidecar, writeSidecar }