@bjornpagen/bumbledb-log 0.17.0

package/README.md

@@ -0,0 +1,119 @@
+# @bjornpagen/bumbledb-log
+
+Braided object-store replication for [bumbledb](https://github.com/bjornpagen/bumbledb):
+a thin peer of `@bjornpagen/bumbledb` (peer dependency, 0.17.x lockstep).
+The package is three things:
+
+1. **The mirrored pure pair**, byte-exact against the Rust driver and
+   pinned by cross-language goldens: `encodeBatch`/`decodeBatch` (the
+   BDBL v2 command codec — a batch is header + ops, nothing else) and
+   `braidsOf(descriptor)` (the schema's own shard map, as data — with
+   `serialAtStatementsOf` naming the degenerate-serial statements beside it).
+2. **The five-verb object store** — `get`, `getIfChanged`, `putCreate`,
+   `putSwap`, `delete` — with `fsStore` as the tier-1 local-directory
+   implementation (deployment case 5's production backend, not a dev
+   double). The S3/R2/OCI store rides `aws4fetch` and is not yet in this
+   build (the dependency was unfetchable offline).
+3. **Replica and writer** composed from the engine SDK's existing verbs:
+   `openReplica` hands out the SDK's own `Db`; `openWriter` adds the
+   right to create log objects; `openTenants` is an LRU of per-tenant
+   replicas. No engine surface is duplicated.
+
+Async ⟺ network: `openReplica`, `refresh`, `waitFor`, `commit`,
+`commitSplit`, and disposal await store verbs; everything on
+`replica.db`, the `batch.*` recorders, and the pure pair are synchronous.
+
+## The Vercel recipe (documented example, not framework code)
+
+```ts
+// lib/db.ts — module scope; Fluid shares this across the instance's requests
+import { fsStore, openReplica, openWriter } from "@bjornpagen/bumbledb-log"
+
+export const replica = await openReplica({ store: s3(env), prefix: "prod/main", dir: "/tmp/store", theory: Ledger })
+export const writer = openWriter(replica)
+
+// route handler
+const out = await writer.commit((b) => b.insert(Booking, [row]))
+if (out.tag === "accepted") ctx.waitUntil(replica.refresh(out.braid))
+```
+
+- **The `/tmp` budget gate**: checkpoint plus working set must stay
+  ≤ 400 MB (100 MB headroom under the 500 MB instance limit); the
+  leaf-blob pattern keeps metadata stores in the tens of MB. Per-tenant
+  fleets get the same gate through `openTenants({ budgetBytes, maxOpen })`.
+- **Cross-instance read-your-writes**: a commit returns
+  `(braid, generation)`; a session token is the pointwise max of every
+  pair a flow has seen; `replica.waitFor(vector)` refreshes until the
+  local vector dominates it. The committing instance always reads its
+  own writes without waiting. A singleton map is the single-braid form.
+- **The `ErrContention` runbook**: the error carries its cause, sourced
+  from the terminal re-judgment itself — `{ kind: "hot-key", statement,
+  determinants }` names the statement and carries the offending facts'
+  raw values from the engine's own violation; the remedies are a
+  reservation relation on the hot capacity (an ordinary weighted child
+  row — the schema idiom) or resident mode. `{ kind: "slot-race", tip }`
+  means the terminal losses were accepted but out-raced: an operational
+  signal to shard the theory into more braids or move the hot braid to
+  a resident Rust writer, whose group commit batches the queue. This
+  package ships no group commit of its own; the recorded reopen trigger
+  is a measured TS deployment at Turso-density write rates where a
+  deliberate batching delay would amortize many writers into one PUT.
+
+## The local-fleet recipe (deployment case 5)
+
+```ts
+// one process per scope loop; all processes share one FsStore prefix
+import { fsStore, openReplica, openWriter } from "@bjornpagen/bumbledb-log"
+
+const replica = await openReplica({
+	store: fsStore("/data/primer/log"), // the five verbs over a directory
+	prefix: "world/v1",
+	dir: `/data/primer/replicas/${scopeName}`, // per-process local dir — never shared
+	theory: Explanation
+})
+const writer = openWriter(replica)
+
+// one pass = refresh, render, emit, lower, one commit
+await replica.refresh()
+const out = await writer.commit((batch) => {
+	batch.insert(Explanation, growth.explanations) // ids from batch.reserve
+	batch.insert(Case, growth.cases)
+	return growth.summary
+})
+// rejected ⇒ the host re-renders against the moved world and re-lowers —
+// a K-conflict double-mint resolves to the winner's row on the next pass.
+```
+
+What makes the case easy: an insert-only theory has no delete races to
+lose; content-keyed determinants keep concurrent scope loops off each
+other's obligations, so a lost slot re-judges to the same accepted
+verdict at the moved base; a one-braid theory serializes slot claims on
+a link publication, which at document-per-minutes commit rates is free.
+Each process owns its local replica directory outright.
+
+**`fsStore`'s discipline, stated** — the one on-disk protocol the Rust
+`FsStore` also speaks, raced against it in the interop conformance
+lane: `putCreate` writes an exclusive synced temp and publishes it with
+`fs.link` (EEXIST is the honest `exists`); etags are the blake3 of the
+content, computed on every read and never stored; `putSwap` serializes
+under a pid-lockfile beside the key, published with the same
+temp-plus-link discipline, and a lock whose owner pid is dead is broken
+and retaken. One machine is load-bearing, not descriptive — pid
+liveness and link exclusivity are the arbitration primitives, and
+network filesystems weaken both; an `fsStore` prefix on a network mount
+is a misdeployment. `putCreate` and `putSwap` resolve only after fsync
+of the object file and its parent directory.
+
+## Error identity
+
+Exported sentinel values on the SDK idiom, checked with `errors.is`,
+never by message strings: `ErrRefused` (typed per cause — batch shape,
+version, fingerprint, manifest shape, checkpoint braid-set drift),
+`ErrSpanningCommit`, `ErrGapDetected`, `ErrReplayDiverged`,
+`ErrChainMismatch` (cause `"prev" | "slot" |
+"timestamp"`), `ErrContention` (cause `hot-key` or `slot-race`),
+`ErrStore` (the vendor channel, present in every wrapped store
+failure's cause chain so the `errors.is` match is by identity). There
+is deliberately no
+`ErrAlreadyApplied`: the state it would name is absorbed by idempotent
+replay and never surfaces.

package/package.json

@@ -0,0 +1,58 @@
+{
+  "name": "@bjornpagen/bumbledb-log",
+  "version": "0.17.0",
+  "description": "Braided object-store replication for bumbledb: the command codec, theory-derived braids, and replica/writer over five store verbs",
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./src/index.ts",
+      "default": "./src/index.ts"
+    }
+  },
+  "imports": {
+    "#test/*.ts": "./test/*.ts",
+    "#*.ts": "./src/*.ts"
+  },
+  "files": [
+    "src",
+    "README.md"
+  ],
+  "keywords": [
+    "database",
+    "replication",
+    "object-storage",
+    "bumbledb"
+  ],
+  "author": "Bjorn Pagen",
+  "license": "0BSD",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/bjornpagen/bumbledb.git",
+    "directory": "ts-log"
+  },
+  "homepage": "https://github.com/bjornpagen/bumbledb#readme",
+  "dependencies": {
+    "@superbuilders/errors": "^4.0.2"
+  },
+  "peerDependencies": {
+    "@bjornpagen/bumbledb": "^0.17.1"
+  },
+  "devDependencies": {
+    "@biomejs/biome": "2.5.4",
+    "@bjornpagen/bumbledb": "link:../ts",
+    "@types/node": "26.1.1",
+    "typescript": "7.0.2"
+  },
+  "engines": {
+    "node": ">=24"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "private": false,
+  "scripts": {
+    "typecheck": "tsc --noEmit",
+    "lint": "biome check .",
+    "test": "node --test 'test/**/*.test.ts'"
+  }
+}

package/src/braids.ts

@@ -0,0 +1,38 @@
+/**
+ * Braid derivation as data (10): connected components of the statement
+ * graph over ordinary relations, the braid id the smallest RelationId in
+ * the component rendered `c{id:08x}`. Assignment is a pure function of
+ * the descriptor, pinned cross-language by the codec goldens.
+ */
+
+import type { LogTheory, SerialStatement } from "#descriptor.ts"
+import { descriptorOf } from "#descriptor.ts"
+
+/** Braid id: `c{smallest RelationId:08x}`, scoped to the schema fingerprint. */
+type Braid = string
+
+/** The schema's own shard map: ordinary relation name → braid id. */
+function braidsOf(theory: LogTheory): ReadonlyMap<string, Braid> {
+	const descriptor = descriptorOf(theory)
+	const out = new Map<string, Braid>()
+	for (const relation of descriptor.relations) {
+		const braid = descriptor.braidOfRelation.get(relation.id)
+		if (braid !== undefined) {
+			out.set(relation.name, braid)
+		}
+	}
+	return out
+}
+
+/**
+ * The degenerate-serial roster (15): key or capacity statements whose
+ * determinant projection is empty name one global group, so their braid
+ * serializes at that statement. Typed data beside the braid map, one
+ * question per verb.
+ */
+function serialAtStatementsOf(theory: LogTheory): readonly SerialStatement[] {
+	return descriptorOf(theory).serialAtStatements
+}
+
+export type { Braid }
+export { braidsOf, serialAtStatementsOf }

package/src/bytes.ts

@@ -0,0 +1,236 @@
+/**
+ * Little-endian byte primitives shared by the codec, the footprint keys,
+ * and the fingerprint mirror. Every multi-byte integer on the batch wire
+ * is little-endian; the fingerprint's canonical literal encoding is the
+ * engine's big-endian order-preserving form — both live here so no third
+ * spelling can appear.
+ */
+
+import * as errors from "@superbuilders/errors"
+
+const U64_MAX = 0xffffffffffffffffn
+const I64_MIN = -0x8000000000000000n
+const I64_MAX = 0x7fffffffffffffffn
+const I64_SIGN_BIT = 0x8000000000000000n
+
+class ByteWriter {
+	private buf: Uint8Array
+	private len = 0
+
+	constructor(capacity = 256) {
+		this.buf = new Uint8Array(capacity)
+	}
+
+	private grow(need: number): void {
+		if (this.len + need <= this.buf.length) {
+			return
+		}
+		let capacity = this.buf.length * 2
+		while (capacity < this.len + need) {
+			capacity *= 2
+		}
+		const next = new Uint8Array(capacity)
+		next.set(this.buf.subarray(0, this.len))
+		this.buf = next
+	}
+
+	u8(value: number): void {
+		this.grow(1)
+		this.buf[this.len] = value
+		this.len += 1
+	}
+
+	bytes(raw: Uint8Array): void {
+		this.grow(raw.length)
+		this.buf.set(raw, this.len)
+		this.len += raw.length
+	}
+
+	u16le(value: number): void {
+		this.grow(2)
+		this.buf[this.len] = value & 0xff
+		this.buf[this.len + 1] = (value >>> 8) & 0xff
+		this.len += 2
+	}
+
+	u32le(value: number): void {
+		this.grow(4)
+		this.buf[this.len] = value & 0xff
+		this.buf[this.len + 1] = (value >>> 8) & 0xff
+		this.buf[this.len + 2] = (value >>> 16) & 0xff
+		this.buf[this.len + 3] = (value >>> 24) & 0xff
+		this.len += 4
+	}
+
+	u64le(value: bigint): void {
+		if (value < 0n || value > U64_MAX) {
+			throw errors.new(`u64 out of range: ${value}`)
+		}
+		this.grow(8)
+		let v = value
+		for (let i = 0; i < 8; i++) {
+			this.buf[this.len + i] = Number(v & 0xffn)
+			v >>= 8n
+		}
+		this.len += 8
+	}
+
+	i64le(value: bigint): void {
+		if (value < I64_MIN || value > I64_MAX) {
+			throw errors.new(`i64 out of range: ${value}`)
+		}
+		this.u64le(value & U64_MAX)
+	}
+
+	u64be(value: bigint): void {
+		if (value < 0n || value > U64_MAX) {
+			throw errors.new(`u64 out of range: ${value}`)
+		}
+		this.grow(8)
+		let v = value
+		for (let i = 7; i >= 0; i--) {
+			this.buf[this.len + i] = Number(v & 0xffn)
+			v >>= 8n
+		}
+		this.len += 8
+	}
+
+	/** The engine's sign-flipped big-endian i64 (lexicographic = numeric). */
+	i64beFlipped(value: bigint): void {
+		if (value < I64_MIN || value > I64_MAX) {
+			throw errors.new(`i64 out of range: ${value}`)
+		}
+		this.u64be((value & U64_MAX) ^ I64_SIGN_BIT)
+	}
+
+	finish(): Uint8Array {
+		return this.buf.slice(0, this.len)
+	}
+}
+
+interface ReadFailure {
+	fail(what: string): never
+}
+
+class ByteReader {
+	private readonly buf: Uint8Array
+	private pos = 0
+	private readonly refusal: ReadFailure
+
+	constructor(buf: Uint8Array, refusal: ReadFailure) {
+		this.buf = buf
+		this.refusal = refusal
+	}
+
+	remaining(): number {
+		return this.buf.length - this.pos
+	}
+
+	private take(count: number, what: string): Uint8Array {
+		if (this.pos + count > this.buf.length) {
+			this.refusal.fail(what)
+		}
+		const out = this.buf.subarray(this.pos, this.pos + count)
+		this.pos += count
+		return out
+	}
+
+	u8(what: string): number {
+		const raw = this.take(1, what)
+		const byte = raw[0]
+		if (byte === undefined) {
+			this.refusal.fail(what)
+		}
+		return byte
+	}
+
+	bytes(count: number, what: string): Uint8Array {
+		return new Uint8Array(this.take(count, what))
+	}
+
+	u16le(what: string): number {
+		const raw = this.take(2, what)
+		return (raw[0] ?? 0) | ((raw[1] ?? 0) << 8)
+	}
+
+	u32le(what: string): number {
+		const raw = this.take(4, what)
+		return (((raw[0] ?? 0) | ((raw[1] ?? 0) << 8) | ((raw[2] ?? 0) << 16)) + (raw[3] ?? 0) * 0x1000000) >>> 0
+	}
+
+	u64le(what: string): bigint {
+		const raw = this.take(8, what)
+		let value = 0n
+		for (let i = 7; i >= 0; i--) {
+			value = (value << 8n) | BigInt(raw[i] ?? 0)
+		}
+		return value
+	}
+
+	i64le(what: string): bigint {
+		const unsigned = this.u64le(what)
+		return unsigned > I64_MAX ? unsigned - (U64_MAX + 1n) : unsigned
+	}
+}
+
+function bytesEqual(a: Uint8Array, b: Uint8Array): boolean {
+	if (a.length !== b.length) {
+		return false
+	}
+	for (let i = 0; i < a.length; i++) {
+		if (a[i] !== b[i]) {
+			return false
+		}
+	}
+	return true
+}
+
+function bytesCompare(a: Uint8Array, b: Uint8Array): number {
+	const shared = Math.min(a.length, b.length)
+	for (let i = 0; i < shared; i++) {
+		const delta = (a[i] ?? 0) - (b[i] ?? 0)
+		if (delta !== 0) {
+			return delta
+		}
+	}
+	return a.length - b.length
+}
+
+const HEX_DIGITS = "0123456789abcdef"
+
+function toHex(bytes: Uint8Array): string {
+	let out = ""
+	for (const byte of bytes) {
+		out += HEX_DIGITS[byte >>> 4]
+		out += HEX_DIGITS[byte & 0xf]
+	}
+	return out
+}
+
+function fromHex(hex: string): Uint8Array {
+	if (hex.length % 2 !== 0 || /[^0-9a-f]/.test(hex)) {
+		throw errors.new(`not lowercase hex: ${hex}`)
+	}
+	const out = new Uint8Array(hex.length / 2)
+	for (let i = 0; i < out.length; i++) {
+		out[i] = Number.parseInt(hex.slice(i * 2, i * 2 + 2), 16)
+	}
+	return out
+}
+
+const utf8Encoder = new TextEncoder()
+const utf8StrictDecoder = new TextDecoder("utf-8", { fatal: true })
+
+export {
+	ByteReader,
+	ByteWriter,
+	bytesCompare,
+	bytesEqual,
+	fromHex,
+	I64_MAX,
+	I64_MIN,
+	toHex,
+	U64_MAX,
+	utf8Encoder,
+	utf8StrictDecoder
+}

package/src/chain.ts

@@ -0,0 +1,109 @@
+/**
+ * The chain sidecar (50): the per-braid split and chain position in
+ * `dir/chain.json`, written atomically (temp + rename, fsync). A floor
+ * cache, never a truth the store reconciles against — recovery is the
+ * catch-up loop, and the one wholeness check lives at the replica.
+ * `pending` is the writer's one extra field: the encoded batch bytes a
+ * local commit owes the log, present until its slot exists.
+ */
+
+import * as crypto from "node:crypto"
+import * as fs from "node:fs/promises"
+import * as path from "node:path"
+import * as errors from "@superbuilders/errors"
+import { utf8StrictDecoder } from "#bytes.ts"
+
+interface ChainEntry {
+	readonly g: bigint
+	readonly prev: string
+	readonly ts: bigint
+}
+
+interface PendingBatch {
+	readonly braid: string
+	readonly gen: bigint
+	readonly bytes: Uint8Array
+}
+
+interface Sidecar {
+	readonly chain: ReadonlyMap<string, ChainEntry>
+	readonly pending: PendingBatch | null
+}
+
+function renderSidecar(sidecar: Sidecar): string {
+	const braids = [...sidecar.chain.keys()].sort()
+	const chain = braids
+		.map(function renderEntry(braid) {
+			const entry = sidecar.chain.get(braid)
+			if (entry === undefined) {
+				throw errors.new(`sidecar chain lost braid ${braid}`)
+			}
+			return `"${braid}":{"g":${entry.g},"prev":"${entry.prev}","ts":${entry.ts}}`
+		})
+		.join(",")
+	const pending =
+		sidecar.pending === null
+			? "null"
+			: `{"braid":"${sidecar.pending.braid}","gen":${sidecar.pending.gen},"bytes":"${Buffer.from(sidecar.pending.bytes).toString("base64")}"}`
+	return `{"v":2,"chain":{${chain}},"pending":${pending}}`
+}
+
+function parseSidecar(text: string): Sidecar {
+	const parsed = JSON.parse(text) as {
+		v: number
+		chain: Record<string, { g: number; prev: string; ts: number }>
+		pending: { braid: string; gen: number; bytes: string } | null
+	}
+	if (parsed.v !== 2 || typeof parsed.chain !== "object" || parsed.chain === null) {
+		throw errors.new("sidecar is not a v2 chain file")
+	}
+	const chain = new Map<string, ChainEntry>()
+	for (const [braid, entry] of Object.entries(parsed.chain)) {
+		chain.set(braid, { g: BigInt(entry.g), prev: entry.prev, ts: BigInt(entry.ts) })
+	}
+	const pending =
+		parsed.pending === null
+			? null
+			: {
+					braid: parsed.pending.braid,
+					gen: BigInt(parsed.pending.gen),
+					bytes: new Uint8Array(Buffer.from(parsed.pending.bytes, "base64"))
+				}
+	return { chain, pending }
+}
+
+async function readSidecar(file: string): Promise<Sidecar | null> {
+	const read = await errors.try(fs.readFile(file))
+	if (read.error) {
+		return null
+	}
+	return parseSidecar(utf8StrictDecoder.decode(read.data))
+}
+
+async function writeSidecar(file: string, sidecar: Sidecar): Promise<void> {
+	const dir = path.dirname(file)
+	await fs.mkdir(dir, { recursive: true })
+	const temp = path.join(dir, `.chain-${process.pid}-${crypto.randomBytes(4).toString("hex")}`)
+	const handle = await fs.open(temp, "wx")
+	const written = await errors.try(
+		(async function writeAll() {
+			await handle.writeFile(renderSidecar(sidecar))
+			await handle.sync()
+		})()
+	)
+	await handle.close()
+	if (written.error) {
+		await fs.rm(temp, { force: true })
+		throw errors.wrap(written.error, `write sidecar ${file}`)
+	}
+	await fs.rename(temp, file)
+	const dirHandle = await fs.open(dir, "r")
+	const synced = await errors.try(dirHandle.sync())
+	await dirHandle.close()
+	if (synced.error) {
+		throw errors.wrap(synced.error, `fsync sidecar directory ${dir}`)
+	}
+}
+
+export type { ChainEntry, PendingBatch, Sidecar }
+export { readSidecar, renderSidecar, writeSidecar }