@bjornpagen/bumbledb-log 0.17.0

package/src/store.ts

@@ -0,0 +1,292 @@
+/**
+ * The object-store capability: exactly five verbs, outcomes as sums,
+ * infrastructure failures on the ErrStore channel. `fsStore` is tier-1,
+ * not a dev double — deployment case 5's production backend.
+ *
+ * The one on-disk protocol, shared with the Rust driver: create-only
+ * publishes an exclusive synced temp with link(2), where EEXIST is the
+ * honest exists; the etag is the blake3 of the content, lowercase hex,
+ * computed on every read and never stored; `putSwap` serializes under a
+ * pid-lockfile beside the key, published with the same exclusive
+ * temp-plus-link discipline so it can never exist without its body (the
+ * owner pid) — a contender breaks the lock iff the owner is dead, which
+ * is sound on one machine only, fsStore's load-bearing deployment law.
+ * `created` and `swapped` resolve only after fsync of the object file
+ * and its parent directory.
+ */
+
+import * as fs from "node:fs/promises"
+import * as path from "node:path"
+import { internalBlake3 } from "@bjornpagen/bumbledb"
+import * as errors from "@superbuilders/errors"
+import { toHex } from "#bytes.ts"
+import { wrapStore } from "#errors.ts"
+
+interface Fetched {
+	readonly bytes: Uint8Array
+	readonly etag: string
+}
+
+type Poll = { readonly tag: "unchanged" } | { readonly tag: "changed"; readonly fetched: Fetched }
+
+type Create = { readonly tag: "created"; readonly etag: string } | { readonly tag: "exists" }
+
+type Swap = { readonly tag: "swapped"; readonly etag: string } | { readonly tag: "moved" }
+
+interface ObjectStore {
+	/** GET; null on 404. */
+	get(key: string): Promise<Fetched | null>
+	/** GET with If-None-Match — the cheap manifest poll. */
+	getIfChanged(key: string, etag: string): Promise<Poll>
+	/** PUT with If-None-Match: * — the log-slot arbitration primitive. */
+	putCreate(key: string, bytes: Uint8Array): Promise<Create>
+	/** PUT with If-Match — the manifest CAS primitive. */
+	putSwap(key: string, bytes: Uint8Array, etag: string): Promise<Swap>
+	/** DELETE, unconditional — the gc verb's tool. */
+	delete(key: string): Promise<void>
+}
+
+/** Suffix of the per-key pid-lockfile that serializes `putSwap`. */
+const LOCK_SUFFIX = ".lock"
+
+/** Ceiling of the jittered wait between probes of a live-held lock. */
+const LOCK_RETRY_MS = 10
+
+function contentEtag(bytes: Uint8Array): string {
+	return toHex(new Uint8Array(internalBlake3(bytes)))
+}
+
+function checkKey(key: string): void {
+	if (key.length === 0 || key.startsWith("/") || key.endsWith("/")) {
+		throw errors.new(`store key is not a slash path: ${key}`)
+	}
+	for (const segment of key.split("/")) {
+		if (segment.length === 0 || segment === "." || segment === "..") {
+			throw errors.new(`store key segment is illegal: ${key}`)
+		}
+		if (segment.endsWith(LOCK_SUFFIX)) {
+			throw errors.new(`store key collides with the lockfile suffix: ${key}`)
+		}
+	}
+}
+
+function codeOf(error: Error): string | undefined {
+	return (error as NodeJS.ErrnoException).code
+}
+
+async function fsyncDir(dir: string): Promise<void> {
+	const handle = await fs.open(dir, "r")
+	const synced = await errors.try(handle.sync())
+	await handle.close()
+	if (synced.error) {
+		throw errors.wrap(synced.error, `fsync directory ${dir}`)
+	}
+}
+
+let tempSeq = 0
+
+/** Write `bytes` to a fresh `wx` temp file beside `target` and fsync it.
+ *  The caller publishes the synced temp atomically. */
+async function syncedTemp(target: string, bytes: Uint8Array): Promise<string> {
+	const dir = path.dirname(target)
+	await fs.mkdir(dir, { recursive: true })
+	tempSeq += 1
+	const temp = path.join(dir, `.${path.basename(target)}.tmp.${process.pid}.${tempSeq}`)
+	const handle = await fs.open(temp, "wx")
+	const written = await errors.try(
+		(async function writeAll() {
+			await handle.writeFile(bytes)
+			await handle.sync()
+		})()
+	)
+	await handle.close()
+	if (written.error) {
+		await fs.rm(temp, { force: true })
+		throw errors.wrap(written.error, `write ${target}`)
+	}
+	return temp
+}
+
+/** link(2) is the exclusivity primitive: rename replaces an existing
+ *  destination, so it cannot arbitrate create-only, while link fails
+ *  atomically with EEXIST across processes — exactly the
+ *  If-None-Match: * contract. */
+async function publishLink(temp: string, dest: string): Promise<"linked" | "occupied"> {
+	const linked = await errors.try(fs.link(temp, dest))
+	if (linked.error === undefined) {
+		return "linked"
+	}
+	if (codeOf(linked.error) === "EEXIST") {
+		return "occupied"
+	}
+	throw linked.error
+}
+
+function pidAlive(pid: number): boolean {
+	const probed = errors.trySync(function probe() {
+		process.kill(pid, 0)
+	})
+	if (probed.error === undefined) {
+		return true
+	}
+	return codeOf(probed.error) !== "ESRCH"
+}
+
+async function acquireLock(lockPath: string): Promise<void> {
+	const body = new TextEncoder().encode(String(process.pid))
+	for (;;) {
+		const temp = await syncedTemp(lockPath, body)
+		const published = await errors.try(publishLink(temp, lockPath))
+		await fs.rm(temp, { force: true })
+		if (published.error) {
+			throw published.error
+		}
+		if (published.data === "linked") {
+			return
+		}
+		const read = await errors.try(fs.readFile(lockPath, "utf8"))
+		if (read.error) {
+			if (codeOf(read.error) === "ENOENT") {
+				continue
+			}
+			throw read.error
+		}
+		const owner = read.data.trim()
+		if (!/^\d+$/.test(owner)) {
+			throw errors.new(`lockfile body is not a pid: ${lockPath}`)
+		}
+		if (pidAlive(Number.parseInt(owner, 10))) {
+			await new Promise(function later(resolve) {
+				setTimeout(resolve, Math.random() * LOCK_RETRY_MS)
+			})
+		} else {
+			await fs.rm(lockPath, { force: true })
+		}
+	}
+}
+
+async function releaseLock(lockPath: string): Promise<void> {
+	await fs.rm(lockPath, { force: true })
+}
+
+/** The five verbs over one local directory. One machine is load-bearing. */
+function fsStore(root: string): ObjectStore {
+	const rootPath = path.resolve(root)
+
+	function objectPath(key: string): string {
+		checkKey(key)
+		return path.join(rootPath, ...key.split("/"))
+	}
+
+	async function readFetched(target: string): Promise<Fetched | null> {
+		const read = await errors.try(fs.readFile(target))
+		if (read.error) {
+			if (codeOf(read.error) === "ENOENT") {
+				return null
+			}
+			throw read.error
+		}
+		const bytes = new Uint8Array(read.data)
+		return { bytes, etag: contentEtag(bytes) }
+	}
+
+	return {
+		async get(key) {
+			const target = objectPath(key)
+			const read = await errors.try(readFetched(target))
+			if (read.error) {
+				throw wrapStore(read.error, `get ${key}`)
+			}
+			return read.data
+		},
+
+		async getIfChanged(key, etag) {
+			const target = objectPath(key)
+			const read = await errors.try(readFetched(target))
+			if (read.error) {
+				throw wrapStore(read.error, `getIfChanged ${key}`)
+			}
+			if (read.data === null) {
+				throw wrapStore(errors.new("poll target absent"), `getIfChanged ${key}`)
+			}
+			if (read.data.etag === etag) {
+				return { tag: "unchanged" }
+			}
+			return { tag: "changed", fetched: read.data }
+		},
+
+		async putCreate(key, bytes) {
+			const target = objectPath(key)
+			const ran = await errors.try(
+				(async function createBody(): Promise<Create> {
+					const temp = await syncedTemp(target, bytes)
+					const published = await errors.try(publishLink(temp, target))
+					await fs.rm(temp, { force: true })
+					if (published.error) {
+						throw published.error
+					}
+					if (published.data === "occupied") {
+						return { tag: "exists" }
+					}
+					await fsyncDir(path.dirname(target))
+					return { tag: "created", etag: contentEtag(bytes) }
+				})()
+			)
+			if (ran.error) {
+				throw wrapStore(ran.error, `putCreate ${key}`)
+			}
+			return ran.data
+		},
+
+		async putSwap(key, bytes, etag) {
+			const target = objectPath(key)
+			const lock = `${target}${LOCK_SUFFIX}`
+			const ran = await errors.try(
+				(async function swapBody(): Promise<Swap> {
+					await acquireLock(lock)
+					const swapped = await errors.try(
+						(async function underLock(): Promise<Swap> {
+							const current = await readFetched(target)
+							if (current === null || current.etag !== etag) {
+								return { tag: "moved" }
+							}
+							const temp = await syncedTemp(target, bytes)
+							const renamed = await errors.try(fs.rename(temp, target))
+							if (renamed.error) {
+								await fs.rm(temp, { force: true })
+								throw renamed.error
+							}
+							await fsyncDir(path.dirname(target))
+							return { tag: "swapped", etag: contentEtag(bytes) }
+						})()
+					)
+					await releaseLock(lock)
+					if (swapped.error) {
+						throw swapped.error
+					}
+					return swapped.data
+				})()
+			)
+			if (ran.error) {
+				throw wrapStore(ran.error, `putSwap ${key}`)
+			}
+			return ran.data
+		},
+
+		async delete(key) {
+			const target = objectPath(key)
+			const ran = await errors.try(
+				(async function deleteBody() {
+					await fs.rm(target, { force: true })
+					await fs.rm(`${target}${LOCK_SUFFIX}`, { force: true })
+				})()
+			)
+			if (ran.error) {
+				throw wrapStore(ran.error, `delete ${key}`)
+			}
+		}
+	}
+}
+
+export type { Create, Fetched, ObjectStore, Poll, Swap }
+export { fsStore }

package/src/tenants.ts

@@ -0,0 +1,140 @@
+/**
+ * Per-tenant replicas (50, case 4): an LRU of replicas keyed by tenant
+ * id — a tenant is a prefix (`<root>/t/<tenant>`), eviction closes and
+ * deletes the local dir (the disposable law), and `t/_shared` is
+ * pinned. Braids shard within a tenant; tenants shard the world.
+ */
+
+import * as fs from "node:fs/promises"
+import * as path from "node:path"
+import type { Schema, SchemaRelations } from "@bjornpagen/bumbledb"
+import * as errors from "@superbuilders/errors"
+import { tenantPrefix } from "#keys.ts"
+import type { Replica } from "#replica.ts"
+import { openReplica } from "#replica.ts"
+import type { ObjectStore } from "#store.ts"
+
+const PINNED_TENANT = "_shared"
+
+interface OpenTenantsOptions<Rels extends SchemaRelations> {
+	readonly store: ObjectStore
+	readonly root: string
+	readonly dir: string
+	readonly theory: Schema<Rels>
+	/** 50's 400 MB gate: checkpoint + working set per instance. Advisory,
+	 *  measured once at each tenant's open — a replica that grows after
+	 *  admission is not re-weighed until it is evicted and re-opened. */
+	readonly budgetBytes?: number
+	readonly maxOpen?: number
+}
+
+interface Tenants<Rels extends SchemaRelations> extends AsyncDisposable {
+	get(tenant: string): Promise<Replica<Rels>>
+}
+
+interface TenantSlot<Rels extends SchemaRelations> {
+	readonly replica: Replica<Rels>
+	readonly dir: string
+	bytes: number
+	lastUsed: number
+}
+
+async function directoryBytes(dir: string): Promise<number> {
+	let total = 0
+	const listed = await errors.try(fs.readdir(dir, { withFileTypes: true, recursive: true }))
+	if (listed.error) {
+		return 0
+	}
+	for (const entry of listed.data) {
+		if (entry.isFile()) {
+			const stat = await errors.try(fs.stat(path.join(entry.parentPath, entry.name)))
+			if (stat.error === undefined) {
+				total += stat.data.size
+			}
+		}
+	}
+	return total
+}
+
+function checkTenantId(tenant: string): void {
+	if (!/^[A-Za-z0-9._-]+$/.test(tenant)) {
+		throw errors.new(`tenant id is not a single path segment: ${tenant}`)
+	}
+}
+
+function openTenants<Rels extends SchemaRelations>(options: OpenTenantsOptions<Rels>): Tenants<Rels> {
+	const budgetBytes = options.budgetBytes ?? 400_000_000
+	const maxOpen = options.maxOpen ?? 32
+	const open = new Map<string, TenantSlot<Rels>>()
+	let tick = 0
+	let closed = false
+
+	async function evictUntilWithin(): Promise<void> {
+		for (;;) {
+			let bytes = 0
+			for (const slot of open.values()) {
+				bytes += slot.bytes
+			}
+			if (open.size <= maxOpen && bytes <= budgetBytes) {
+				return
+			}
+			let victim: string | null = null
+			let oldest = Number.POSITIVE_INFINITY
+			for (const [tenant, slot] of open) {
+				if (tenant === PINNED_TENANT) {
+					continue
+				}
+				if (slot.lastUsed < oldest) {
+					oldest = slot.lastUsed
+					victim = tenant
+				}
+			}
+			if (victim === null) {
+				return
+			}
+			const slot = open.get(victim)
+			open.delete(victim)
+			if (slot !== undefined) {
+				await slot.replica[Symbol.asyncDispose]()
+				await fs.rm(slot.dir, { recursive: true, force: true })
+			}
+		}
+	}
+
+	return {
+		async get(tenant) {
+			if (closed) {
+				throw errors.new("tenants pool is disposed")
+			}
+			checkTenantId(tenant)
+			tick += 1
+			const hit = open.get(tenant)
+			if (hit !== undefined) {
+				hit.lastUsed = tick
+				return hit.replica
+			}
+			const dir = path.join(options.dir, tenant)
+			const replica = await openReplica({
+				store: options.store,
+				prefix: tenantPrefix(options.root, tenant),
+				dir,
+				theory: options.theory
+			})
+			const slot: TenantSlot<Rels> = { replica, dir, bytes: await directoryBytes(dir), lastUsed: tick }
+			open.set(tenant, slot)
+			await evictUntilWithin()
+			return replica
+		},
+
+		async [Symbol.asyncDispose]() {
+			closed = true
+			for (const slot of open.values()) {
+				await slot.replica[Symbol.asyncDispose]()
+			}
+			open.clear()
+		}
+	}
+}
+
+export type { OpenTenantsOptions, Tenants }
+export { openTenants }

package/src/value.ts

@@ -0,0 +1,285 @@
+/**
+ * The driver's raw-value vocabulary and its two canonical encodings:
+ * the command codec's tagged little-endian form (20's tag table, the one
+ * shared tagged-value encoding the footprint keys also hash), and the
+ * engine's big-endian order-preserving literal form the fingerprint
+ * mirror reproduces. Values here are always RAW — strings as UTF-8,
+ * never intern ids — which is what makes footprint keys state-independent.
+ */
+
+import type { ValueTypeSpec } from "@bjornpagen/bumbledb"
+import * as errors from "@superbuilders/errors"
+import type { ByteReader, ByteWriter } from "#bytes.ts"
+import { bytesEqual, I64_MAX, I64_MIN, U64_MAX, utf8Encoder, utf8StrictDecoder } from "#bytes.ts"
+
+interface LogInterval {
+	readonly start: bigint
+	readonly end: bigint
+}
+
+type LogValue = boolean | bigint | string | Uint8Array | LogInterval
+
+/** 20's tag table: the codec's own numbering, normative to the byte. */
+const TAG = {
+	bool: 0,
+	u64: 1,
+	i64: 2,
+	string: 3,
+	fixedBytes: 4,
+	interval: 5,
+	fixedInterval: 6
+} as const
+
+function isInterval(value: LogValue): value is LogInterval {
+	return typeof value === "object" && !(value instanceof Uint8Array)
+}
+
+function wireTagOf(type: ValueTypeSpec): number {
+	switch (type.kind) {
+		case "bool":
+			return TAG.bool
+		case "u64":
+			return TAG.u64
+		case "i64":
+			return TAG.i64
+		case "string":
+			return TAG.string
+		case "fixedBytes":
+			return TAG.fixedBytes
+		case "interval":
+			return type.width === undefined ? TAG.interval : TAG.fixedInterval
+	}
+}
+
+function checkAgainst(context: string, type: ValueTypeSpec, value: LogValue): void {
+	switch (type.kind) {
+		case "bool": {
+			if (typeof value !== "boolean") {
+				throw errors.new(`${context}: expected boolean`)
+			}
+			return
+		}
+		case "u64": {
+			if (typeof value !== "bigint" || value < 0n || value > U64_MAX) {
+				throw errors.new(`${context}: expected u64 bigint`)
+			}
+			return
+		}
+		case "i64": {
+			if (typeof value !== "bigint" || value < I64_MIN || value > I64_MAX) {
+				throw errors.new(`${context}: expected i64 bigint`)
+			}
+			return
+		}
+		case "string": {
+			if (typeof value !== "string" || !value.isWellFormed()) {
+				throw errors.new(`${context}: expected well-formed string`)
+			}
+			return
+		}
+		case "fixedBytes": {
+			if (!(value instanceof Uint8Array) || value.length !== type.len) {
+				throw errors.new(`${context}: expected ${type.len}-byte Uint8Array`)
+			}
+			return
+		}
+		case "interval": {
+			if (!(typeof value === "object") || value instanceof Uint8Array) {
+				throw errors.new(`${context}: expected interval value`)
+			}
+			const lo = type.element === "u64" ? 0n : I64_MIN
+			const hi = type.element === "u64" ? U64_MAX : I64_MAX
+			if (value.start < lo || value.end > hi || value.start >= value.end) {
+				throw errors.new(`${context}: interval bounds out of range or empty`)
+			}
+			if (type.width !== undefined && value.end - value.start !== type.width) {
+				throw errors.new(`${context}: interval width must be ${type.width}`)
+			}
+			return
+		}
+	}
+}
+
+/** The codec's tagged form: `tag u8` + payload, at the field's layout. */
+function writeTagged(out: ByteWriter, type: ValueTypeSpec, value: LogValue): void {
+	switch (type.kind) {
+		case "bool": {
+			out.u8(TAG.bool)
+			out.u8(value === true ? 1 : 0)
+			return
+		}
+		case "u64": {
+			out.u8(TAG.u64)
+			out.u64le(value as bigint)
+			return
+		}
+		case "i64": {
+			out.u8(TAG.i64)
+			out.i64le(value as bigint)
+			return
+		}
+		case "string": {
+			const raw = utf8Encoder.encode(value as string)
+			out.u8(TAG.string)
+			out.u32le(raw.length)
+			out.bytes(raw)
+			return
+		}
+		case "fixedBytes": {
+			out.u8(TAG.fixedBytes)
+			out.bytes(value as Uint8Array)
+			return
+		}
+		case "interval": {
+			const interval = value as LogInterval
+			if (type.width === undefined) {
+				out.u8(TAG.interval)
+				if (type.element === "u64") {
+					out.u64le(interval.start)
+					out.u64le(interval.end)
+				} else {
+					out.i64le(interval.start)
+					out.i64le(interval.end)
+				}
+				return
+			}
+			out.u8(TAG.fixedInterval)
+			if (type.element === "u64") {
+				out.u64le(interval.start)
+			} else {
+				out.i64le(interval.start)
+			}
+			return
+		}
+	}
+}
+
+/** The value parser's refusal channel: one method per proved cause,
+ *  matching the Rust decoder's cross-implementation identities. */
+interface TaggedRefusal {
+	badTag(expected: number, actual: number): never
+	boolByte(byte: number): never
+	invalidUtf8(): never
+	emptyInterval(): never
+	intervalOverflow(): never
+}
+
+/** Full parse at the layout's type; every illegal byte is a typed refusal. */
+function readTagged(reader: ByteReader, type: ValueTypeSpec, refusal: TaggedRefusal): LogValue {
+	const expected = wireTagOf(type)
+	const tag = reader.u8("value tag")
+	if (tag !== expected) {
+		refusal.badTag(expected, tag)
+	}
+	switch (type.kind) {
+		case "bool": {
+			const byte = reader.u8("bool payload")
+			if (byte > 1) {
+				refusal.boolByte(byte)
+			}
+			return byte === 1
+		}
+		case "u64":
+			return reader.u64le("u64 payload")
+		case "i64":
+			return reader.i64le("i64 payload")
+		case "string": {
+			const len = reader.u32le("string length")
+			const raw = reader.bytes(len, "string payload")
+			const decoded = errors.trySync(function decodeUtf8() {
+				return utf8StrictDecoder.decode(raw)
+			})
+			if (decoded.error) {
+				refusal.invalidUtf8()
+			}
+			return decoded.data
+		}
+		case "fixedBytes":
+			return reader.bytes(type.len, "fixedBytes payload")
+		case "interval": {
+			if (type.width === undefined) {
+				const start = type.element === "u64" ? reader.u64le("interval start") : reader.i64le("interval start")
+				const end = type.element === "u64" ? reader.u64le("interval end") : reader.i64le("interval end")
+				if (start >= end) {
+					refusal.emptyInterval()
+				}
+				return { start, end }
+			}
+			const start = type.element === "u64" ? reader.u64le("interval start") : reader.i64le("interval start")
+			const end = start + type.width
+			if (type.element === "u64" ? end > U64_MAX : end > I64_MAX) {
+				refusal.intervalOverflow()
+			}
+			return { start, end }
+		}
+	}
+}
+
+function valuesEqual(a: LogValue, b: LogValue): boolean {
+	if (typeof a === "boolean" || typeof a === "bigint" || typeof a === "string") {
+		return a === b
+	}
+	if (a instanceof Uint8Array) {
+		return b instanceof Uint8Array && bytesEqual(a, b)
+	}
+	if (typeof b !== "object" || b instanceof Uint8Array) {
+		return false
+	}
+	return a.start === b.start && a.end === b.end
+}
+
+/**
+ * The engine's canonical big-endian literal form (`encode_literal`) —
+ * the fingerprint mirror's alphabet. Strings never reach this encoder:
+ * the fingerprint's `put_literal` length-prefixes them separately, and
+ * closed ground axioms with string columns are the mirror's recorded gap.
+ */
+function writeCanonicalLiteral(out: ByteWriter, type: ValueTypeSpec, value: LogValue): void {
+	switch (type.kind) {
+		case "bool": {
+			out.u8(value === true ? 1 : 0)
+			return
+		}
+		case "u64": {
+			out.u64be(value as bigint)
+			return
+		}
+		case "i64": {
+			out.i64beFlipped(value as bigint)
+			return
+		}
+		case "string":
+			throw errors.new("canonical literal: strings are length-prefixed by the caller, never encoded here")
+		case "fixedBytes": {
+			const raw = value as Uint8Array
+			const padded = Math.ceil(type.len / 8) * 8
+			out.bytes(raw)
+			for (let i = raw.length; i < padded; i++) {
+				out.u8(0)
+			}
+			return
+		}
+		case "interval": {
+			const interval = value as LogInterval
+			if (type.width !== undefined) {
+				if (type.element === "u64") {
+					out.u64be(interval.start)
+				} else {
+					out.i64beFlipped(interval.start)
+				}
+				return
+			}
+			if (type.element === "u64") {
+				out.u64be(interval.start)
+				out.u64be(interval.end)
+			} else {
+				out.i64beFlipped(interval.start)
+				out.i64beFlipped(interval.end)
+			}
+			return
+		}
+	}
+}
+
+export type { LogInterval, LogValue, TaggedRefusal }
+export { checkAgainst, isInterval, readTagged, TAG, valuesEqual, wireTagOf, writeCanonicalLiteral, writeTagged }