@bjornpagen/bumbledb-log 0.18.0 → 0.19.0

package/src/index.ts

@@ -10,7 +10,7 @@
 
 export type { Braid } from "#braids.ts"
 export { braid, braidsOf, serialAtStatementsOf } from "#braids.ts"
-export type { BatchHeader, ChainEntry, DecodedBatch, Op } from "#codec.ts"
+export type { BatchHeader, ChainEntry, DecodedBatch, EncodeHeader, Op } from "#codec.ts"
 export { decodeBatch, encodeBatch, verifyChain } from "#codec.ts"
 export type { Descriptor, SerialStatement, Theory } from "#descriptor.ts"
 export { descriptorOf } from "#descriptor.ts"
@@ -21,14 +21,25 @@ export {
 	ErrChainMismatch,
 	ErrContention,
 	ErrGapDetected,
+	ErrManifestMissing,
 	ErrRefused,
 	ErrReplayDiverged,
 	ErrSpanningCommit,
 	ErrStore,
+	isManifestMissing,
 	refusalOf
 } from "#errors.ts"
 export type { Generation, StoreKey } from "#keys.ts"
-export { generation, storeKey } from "#keys.ts"
+export {
+	CKPT_SCRATCH_LEASE,
+	encodeCkptScratch,
+	generation,
+	LEASE_NAMESPACE,
+	parseCkptScratch,
+	scratchCkptDigest,
+	scratchCkptName,
+	storeKey
+} from "#keys.ts"
 export type { OpenReplicaOptions, Replica } from "#replica.ts"
 export { openReplica } from "#replica.ts"
 export type { Create, Etag, Fetched, ObjectStore, Poll, S3Config, S3Credentials, Swap } from "#store.ts"
@@ -36,5 +47,7 @@ export { etag, fsStore, memStore, s3Store } from "#store.ts"
 export type { OpenTenantsOptions, Tenants } from "#tenants.ts"
 export { openTenants } from "#tenants.ts"
 export type { Interval, Value } from "#value.ts"
+export type { CheckpointOrder, VectorParse } from "#vector.ts"
+export { Overflow, Vector } from "#vector.ts"
 export type { Batch, BraidOutcome, Commit, CommitSplit, Durability, Writer } from "#writer.ts"
 export { openWriter } from "#writer.ts"

package/src/keys.ts

@@ -3,37 +3,164 @@
  * chars; a prefix is a store; a tenant is a prefix under `t/`. A
  * StoreKey is parsed here, once — the verbs take the proof and never
  * re-check. An empty prefix joins as the rest alone, so a leading
- * slash is unrepresentable.
+ * slash is unrepresentable. Temps and leases live under a reserved
+ * tilde-family first segment no StoreKey can spell — ASCII `~` and
+ * its lookalikes — so those namespaces stay disjoint from every
+ * honest key. Format characters, line and paragraph separators, and
+ * space separators cannot hide a reserved prefix or a `.lock` suffix.
  */
 
 import * as errors from "@superbuilders/errors"
-import { U64_MAX } from "#bytes.ts"
+import { regex } from "arkregex"
+import type { Digest32 } from "#bytes.ts"
+import { digest32, hex32, U64_MAX } from "#bytes.ts"
 import type { Braid } from "#descriptor.ts"
 
-/** Suffix of the per-key pid-lockfile. A segment wearing it cannot be a key. */
+/** A segment wearing this suffix — after format characters are stripped — is not a key. */
 const LOCK_SUFFIX = ".lock"
 
+/** Reserved first-segment names no StoreKey can spell. Temps and leases live here. */
+const TEMP_NAMESPACE = "~tmp"
+const LEASE_NAMESPACE = "~lease"
+
+/** ASCII tilde and lookalikes that can spell a reserved first segment. */
+const TILDE_FAMILY = new Set([
+	"~",
+	"\u02DC",
+	"\u02F7",
+	"\u1FC0",
+	"\u2053",
+	"\u223C",
+	"\u223D",
+	"\u223F",
+	"\u2E1E",
+	"\u2E1F",
+	"\u301C",
+	"\u3030",
+	"\uFE4B",
+	"\uFE4F",
+	"\uFF5E"
+])
+
+const FORMAT_OR_SEPARATOR = regex("[\\p{Cc}\\p{Cf}\\p{Zl}\\p{Zp}\\p{Zs}]", "u")
+const FORMAT_CHARS = regex("\\p{Cf}", "gu")
+const SLASH_TRIM = regex("^/+|/+$", "g")
+
 declare const storeKeyBrand: unique symbol
 type StoreKey = string & { readonly [storeKeyBrand]: typeof storeKeyBrand }
 
 declare const generationBrand: unique symbol
 type Generation = bigint & { readonly [generationBrand]: typeof generationBrand }
 
+function firstCodePoint(seg: string): string | undefined {
+	for (const ch of seg) {
+		return ch
+	}
+	return undefined
+}
+
+function tildeFamilyPrefix(seg: string): boolean {
+	const first = firstCodePoint(seg)
+	if (first === undefined) {
+		return false
+	}
+	return TILDE_FAMILY.has(first) || first.normalize("NFKC") === "~"
+}
+
+function stripFormat(seg: string): string {
+	return seg.replace(FORMAT_CHARS, "")
+}
+
+/** One path segment of a key or a tenant id: the same grammar. */
+function segmentOk(seg: string): boolean {
+	if (seg.length === 0 || seg.includes("/") || seg === "." || seg === "..") {
+		return false
+	}
+	if (FORMAT_OR_SEPARATOR.test(seg)) {
+		return false
+	}
+	const stripped = stripFormat(seg)
+	if (stripped.length === 0 || stripped === "." || stripped === "..") {
+		return false
+	}
+	return !tildeFamilyPrefix(stripped) && !stripped.endsWith(LOCK_SUFFIX)
+}
+
 function storeKey(raw: string): StoreKey {
-	if (raw.length === 0 || raw.startsWith("/") || raw.endsWith("/")) {
+	const wellFormed = raw.length > 0 && !raw.startsWith("/") && !raw.endsWith("/") && raw.split("/").every(segmentOk)
+	if (!wellFormed) {
 		throw errors.new(`store key is not a slash path: ${raw}`)
 	}
-	for (const segment of raw.split("/")) {
-		if (segment.length === 0 || segment === "." || segment === "..") {
-			throw errors.new(`store key segment is illegal: ${raw}`)
-		}
-		if (segment.endsWith(LOCK_SUFFIX)) {
-			throw errors.new(`store key collides with the lockfile suffix: ${raw}`)
-		}
-	}
 	return raw as StoreKey
 }
 
+/** Empty, or a StoreKey spelling (the same segment grammar, no leading or trailing slash). */
+function parsePrefix(raw: string): string {
+	if (raw.length === 0) {
+		return ""
+	}
+	return storeKey(raw.replace(SLASH_TRIM, ""))
+}
+
+function isReservedName(name: string): boolean {
+	return tildeFamilyPrefix(stripFormat(name))
+}
+
+function reservedTemp(pid: number, seq: number): string {
+	return `${TEMP_NAMESPACE}/${String(pid)}.${String(seq)}`
+}
+
+function reservedLease(key: string, token: bigint): string {
+	return `${LEASE_NAMESPACE}/${key}/${String(token)}`
+}
+
+/** A slash path whose first segment is `~tmp` or `~lease`. Not a StoreKey. */
+function reservedName(raw: string): string {
+	const segs = raw.split("/")
+	if (segs.length === 0 || segs.some((seg) => seg.length === 0 || seg === "." || seg === "..")) {
+		throw errors.new(`reserved name is not a slash path: ${raw}`)
+	}
+	const first = segs[0]
+	if (first !== TEMP_NAMESPACE && first !== LEASE_NAMESPACE) {
+		throw errors.new(`reserved name is not under ${TEMP_NAMESPACE} or ${LEASE_NAMESPACE}: ${raw}`)
+	}
+	return raw
+}
+
+/** Known scratch-lease document under `~lease`. One path, no LIST. */
+const CKPT_SCRATCH_LEASE = "ckpt-scratch"
+
+/** The reserved scratch name: `~lease/ckpt-scratch`. */
+function scratchCkptName(): string {
+	return reservedName(`${LEASE_NAMESPACE}/${CKPT_SCRATCH_LEASE}`)
+}
+
+const SCRATCH_VERSION = 3
+
+/** The scratch-lease body: version byte 3, then the 32-byte digest. */
+function encodeCkptScratch(digest: Digest32): Uint8Array {
+	const out = new Uint8Array(33)
+	out[0] = SCRATCH_VERSION
+	out.set(digest, 1)
+	return out
+}
+
+/** The digest a scratch-lease body names, or null. */
+function scratchCkptDigest(bytes: Uint8Array): Digest32 | null {
+	if (bytes.length !== 33) {
+		return null
+	}
+	const version = bytes[0]
+	if (version !== SCRATCH_VERSION) {
+		return null
+	}
+	return digest32(bytes.subarray(1))
+}
+
+function parseCkptScratch(bytes: Uint8Array): Digest32 | null {
+	return scratchCkptDigest(bytes)
+}
+
 function generation(raw: bigint): Generation {
 	if (raw < 0n || raw > U64_MAX) {
 		throw errors.new(`generation is a u64: ${raw}`)
@@ -50,19 +177,19 @@ function assemble(prefix: string, rest: string): StoreKey {
 }
 
 function manifestKey(prefix: string): StoreKey {
-	return assemble(prefix, "manifest.json")
+	return assemble(prefix, "manifest")
 }
 
 function logKey(prefix: string, braid: Braid, g: Generation): StoreKey {
 	return assemble(prefix, `log/${braid}/${hex16(g)}`)
 }
 
-function checkpointMdbKey(prefix: string, digest: string): StoreKey {
-	return assemble(prefix, `ckpt/${digest}.mdb`)
+function checkpointMdbKey(prefix: string, digest: Digest32): StoreKey {
+	return assemble(prefix, `ckpt/${hex32(digest)}.mdb`)
 }
 
-function checkpointJsonKey(prefix: string, digest: string): StoreKey {
-	return assemble(prefix, `ckpt/${digest}.json`)
+function ckptDocKey(prefix: string, digest: Digest32): StoreKey {
+	return assemble(prefix, `ckpt/${hex32(digest)}`)
 }
 
 function idsKey(prefix: string, relation: number, field: number): StoreKey {
@@ -70,19 +197,35 @@ function idsKey(prefix: string, relation: number, field: number): StoreKey {
 }
 
 function tenantPrefix(root: string, tenant: string): string {
+	if (!segmentOk(tenant)) {
+		throw errors.new(`tenant id is not a single path segment: ${tenant}`)
+	}
 	return assemble(root, `t/${tenant}`)
 }
 
 export type { Generation, StoreKey }
 export {
-	checkpointJsonKey,
+	CKPT_SCRATCH_LEASE,
 	checkpointMdbKey,
+	ckptDocKey,
+	encodeCkptScratch,
 	generation,
 	hex16,
 	idsKey,
+	isReservedName,
+	LEASE_NAMESPACE,
 	LOCK_SUFFIX,
 	logKey,
 	manifestKey,
+	parseCkptScratch,
+	parsePrefix,
+	reservedLease,
+	reservedName,
+	reservedTemp,
+	scratchCkptDigest,
+	scratchCkptName,
+	segmentOk,
 	storeKey,
+	TEMP_NAMESPACE,
 	tenantPrefix
 }

package/src/manifest.ts

@@ -1,117 +1,209 @@
 /**
- * The protocol's one mutable object (10): canonical single-line UTF-8
- * JSON, strict parse, field order fixed — strictness is proved by
- * re-rendering the parse and demanding byte equality, so a
- * non-canonical manifest is a typed refusal, never a tolerated variant.
- * The checkpoint json beside it is immutable and digest-keyed.
+ * The protocol's one mutable object: a v:3 binary record — version
+ * byte 3, a branded Digest32 fingerprint, an optional branded Digest32
+ * checkpoint. The checkpoint record is digest-keyed and immutable:
+ * version byte 3, a count-bounded braid roster (id, g, hash, ts) in
+ * ascending id order, catalog digest, writer, optional prev. Every
+ * digest field is Digest32. The content address is blake3 of the
+ * record bytes. Seed audits the catalog claim when a checkpoint is
+ * present.
  */
 
 import * as errors from "@superbuilders/errors"
-import { utf8Encoder, utf8StrictDecoder } from "#bytes.ts"
+import type { Digest32 } from "#bytes.ts"
+import { ByteReader, ByteWriter, bytesEqual, hex32 } from "#bytes.ts"
 import type { Braid } from "#descriptor.ts"
-import { braid } from "#descriptor.ts"
+import { braidHex } from "#descriptor.ts"
 import { refuse } from "#errors.ts"
 import type { Generation } from "#keys.ts"
 import { generation } from "#keys.ts"
+import { Vector } from "#vector.ts"
+
+const DOC_VERSION = 3
+const U32_MAX = 0xffffffffn
+/** braid u32 + g u64 + hash 32 + ts u64 */
+const MIN_HEAD_BYTES = 52n
 
 interface Manifest {
-	readonly fingerprint: string
-	readonly checkpoint: string | null
+	readonly fingerprint: Digest32
+	readonly checkpoint: Digest32 | null
 }
 
-const HEX64 = /^[0-9a-f]{64}$/
-
-function renderManifest(manifest: Manifest): Uint8Array {
-	const checkpoint = manifest.checkpoint === null ? "null" : `"${manifest.checkpoint}"`
-	return utf8Encoder.encode(`{"v":2,"fingerprint":"${manifest.fingerprint}","checkpoint":${checkpoint}}`)
+function readerOf(bytes: Uint8Array): ByteReader {
+	return new ByteReader(bytes, {
+		fail(what: string): never {
+			refuse({ kind: "Malformed", at: bytes.length }, `document truncated at ${what}`)
+		}
+	})
 }
 
-function parseManifest(bytes: Uint8Array): Manifest {
-	const decoded = errors.trySync(function decodeText() {
-		return utf8StrictDecoder.decode(bytes)
-	})
-	if (decoded.error) {
-		refuse({ kind: "ManifestShape" }, "manifest is not UTF-8")
+function finish(reader: ByteReader, at: string): void {
+	if (reader.remaining() !== 0) {
+		refuse({ kind: "Malformed", at: reader.remaining() }, `${reader.remaining()} trailing bytes after ${at}`)
 	}
-	const parsed = errors.trySync(function parseJson() {
-		return JSON.parse(decoded.data) as unknown
-	})
-	if (parsed.error || typeof parsed.data !== "object" || parsed.data === null) {
-		refuse({ kind: "ManifestShape" }, "manifest is not a JSON object")
+}
+
+function refuseUnbacked(count: bigint, remaining: number, minItem: bigint, at: string): void {
+	if (count === 0n) {
+		return
 	}
-	const record = parsed.data as Record<string, unknown>
-	if (typeof record.v !== "number") {
-		refuse({ kind: "ManifestShape" }, "manifest carries no version")
+	if (minItem === 0n || BigInt(remaining) / minItem < count) {
+		refuse({ kind: "Malformed", at: remaining }, `declared ${at} ${count} outruns the remaining ${remaining} bytes`)
 	}
-	if (record.v !== 2) {
-		refuse({ kind: "ManifestVersion", version: record.v }, `manifest version ${record.v}, consumers refuse ≠ 2`)
+}
+
+function writeOptionalDigest(out: ByteWriter, digest: Digest32 | null): void {
+	if (digest === null) {
+		out.u8(0)
+		return
 	}
-	const fingerprint = record.fingerprint
-	const checkpoint = record.checkpoint
-	if (typeof fingerprint !== "string" || !HEX64.test(fingerprint)) {
-		refuse({ kind: "ManifestShape" }, "manifest fingerprint is not 64 hex")
+	out.u8(1)
+	out.array32(digest)
+}
+
+function readOptionalDigest(reader: ByteReader, at: string): Digest32 | null {
+	const tag = reader.u8(at)
+	if (tag === 0) {
+		return null
 	}
-	if (checkpoint !== null && (typeof checkpoint !== "string" || !HEX64.test(checkpoint))) {
-		refuse({ kind: "ManifestShape" }, "manifest checkpoint is neither null nor 64 hex")
+	if (tag === 1) {
+		return reader.array32(at)
 	}
-	const manifest: Manifest = { fingerprint, checkpoint: checkpoint === null ? null : checkpoint }
-	const canonical = renderManifest(manifest)
-	if (canonical.length !== bytes.length || utf8StrictDecoder.decode(canonical) !== decoded.data) {
-		refuse({ kind: "ManifestShape" }, "manifest is not the canonical single-line rendering")
+	refuse({ kind: "Flags", flags: tag }, `${at} presence is not 0 or 1`)
+}
+
+function readVersion(reader: ByteReader, at: string): void {
+	const version = reader.u8("version")
+	if (version !== DOC_VERSION) {
+		refuse(
+			{ kind: "Version", version },
+			`${at} version ${version}, consumers refuse every version other than ${DOC_VERSION}`
+		)
 	}
-	return manifest
+}
+
+function braidIdOf(id: Braid): number {
+	return Number.parseInt(id.slice(1), 16)
+}
+
+function renderManifest(manifest: Manifest): Uint8Array {
+	const out = new ByteWriter(66)
+	out.u8(DOC_VERSION)
+	out.array32(manifest.fingerprint)
+	writeOptionalDigest(out, manifest.checkpoint)
+	return out.finish()
+}
+
+function parseManifest(bytes: Uint8Array): Manifest {
+	const reader = readerOf(bytes)
+	readVersion(reader, "manifest")
+	const fingerprint = reader.array32("fingerprint")
+	const checkpoint = readOptionalDigest(reader, "checkpoint")
+	finish(reader, "the manifest")
+	return { fingerprint, checkpoint }
 }
 
 interface CheckpointHead {
 	readonly g: Generation
-	readonly hash: string
+	readonly hash: Digest32
 	readonly ts: bigint
 }
 
 interface CheckpointFacts {
 	readonly braids: ReadonlyMap<Braid, CheckpointHead>
-	readonly catalog: string
+	readonly catalog: Digest32
 	readonly writer: bigint
-	readonly prev: string | null
+	readonly prev: Digest32 | null
+	readonly sum: bigint
 }
 
-function parseCheckpoint(bytes: Uint8Array): CheckpointFacts {
-	const parsed = errors.trySync(function parseJson() {
-		return JSON.parse(utf8StrictDecoder.decode(bytes)) as unknown
-	})
-	if (parsed.error || typeof parsed.data !== "object" || parsed.data === null) {
-		refuse({ kind: "CheckpointShape" }, "checkpoint json is not an object")
+function vectorOfHeads(braids: ReadonlyMap<Braid, CheckpointHead>): Vector {
+	const counts = new Map<Braid, bigint>()
+	for (const [id, head] of braids) {
+		counts.set(id, head.g)
+	}
+	return Vector.from(counts)
+}
+
+function checkpointVector(facts: CheckpointFacts): Vector {
+	return vectorOfHeads(facts.braids)
+}
+
+function renderCheckpoint(facts: CheckpointFacts): Uint8Array {
+	const braids = [...facts.braids.keys()].sort()
+	const count = BigInt(braids.length)
+	if (count > U32_MAX) {
+		throw errors.new(`checkpoint braid count ${count} exceeds u32`)
 	}
-	const record = parsed.data as Record<string, unknown>
-	const rawBraids = record.braids
-	if (typeof rawBraids !== "object" || rawBraids === null) {
-		refuse({ kind: "CheckpointShape" }, "checkpoint json carries no braids map")
+	const out = new ByteWriter(1 + 4 + braids.length * 52 + 32 + 8 + 1 + 32)
+	out.u8(DOC_VERSION)
+	out.u32le(Number(count))
+	for (const id of braids) {
+		const head = facts.braids.get(id)
+		if (head === undefined) {
+			throw errors.new(`checkpoint lost braid ${id}`)
+		}
+		out.u32le(braidIdOf(id))
+		out.u64le(head.g)
+		out.array32(head.hash)
+		out.u64le(head.ts)
 	}
+	out.array32(facts.catalog)
+	out.u64le(facts.writer)
+	writeOptionalDigest(out, facts.prev)
+	return out.finish()
+}
+
+function parseCheckpoint(bytes: Uint8Array, known?: ReadonlySet<Braid>): CheckpointFacts {
+	const reader = readerOf(bytes)
+	readVersion(reader, "checkpoint")
+	const count = BigInt(reader.u32le("braid count"))
+	refuseUnbacked(count, reader.remaining(), MIN_HEAD_BYTES, "braid count")
 	const braids = new Map<Braid, CheckpointHead>()
-	for (const [name, head] of Object.entries(rawBraids as Record<string, unknown>)) {
-		if (typeof head !== "object" || head === null) {
-			refuse({ kind: "CheckpointShape" }, `checkpoint braid ${name} head is not an object`)
+	for (let i = 0n; i < count; i++) {
+		const raw = reader.u32le("braid")
+		const name = braidHex(raw)
+		if (known !== undefined && !known.has(name)) {
+			refuse({ kind: "UnknownBraid", braid: raw }, `checkpoint cites unknown braid ${name}`)
 		}
-		const headRecord = head as Record<string, unknown>
-		const g = headRecord.g
-		const hash = headRecord.hash
-		const ts = headRecord.ts
-		if (typeof g !== "number" || typeof ts !== "number" || typeof hash !== "string" || !HEX64.test(hash)) {
-			refuse({ kind: "CheckpointShape" }, `checkpoint braid ${name} head is malformed`)
+		const g = reader.u64le("generation")
+		const hash = reader.array32("hash")
+		const ts = reader.u64le("timestamp")
+		const last = [...braids.keys()].at(-1)
+		if (last !== undefined && last >= name) {
+			refuse(
+				{ kind: "Malformed", at: bytes.length - reader.remaining() },
+				"checkpoint braid roster is not strictly ascending"
+			)
 		}
-		braids.set(braid(name), { g: generation(BigInt(g)), hash, ts: BigInt(ts) })
+		braids.set(name, { g: generation(g), hash, ts })
+	}
+	const catalog = reader.array32("catalog")
+	const writer = reader.u64le("writer")
+	const prev = readOptionalDigest(reader, "prev")
+	finish(reader, "the checkpoint")
+	const summed = vectorOfHeads(braids).sum()
+	if (typeof summed !== "bigint") {
+		refuse({ kind: "Overflow" }, "checkpoint vector sum overflows u64")
 	}
-	const catalog = record.catalog
-	const writer = record.writer
-	const prev = record.prev
-	if (typeof catalog !== "string" || !HEX64.test(catalog) || typeof writer !== "number") {
-		refuse({ kind: "CheckpointShape" }, "checkpoint json catalog or writer is malformed")
+	return { braids, catalog, writer, prev, sum: summed }
+}
+
+/** The seed transition's catalog claim: a present checkpoint
+ *  compares the opened store's `catalog_digest` against the document.
+ *  Genesis (no checkpoint) has no claim. A mismatch names the publisher. */
+function auditCatalog(facts: CheckpointFacts | null, computed: Digest32): void {
+	if (facts === null) {
+		return
 	}
-	if (prev !== null && (typeof prev !== "string" || !HEX64.test(prev))) {
-		refuse({ kind: "CheckpointShape" }, "checkpoint json prev is neither null nor a digest")
+	if (bytesEqual(facts.catalog, computed)) {
+		return
 	}
-	return { braids, catalog, writer: BigInt(writer), prev: prev === null ? null : prev }
+	refuse(
+		{ kind: "CheckpointDigest", expected: hex32(facts.catalog), computed: hex32(computed) },
+		`checkpoint catalog ${hex32(facts.catalog)} disagrees with the opened store ${hex32(computed)}; publisher ${facts.writer}`
+	)
 }
 
 export type { CheckpointFacts, CheckpointHead, Manifest }
-export { parseCheckpoint, parseManifest, renderManifest }
+export { auditCatalog, checkpointVector, DOC_VERSION, parseCheckpoint, parseManifest, renderCheckpoint, renderManifest }