@bjornpagen/bumbledb-log 0.17.0 → 0.18.0

package/src/errors.ts

@@ -8,6 +8,8 @@
  */
 
 import * as errors from "@superbuilders/errors"
+import type { Braid } from "#descriptor.ts"
+import type { Generation } from "#keys.ts"
 
 /**
  * Typed refusal of a protocol object before any apply: batch shape,
@@ -54,7 +56,7 @@ type RefusalCause =
 	| { readonly kind: "UnknownOpKind"; readonly op: number; readonly opKind: number }
 	| { readonly kind: "UnknownRelation"; readonly op: number; readonly relation: number }
 	| { readonly kind: "ClosedRelation"; readonly op: number; readonly relation: number }
-	| { readonly kind: "OpRelationOutsideBraid"; readonly op: number; readonly relation: number; readonly braid: string }
+	| { readonly kind: "OpRelationOutsideBraid"; readonly op: number; readonly relation: number; readonly braid: Braid }
 	| { readonly kind: "TagMismatch"; readonly relation: string; readonly row: number; readonly field: string }
 	| { readonly kind: "BoolByte"; readonly relation: string; readonly row: number; readonly field: string }
 	| { readonly kind: "InvalidUtf8"; readonly relation: string; readonly row: number; readonly field: string }
@@ -66,14 +68,14 @@ type RefusalCause =
 	| { readonly kind: "CheckpointShape" }
 	| { readonly kind: "CheckpointBraids"; readonly carried: readonly string[]; readonly derived: readonly string[] }
 	| { readonly kind: "CheckpointDigest"; readonly expected: string; readonly computed: string }
-	| { readonly kind: "NoOpSlot"; readonly braid: string; readonly slot: bigint; readonly writer: bigint }
+	| { readonly kind: "NoOpSlot"; readonly braid: Braid; readonly slot: Generation; readonly writer: bigint }
 
 type ChainCause = "prev" | "slot" | "timestamp"
 
 interface ChainMismatchData {
 	readonly cause: ChainCause
-	readonly braid: string
-	readonly slot: bigint
+	readonly braid: Braid
+	readonly slot: Generation
 	readonly writer: bigint
 }
 
@@ -83,10 +85,10 @@ type ContentionCause =
 			readonly statement: string
 			readonly determinants: ReadonlyArray<Readonly<Record<string, unknown>>>
 	  }
-	| { readonly kind: "slot-race"; readonly tip: bigint }
+	| { readonly kind: "slot-race"; readonly tip: Generation }
 
 interface ContentionData {
-	readonly braid: string
+	readonly braid: Braid
 	readonly cause: ContentionCause
 }
 

package/src/index.ts

@@ -3,16 +3,16 @@
  * bumbledb, a thin peer of the engine SDK. Public surface: the pure
  * protocol pair (`encodeBatch`/`decodeBatch`, `braidsOf`) mirrored
  * byte-exactly against the Rust driver, the five-verb object store
- * with its tier-1 `fsStore`, and `openReplica`/`openWriter`/
+ * with its tier-1 `fsStore`, in-process `memStore`, and the `s3Store` AWS S3 client, and `openReplica`/`openWriter`/
  * `openTenants` composed from the engine SDK's own verbs — the replica
  * hands out the SDK's `Db`, and no engine surface is duplicated.
  */
 
 export type { Braid } from "#braids.ts"
-export { braidsOf, serialAtStatementsOf } from "#braids.ts"
-export type { BatchHeader, BatchOp, ChainPosition, DecodedBatch } from "#codec.ts"
+export { braid, braidsOf, serialAtStatementsOf } from "#braids.ts"
+export type { BatchHeader, ChainEntry, DecodedBatch, Op } from "#codec.ts"
 export { decodeBatch, encodeBatch, verifyChain } from "#codec.ts"
-export type { LogDescriptor, LogTheory, SerialStatement } from "#descriptor.ts"
+export type { Descriptor, SerialStatement, Theory } from "#descriptor.ts"
 export { descriptorOf } from "#descriptor.ts"
 export type { ChainCause, ChainMismatchData, ContentionCause, ContentionData, RefusalCause } from "#errors.ts"
 export {
@@ -27,12 +27,14 @@ export {
 	ErrStore,
 	refusalOf
 } from "#errors.ts"
+export type { Generation, StoreKey } from "#keys.ts"
+export { generation, storeKey } from "#keys.ts"
 export type { OpenReplicaOptions, Replica } from "#replica.ts"
 export { openReplica } from "#replica.ts"
-export type { Create, Fetched, ObjectStore, Poll, Swap } from "#store.ts"
-export { fsStore } from "#store.ts"
+export type { Create, Etag, Fetched, ObjectStore, Poll, S3Config, S3Credentials, Swap } from "#store.ts"
+export { etag, fsStore, memStore, s3Store } from "#store.ts"
 export type { OpenTenantsOptions, Tenants } from "#tenants.ts"
 export { openTenants } from "#tenants.ts"
-export type { LogInterval, LogValue } from "#value.ts"
-export type { BraidOutcome, Commit, CommitSplit, Durability, LogBatch, Writer } from "#writer.ts"
+export type { Interval, Value } from "#value.ts"
+export type { Batch, BraidOutcome, Commit, CommitSplit, Durability, Writer } from "#writer.ts"
 export { openWriter } from "#writer.ts"

package/src/keys.ts

@@ -1,34 +1,88 @@
 /**
- * The key layout (10): generation numbers zero-padded lowercase hex,
- * 16 chars; a prefix is a store; a tenant is a prefix under `t/`.
+ * The key layout: generation numbers zero-padded lowercase hex, 16
+ * chars; a prefix is a store; a tenant is a prefix under `t/`. A
+ * StoreKey is parsed here, once — the verbs take the proof and never
+ * re-check. An empty prefix joins as the rest alone, so a leading
+ * slash is unrepresentable.
  */
 
-function hex16(g: bigint): string {
+import * as errors from "@superbuilders/errors"
+import { U64_MAX } from "#bytes.ts"
+import type { Braid } from "#descriptor.ts"
+
+/** Suffix of the per-key pid-lockfile. A segment wearing it cannot be a key. */
+const LOCK_SUFFIX = ".lock"
+
+declare const storeKeyBrand: unique symbol
+type StoreKey = string & { readonly [storeKeyBrand]: typeof storeKeyBrand }
+
+declare const generationBrand: unique symbol
+type Generation = bigint & { readonly [generationBrand]: typeof generationBrand }
+
+function storeKey(raw: string): StoreKey {
+	if (raw.length === 0 || raw.startsWith("/") || raw.endsWith("/")) {
+		throw errors.new(`store key is not a slash path: ${raw}`)
+	}
+	for (const segment of raw.split("/")) {
+		if (segment.length === 0 || segment === "." || segment === "..") {
+			throw errors.new(`store key segment is illegal: ${raw}`)
+		}
+		if (segment.endsWith(LOCK_SUFFIX)) {
+			throw errors.new(`store key collides with the lockfile suffix: ${raw}`)
+		}
+	}
+	return raw as StoreKey
+}
+
+function generation(raw: bigint): Generation {
+	if (raw < 0n || raw > U64_MAX) {
+		throw errors.new(`generation is a u64: ${raw}`)
+	}
+	return raw as Generation
+}
+
+function hex16(g: Generation): string {
 	return g.toString(16).padStart(16, "0")
 }
 
-function manifestKey(prefix: string): string {
-	return `${prefix}/manifest.json`
+function assemble(prefix: string, rest: string): StoreKey {
+	return storeKey(prefix.length === 0 ? rest : `${prefix}/${rest}`)
 }
 
-function logKey(prefix: string, braid: string, g: bigint): string {
-	return `${prefix}/log/${braid}/${hex16(g)}`
+function manifestKey(prefix: string): StoreKey {
+	return assemble(prefix, "manifest.json")
 }
 
-function checkpointMdbKey(prefix: string, digest: string): string {
-	return `${prefix}/ckpt/${digest}.mdb`
+function logKey(prefix: string, braid: Braid, g: Generation): StoreKey {
+	return assemble(prefix, `log/${braid}/${hex16(g)}`)
 }
 
-function checkpointJsonKey(prefix: string, digest: string): string {
-	return `${prefix}/ckpt/${digest}.json`
+function checkpointMdbKey(prefix: string, digest: string): StoreKey {
+	return assemble(prefix, `ckpt/${digest}.mdb`)
 }
 
-function idsKey(prefix: string, relation: number, field: number): string {
-	return `${prefix}/ids/${relation.toString(16).padStart(8, "0")}/${field.toString(16).padStart(4, "0")}`
+function checkpointJsonKey(prefix: string, digest: string): StoreKey {
+	return assemble(prefix, `ckpt/${digest}.json`)
+}
+
+function idsKey(prefix: string, relation: number, field: number): StoreKey {
+	return assemble(prefix, `ids/${relation.toString(16).padStart(8, "0")}/${field.toString(16).padStart(4, "0")}`)
 }
 
 function tenantPrefix(root: string, tenant: string): string {
-	return `${root}/t/${tenant}`
+	return assemble(root, `t/${tenant}`)
 }
 
-export { checkpointJsonKey, checkpointMdbKey, hex16, idsKey, logKey, manifestKey, tenantPrefix }
+export type { Generation, StoreKey }
+export {
+	checkpointJsonKey,
+	checkpointMdbKey,
+	generation,
+	hex16,
+	idsKey,
+	LOCK_SUFFIX,
+	logKey,
+	manifestKey,
+	storeKey,
+	tenantPrefix
+}

package/src/manifest.ts

@@ -8,7 +8,11 @@
 
 import * as errors from "@superbuilders/errors"
 import { utf8Encoder, utf8StrictDecoder } from "#bytes.ts"
+import type { Braid } from "#descriptor.ts"
+import { braid } from "#descriptor.ts"
 import { refuse } from "#errors.ts"
+import type { Generation } from "#keys.ts"
+import { generation } from "#keys.ts"
 
 interface Manifest {
 	readonly fingerprint: string
@@ -59,13 +63,13 @@ function parseManifest(bytes: Uint8Array): Manifest {
 }
 
 interface CheckpointHead {
-	readonly g: bigint
+	readonly g: Generation
 	readonly hash: string
 	readonly ts: bigint
 }
 
 interface CheckpointFacts {
-	readonly braids: ReadonlyMap<string, CheckpointHead>
+	readonly braids: ReadonlyMap<Braid, CheckpointHead>
 	readonly catalog: string
 	readonly writer: bigint
 	readonly prev: string | null
@@ -83,19 +87,19 @@ function parseCheckpoint(bytes: Uint8Array): CheckpointFacts {
 	if (typeof rawBraids !== "object" || rawBraids === null) {
 		refuse({ kind: "CheckpointShape" }, "checkpoint json carries no braids map")
 	}
-	const braids = new Map<string, CheckpointHead>()
-	for (const [braid, head] of Object.entries(rawBraids as Record<string, unknown>)) {
+	const braids = new Map<Braid, CheckpointHead>()
+	for (const [name, head] of Object.entries(rawBraids as Record<string, unknown>)) {
 		if (typeof head !== "object" || head === null) {
-			refuse({ kind: "CheckpointShape" }, `checkpoint braid ${braid} head is not an object`)
+			refuse({ kind: "CheckpointShape" }, `checkpoint braid ${name} head is not an object`)
 		}
 		const headRecord = head as Record<string, unknown>
 		const g = headRecord.g
 		const hash = headRecord.hash
 		const ts = headRecord.ts
 		if (typeof g !== "number" || typeof ts !== "number" || typeof hash !== "string" || !HEX64.test(hash)) {
-			refuse({ kind: "CheckpointShape" }, `checkpoint braid ${braid} head is malformed`)
+			refuse({ kind: "CheckpointShape" }, `checkpoint braid ${name} head is malformed`)
 		}
-		braids.set(braid, { g: BigInt(g), hash, ts: BigInt(ts) })
+		braids.set(braid(name), { g: generation(BigInt(g)), hash, ts: BigInt(ts) })
 	}
 	const catalog = record.catalog
 	const writer = record.writer

package/src/replica.ts

@@ -14,18 +14,19 @@ import type { Db, Fact, MemberRelation, Schema, SchemaRelations, WriteOutcome }
 import { internalBlake3, Db as SdkDb } from "@bjornpagen/bumbledb"
 import * as errors from "@superbuilders/errors"
 import { bytesEqual, toHex } from "#bytes.ts"
-import type { ChainEntry, PendingBatch } from "#chain.ts"
+import type { ChainEntry, Pending } from "#chain.ts"
 import { readSidecar, writeSidecar } from "#chain.ts"
-import type { BatchOp } from "#codec.ts"
+import type { Op } from "#codec.ts"
 import { decodeBatch, encodeBatch, verifyChain } from "#codec.ts"
-import type { LogDescriptor, RelationInfo } from "#descriptor.ts"
+import type { Braid, Descriptor, RelationInfo } from "#descriptor.ts"
 import { descriptorOf } from "#descriptor.ts"
 import { ErrGapDetected, ErrReplayDiverged, refuse } from "#errors.ts"
-import { checkpointJsonKey, checkpointMdbKey, logKey, manifestKey } from "#keys.ts"
+import type { Generation } from "#keys.ts"
+import { checkpointJsonKey, checkpointMdbKey, generation, logKey, manifestKey } from "#keys.ts"
 import type { CheckpointFacts } from "#manifest.ts"
 import { parseCheckpoint, parseManifest, renderManifest } from "#manifest.ts"
-import type { ObjectStore } from "#store.ts"
-import type { LogValue } from "#value.ts"
+import type { Etag, ObjectStore } from "#store.ts"
+import type { Value } from "#value.ts"
 
 const ZERO_HASH = "0".repeat(64)
 
@@ -45,9 +46,9 @@ interface OpenReplicaOptions<Rels extends SchemaRelations> {
 
 interface Replica<Rels extends SchemaRelations> extends AsyncDisposable {
 	readonly db: Db<Rels>
-	readonly vector: ReadonlyMap<string, bigint>
-	refresh(braid?: string): Promise<ReadonlyMap<string, bigint>>
-	waitFor(vector: ReadonlyMap<string, bigint>): Promise<void>
+	readonly vector: ReadonlyMap<Braid, Generation>
+	refresh(braid?: Braid): Promise<ReadonlyMap<Braid, Generation>>
+	waitFor(vector: ReadonlyMap<Braid, Generation>): Promise<void>
 }
 
 interface Core<Rels extends SchemaRelations> {
@@ -55,13 +56,13 @@ interface Core<Rels extends SchemaRelations> {
 	readonly prefix: string
 	readonly dir: string
 	readonly theory: Schema<Rels>
-	readonly descriptor: LogDescriptor
+	readonly descriptor: Descriptor
 	db: Db<Rels>
-	chain: Map<string, ChainEntry>
-	pending: PendingBatch | null
-	pendingOps: readonly BatchOp[] | null
+	chain: Map<Braid, ChainEntry>
+	pending: Pending | null
+	pendingOps: readonly Op[] | null
 	pendingApplied: boolean
-	manifestEtag: string | null
+	manifestEtag: Etag | null
 	checkpoint: CheckpointFacts | null
 	checkpointDigest: string | null
 	passes: number
@@ -117,15 +118,15 @@ function storePath<Rels extends SchemaRelations>(core: Core<Rels>): string {
 	return path.join(core.dir, core.storeName)
 }
 
-function zeroChain(descriptor: LogDescriptor): Map<string, ChainEntry> {
-	const chain = new Map<string, ChainEntry>()
-	for (const braid of descriptor.braidMembers.keys()) {
-		chain.set(braid, { g: 0n, prev: ZERO_HASH, ts: 0n })
+function zeroChain(descriptor: Descriptor): Map<Braid, ChainEntry> {
+	const chain = new Map<Braid, ChainEntry>()
+	for (const id of descriptor.braidMembers.keys()) {
+		chain.set(id, { g: generation(0n), prev: ZERO_HASH, ts: 0n })
 	}
 	return chain
 }
 
-function chainEntry<Rels extends SchemaRelations>(core: Core<Rels>, braid: string): ChainEntry {
+function chainEntry<Rels extends SchemaRelations>(core: Core<Rels>, braid: Braid): ChainEntry {
 	const entry = core.chain.get(braid)
 	if (entry === undefined) {
 		throw errors.new(`braid ${braid} is not derived from this theory`)
@@ -166,7 +167,7 @@ async function clearPending<Rels extends SchemaRelations>(core: Core<Rels>): Pro
 function factOf<Rels extends SchemaRelations>(
 	core: Core<Rels>,
 	relation: RelationInfo,
-	row: readonly LogValue[]
+	row: readonly Value[]
 ): Record<string, unknown> {
 	const fact: Record<string, unknown> = {}
 	relation.fields.forEach(function liftCell(field, ordinal) {
@@ -191,7 +192,7 @@ function factOf<Rels extends SchemaRelations>(
 }
 
 /** One `db.write` applying ops in listed order, rows in listed order. */
-function applyOps<Rels extends SchemaRelations>(core: Core<Rels>, ops: readonly BatchOp[]): WriteOutcome<Rels, number> {
+function applyOps<Rels extends SchemaRelations>(core: Core<Rels>, ops: readonly Op[]): WriteOutcome<Rels, number> {
 	return core.db.write(function applyBatch(tx) {
 		for (const op of ops) {
 			const info = core.descriptor.relationByName.get(op.relation)
@@ -225,8 +226,8 @@ type SlotApply = { readonly tag: "applied"; readonly generation: bigint } | { re
  */
 async function applySlot<Rels extends SchemaRelations>(
 	core: Core<Rels>,
-	braid: string,
-	slot: bigint,
+	braid: Braid,
+	slot: Generation,
 	bytes: Uint8Array,
 	phase: ApplyPhase
 ): Promise<SlotApply> {
@@ -256,7 +257,7 @@ async function applySlot<Rels extends SchemaRelations>(
  * The gc floor rule (10): below the current checkpoint's vector a 404 is
  * a collected hole, at or above it the honest tip.
  */
-function holeAt<Rels extends SchemaRelations>(core: Core<Rels>, braid: string, next: bigint): boolean {
+function holeAt<Rels extends SchemaRelations>(core: Core<Rels>, braid: Braid, next: Generation): boolean {
 	if (core.checkpoint === null) {
 		return false
 	}
@@ -269,11 +270,11 @@ function holeAt<Rels extends SchemaRelations>(core: Core<Rels>, braid: string, n
 
 async function catchUpBraid<Rels extends SchemaRelations>(
 	core: Core<Rels>,
-	braid: string,
+	braid: Braid,
 	phase: ApplyPhase
 ): Promise<"tip" | "discard"> {
 	for (;;) {
-		const next = chainEntry(core, braid).g + 1n
+		const next = generation(chainEntry(core, braid).g + 1n)
 		const fetched = await core.store.get(logKey(core.prefix, braid, next))
 		if (fetched === null) {
 			if (holeAt(core, braid, next)) {
@@ -322,7 +323,7 @@ function wholenessHolds<Rels extends SchemaRelations>(core: Core<Rels>): boolean
 async function adoptManifest<Rels extends SchemaRelations>(
 	core: Core<Rels>,
 	bytes: Uint8Array,
-	etag: string
+	etag: Etag
 ): Promise<void> {
 	const manifest = parseManifest(bytes)
 	if (manifest.fingerprint !== core.descriptor.fingerprint) {
@@ -502,7 +503,7 @@ async function resolvePendingAtOpen<Rels extends SchemaRelations>(core: Core<Rel
 		return
 	}
 	const decoded = errors.trySync(function decodePending() {
-		return decodeBatch(core.descriptor, (core.pending as PendingBatch).bytes)
+		return decodeBatch(core.descriptor, (core.pending as Pending).bytes)
 	})
 	if (decoded.error) {
 		await clearPending(core)
@@ -529,10 +530,7 @@ async function resolvePendingAtOpen<Rels extends SchemaRelations>(core: Core<Rel
  * absorbs it (L10's idempotent replay), and everything else takes the
  * one loss path's re-judgment at the tip.
  */
-async function resolveColdPending<Rels extends SchemaRelations>(
-	core: Core<Rels>,
-	pending: PendingBatch
-): Promise<void> {
+async function resolveColdPending<Rels extends SchemaRelations>(core: Core<Rels>, pending: Pending): Promise<void> {
 	const decoded = errors.trySync(function decodePending() {
 		return decodeBatch(core.descriptor, pending.bytes)
 	})
@@ -583,7 +581,7 @@ async function openCore<Rels extends SchemaRelations>(options: OpenReplicaOption
 	const existing = await newestStoreDir(core.dir)
 	const sidecar = await readSidecar(sidecarPath(core))
 	let opened = false
-	let coldPending: PendingBatch | null = null
+	let coldPending: Pending | null = null
 	if (existing !== null) {
 		core.storeName = existing
 		const openedDb = await errors.try(SdkDb.open(storePath(core), options.theory))
@@ -592,7 +590,7 @@ async function openCore<Rels extends SchemaRelations>(options: OpenReplicaOption
 			core.chain = new Map(sidecar.chain)
 			for (const braid of descriptor.braidMembers.keys()) {
 				if (!core.chain.has(braid)) {
-					core.chain.set(braid, { g: 0n, prev: ZERO_HASH, ts: 0n })
+					core.chain.set(braid, { g: generation(0n), prev: ZERO_HASH, ts: 0n })
 				}
 			}
 			core.pending = sidecar.pending
@@ -645,7 +643,7 @@ async function repairDiscard<Rels extends SchemaRelations>(core: Core<Rels>): Pr
  */
 async function readdressPending<Rels extends SchemaRelations>(
 	core: Core<Rels>,
-	ops: readonly BatchOp[],
+	ops: readonly Op[],
 	writerId: bigint
 ): Promise<void> {
 	const first = ops[0]
@@ -666,14 +664,14 @@ async function readdressPending<Rels extends SchemaRelations>(
 		{
 			fingerprint: core.descriptor.fingerprint,
 			braid,
-			braidGen: entry.g + 1n,
+			braidGen: generation(entry.g + 1n),
 			prev: entry.prev,
 			writer: writerId,
 			timestamp
 		},
 		ops
 	)
-	core.pending = { braid, gen: entry.g + 1n, bytes }
+	core.pending = { braid, gen: generation(entry.g + 1n), bytes }
 	core.pendingOps = ops
 	core.pendingApplied = true
 	await persistSidecar(core)
@@ -683,15 +681,15 @@ function maxBigint(a: bigint, b: bigint): bigint {
 	return a > b ? a : b
 }
 
-function vectorOf<Rels extends SchemaRelations>(core: Core<Rels>): ReadonlyMap<string, bigint> {
-	const vector = new Map<string, bigint>()
-	for (const [braid, entry] of core.chain) {
-		vector.set(braid, entry.g)
+function vectorOf<Rels extends SchemaRelations>(core: Core<Rels>): ReadonlyMap<Braid, Generation> {
+	const vector = new Map<Braid, Generation>()
+	for (const [id, entry] of core.chain) {
+		vector.set(id, entry.g)
 	}
 	return vector
 }
 
-function dominates(have: ReadonlyMap<string, bigint>, want: ReadonlyMap<string, bigint>): boolean {
+function dominates(have: ReadonlyMap<Braid, Generation>, want: ReadonlyMap<Braid, Generation>): boolean {
 	for (const [braid, generation] of want) {
 		if ((have.get(braid) ?? -1n) < generation) {
 			return false
@@ -700,7 +698,7 @@ function dominates(have: ReadonlyMap<string, bigint>, want: ReadonlyMap<string,
 	return true
 }
 
-async function refreshPass<Rels extends SchemaRelations>(core: Core<Rels>, braid?: string): Promise<void> {
+async function refreshPass<Rels extends SchemaRelations>(core: Core<Rels>, braid?: Braid): Promise<void> {
 	core.passes += 1
 	if (core.passes % HEARTBEAT_PASSES === 0) {
 		await refreshManifest(core, false)
@@ -729,7 +727,7 @@ async function openReplica<Rels extends SchemaRelations>(options: OpenReplicaOpt
 		get vector() {
 			return vectorOf(core)
 		},
-		async refresh(braid?: string) {
+		async refresh(braid?: Braid) {
 			return withGate(core, async function refreshBody() {
 				if (core.closed) {
 					throw errors.new("replica is disposed")