@bjornpagen/bumbledb-log 0.20.0 → 0.20.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/dist/braids.d.ts +28 -0
  2. package/dist/braids.d.ts.map +1 -0
  3. package/{src/braids.ts → dist/braids.js} +22 -29
  4. package/dist/braids.js.map +1 -0
  5. package/dist/bytes.d.ts +26 -0
  6. package/dist/bytes.d.ts.map +1 -0
  7. package/dist/bytes.js +86 -0
  8. package/dist/bytes.js.map +1 -0
  9. package/dist/chain.d.ts +56 -0
  10. package/dist/chain.d.ts.map +1 -0
  11. package/dist/chain.js +137 -0
  12. package/dist/chain.js.map +1 -0
  13. package/dist/codec.d.ts +71 -0
  14. package/dist/codec.d.ts.map +1 -0
  15. package/dist/codec.js +179 -0
  16. package/dist/codec.js.map +1 -0
  17. package/dist/descriptor.d.ts +61 -0
  18. package/dist/descriptor.d.ts.map +1 -0
  19. package/dist/descriptor.js +160 -0
  20. package/dist/descriptor.js.map +1 -0
  21. package/dist/errors.d.ts +202 -0
  22. package/dist/errors.d.ts.map +1 -0
  23. package/dist/errors.js +122 -0
  24. package/dist/errors.js.map +1 -0
  25. package/dist/index.d.ts +34 -0
  26. package/dist/index.d.ts.map +1 -0
  27. package/dist/index.js +24 -0
  28. package/dist/index.js.map +1 -0
  29. package/dist/keys.d.ts +52 -0
  30. package/dist/keys.d.ts.map +1 -0
  31. package/dist/keys.js +166 -0
  32. package/dist/keys.js.map +1 -0
  33. package/dist/manifest.d.ts +42 -0
  34. package/dist/manifest.d.ts.map +1 -0
  35. package/dist/manifest.js +116 -0
  36. package/dist/manifest.js.map +1 -0
  37. package/dist/replica.d.ts +154 -0
  38. package/dist/replica.d.ts.map +1 -0
  39. package/dist/replica.js +820 -0
  40. package/dist/replica.js.map +1 -0
  41. package/dist/store-s3.d.ts +31 -0
  42. package/dist/store-s3.d.ts.map +1 -0
  43. package/dist/store-s3.js +299 -0
  44. package/dist/store-s3.js.map +1 -0
  45. package/dist/store.d.ts +129 -0
  46. package/dist/store.d.ts.map +1 -0
  47. package/dist/store.js +577 -0
  48. package/dist/store.js.map +1 -0
  49. package/dist/tenants.d.ts +56 -0
  50. package/dist/tenants.d.ts.map +1 -0
  51. package/dist/tenants.js +333 -0
  52. package/dist/tenants.js.map +1 -0
  53. package/dist/tilde-family.json +21 -0
  54. package/dist/vector.d.ts +31 -0
  55. package/dist/vector.d.ts.map +1 -0
  56. package/dist/vector.js +87 -0
  57. package/dist/vector.js.map +1 -0
  58. package/dist/writer.d.ts +106 -0
  59. package/dist/writer.d.ts.map +1 -0
  60. package/dist/writer.js +677 -0
  61. package/dist/writer.js.map +1 -0
  62. package/package.json +12 -7
  63. package/src/bytes.ts +0 -115
  64. package/src/chain.ts +0 -181
  65. package/src/codec.ts +0 -252
  66. package/src/descriptor.ts +0 -240
  67. package/src/errors.ts +0 -309
  68. package/src/index.ts +0 -57
  69. package/src/keys.ts +0 -218
  70. package/src/manifest.ts +0 -161
  71. package/src/replica.ts +0 -1061
  72. package/src/store-s3.ts +0 -377
  73. package/src/store.ts +0 -727
  74. package/src/tenants.ts +0 -414
  75. package/src/vector.ts +0 -103
  76. package/src/writer.ts +0 -977
package/src/replica.ts DELETED
@@ -1,1061 +0,0 @@
1
- /**
2
- * The replica: a local store that is a materialized view of the
3
- * braids' prefixes, plus the loop that keeps it current. Disposable by
4
- * construction — the sidecar is a floor cache with one wholeness
5
- * check; recovery IS the catch-up loop (L10). Generation is a total
6
- * function of the chain: Settled sums the vector, Pending is that sum
7
- * plus one. A replica that finds no manifest refuses; only a writer
8
- * births a store.
9
- */
10
-
11
- import * as fs from "node:fs/promises"
12
- import * as path from "node:path"
13
- import type { Db, MemberRelation, Schema, SchemaRelations, WriteOutcome } from "@bjornpagen/bumbledb"
14
- import { factOf, internalBlake3, Db as SdkDb } from "@bjornpagen/bumbledb"
15
- import * as errors from "@superbuilders/errors"
16
- import { parse } from "#braids.ts"
17
- import type { Digest32 } from "#bytes.ts"
18
- import { bytesEqual, digest32, hex32, saturatingAddU64 } from "#bytes.ts"
19
- import type { Chain, ChainEntry, Pending } from "#chain.ts"
20
- import { chainGeneration, chainSum, readSidecar, writeSidecar } from "#chain.ts"
21
- import type { Op } from "#codec.ts"
22
- import { decodeBatch, encodeBatch, verifyChain } from "#codec.ts"
23
- import type { Braid, Descriptor } from "#descriptor.ts"
24
- import { descriptorOf } from "#descriptor.ts"
25
- import { ErrRefused, ErrReplayDiverged, refuse, refuseManifestMissing, wrapStore } from "#errors.ts"
26
- import type { Generation } from "#keys.ts"
27
- import {
28
- CKPT_SCRATCH_LEASE,
29
- checkpointMdbKey,
30
- ckptDocKey,
31
- generation,
32
- LEASE_NAMESPACE,
33
- logKey,
34
- manifestKey,
35
- parseCkptScratch,
36
- TEMP_NAMESPACE
37
- } from "#keys.ts"
38
- import type { CheckpointFacts } from "#manifest.ts"
39
- import { auditCatalog, parseCheckpoint, parseManifest } from "#manifest.ts"
40
- import type { Etag, ObjectStore } from "#store.ts"
41
- import { Vector } from "#vector.ts"
42
-
43
- const ZERO_HASH = digest32(new Uint8Array(32))
44
-
45
- /** The gc-safety heartbeat cadence: every N-th pass re-reads the manifest. */
46
- const HEARTBEAT_PASSES = 16
47
-
48
- /** The re-poll cadence of waitFor, its one consumer — the
49
- * machine-constants table's `wait_for_poll_ms` fact. */
50
- const WAIT_FOR_POLL_MS = 10
51
-
52
- type ReplicaState =
53
- | { readonly tag: "bootstrapped" }
54
- | { readonly tag: "checkpoint-seeded"; readonly catalog: Digest32 }
55
- | { readonly tag: "sidecar-resumed"; readonly floor: ReadonlyMap<Braid, Generation> }
56
-
57
- type RefreshOutcome =
58
- | { readonly tag: "advanced"; readonly vector: ReadonlyMap<Braid, Generation> }
59
- | { readonly tag: "wedged"; readonly braid: Braid; readonly cause: string }
60
- | { readonly tag: "reseed"; readonly cause: string }
61
- | { readonly tag: "refused"; readonly detail: string }
62
-
63
- type Step =
64
- | { readonly tag: "applied" }
65
- | { readonly tag: "tip" }
66
- | { readonly tag: "wedged"; readonly braid: Braid; readonly cause: string }
67
- | { readonly tag: "reseed"; readonly cause: string }
68
-
69
- /** The full waitFor sum: a wedged braid the target needs is an
70
- * outcome, never an infinite poll. */
71
- type Waited =
72
- | { readonly tag: "reached"; readonly vector: ReadonlyMap<Braid, Generation> }
73
- | { readonly tag: "wedged"; readonly braid: Braid; readonly cause: string }
74
- | { readonly tag: "refused"; readonly detail: string }
75
-
76
- type ApplyPhase = "open" | "steady"
77
-
78
- type SlotApply = { readonly tag: "applied"; readonly generation: bigint } | { readonly tag: "discard" }
79
-
80
- interface OpenReplicaOptions<Rels extends SchemaRelations> {
81
- readonly store: ObjectStore
82
- readonly prefix: string
83
- readonly dir: string
84
- readonly theory: Schema<Rels>
85
- }
86
-
87
- interface Replica<Rels extends SchemaRelations> extends AsyncDisposable {
88
- readonly db: Db<Rels>
89
- readonly vector: ReadonlyMap<Braid, Generation>
90
- refresh(braid?: Braid): Promise<ReadonlyMap<Braid, Generation>>
91
- waitFor(vector: ReadonlyMap<Braid, Generation>): Promise<Waited>
92
- }
93
-
94
- interface Core<Rels extends SchemaRelations> {
95
- readonly store: ObjectStore
96
- readonly prefix: string
97
- readonly dir: string
98
- readonly theory: Schema<Rels>
99
- readonly descriptor: Descriptor
100
- db: Db<Rels>
101
- /** Settled|Pending. Generation is chainGeneration(this.chain) — a
102
- * total function of the sum. There is no pending Option beside the
103
- * vector and no 0|1 addend a reader can skip (§30). */
104
- chain: Chain
105
- manifestEtag: Etag | null
106
- checkpoint: CheckpointFacts | null
107
- checkpointDigest: Digest32 | null
108
- passes: number
109
- closed: boolean
110
- storeName: string
111
- gate: Promise<unknown>
112
- wedged: Map<Braid, string>
113
- provenance: ReplicaState
114
- }
115
-
116
- const cores = new WeakMap<object, Core<SchemaRelations>>()
117
-
118
- function coreOf<Rels extends SchemaRelations>(replica: Replica<Rels>): Core<Rels> {
119
- const core = cores.get(replica)
120
- if (core === undefined) {
121
- throw errors.new("not a replica of this driver")
122
- }
123
- return core as unknown as Core<Rels>
124
- }
125
-
126
- /** One asynchronous door per replica: refreshes, commits, and disposal serialize. */
127
- function withGate<Rels extends SchemaRelations, R>(core: Core<Rels>, body: () => Promise<R>): Promise<R> {
128
- const run = core.gate.then(body, body)
129
- core.gate = run.then(
130
- function absorb() {
131
- return undefined
132
- },
133
- function absorbFailure() {
134
- return undefined
135
- }
136
- )
137
- return run
138
- }
139
-
140
- function blake3Digest(bytes: Uint8Array): Digest32 {
141
- return digest32(new Uint8Array(internalBlake3(bytes)))
142
- }
143
-
144
- function sidecarPath<Rels extends SchemaRelations>(core: Core<Rels>): string {
145
- return path.join(core.dir, "chain")
146
- }
147
-
148
- let storeSequence = 0
149
-
150
- /** LMDB registers environments per canonical path for the life of the
151
- * process and the engine has no close verb, so a store path is never
152
- * reused: every bootstrap gets a fresh name and discards leave the old
153
- * environment to GC. */
154
- function freshStoreName(): string {
155
- storeSequence += 1
156
- return `store-${process.pid.toString(36)}-${Date.now().toString(36)}-${storeSequence}`
157
- }
158
-
159
- function storePath<Rels extends SchemaRelations>(core: Core<Rels>): string {
160
- return path.join(core.dir, core.storeName)
161
- }
162
-
163
- /** A wire braid id is a braid of this theory or a typed refuse. */
164
- function braidOf(theory: Schema<SchemaRelations> | Schema<never>, raw: Braid): Braid {
165
- const id = Number.parseInt(raw.slice(1), 16)
166
- const parsed = parse(theory, id)
167
- if (parsed === undefined) {
168
- refuse({ kind: "UnknownBraid" }, `unknown braid ${raw}`)
169
- }
170
- return parsed
171
- }
172
-
173
- function zeroChain(descriptor: Descriptor): Map<Braid, ChainEntry> {
174
- const chain = new Map<Braid, ChainEntry>()
175
- for (const id of descriptor.braidMembers.keys()) {
176
- chain.set(id, { g: generation(0n), prev: ZERO_HASH, ts: 0n })
177
- }
178
- return chain
179
- }
180
-
181
- function entriesOf<Rels extends SchemaRelations>(core: Core<Rels>): Map<Braid, ChainEntry> {
182
- return core.chain.entries as Map<Braid, ChainEntry>
183
- }
184
-
185
- function chainEntry<Rels extends SchemaRelations>(core: Core<Rels>, braid: Braid): ChainEntry {
186
- const id = braidOf(core.theory, braid)
187
- const entry = entriesOf(core).get(id)
188
- if (entry === undefined) {
189
- throw errors.new(`braid ${id} is not derived from this theory`)
190
- }
191
- return entry
192
- }
193
-
194
- function generationOf<Rels extends SchemaRelations>(core: Core<Rels>): bigint {
195
- return core.db.read(function readGeneration(instance) {
196
- return instance.generation
197
- })
198
- }
199
-
200
- /** The opened store's catalog digest — the engine handle's computed claim. */
201
- function catalogDigestOf<Rels extends SchemaRelations>(core: Core<Rels>): Digest32 {
202
- return digest32(core.db.catalogDigest())
203
- }
204
-
205
- /** Pending-arm payload: recorded ops and the batch header's timestamp
206
- * ride on the batch, not a third Option beside the vector. Wire
207
- * Pending is {braid,gen,bytes}; ops/ts are the in-memory fields of
208
- * that arm (§30) — null on a batch adopted as bytes the seat has not
209
- * decoded. */
210
- type HeldBatch = Pending & { readonly ops: readonly Op[] | null; readonly ts: bigint | null }
211
-
212
- function holdPending<Rels extends SchemaRelations>(
213
- core: Core<Rels>,
214
- batch: Pending,
215
- ops: readonly Op[] | null,
216
- ts: bigint | null
217
- ): void {
218
- const held: HeldBatch = { braid: batch.braid, slot: batch.slot, bytes: batch.bytes, ops, ts }
219
- core.chain = { tag: "pending", entries: entriesOf(core), batch: held }
220
- }
221
-
222
- function pendingOf<Rels extends SchemaRelations>(core: Core<Rels>): HeldBatch | null {
223
- if (core.chain.tag !== "pending") {
224
- return null
225
- }
226
- return core.chain.batch as HeldBatch
227
- }
228
-
229
- function settleHeld<Rels extends SchemaRelations>(core: Core<Rels>): void {
230
- core.chain = { tag: "settled", entries: entriesOf(core) }
231
- }
232
-
233
- /** The local vector equals the published checkpoint vector and the
234
- * chain is Settled — the seed/open floor the catalog claim is audited
235
- * against. */
236
- function atCheckpointFloor<Rels extends SchemaRelations>(core: Core<Rels>): boolean {
237
- if (core.checkpoint === null || core.chain.tag === "pending") {
238
- return false
239
- }
240
- for (const [braid, head] of core.checkpoint.braids) {
241
- if (chainEntry(core, braid).g !== head.g) {
242
- return false
243
- }
244
- }
245
- return true
246
- }
247
-
248
- async function persistSidecar<Rels extends SchemaRelations>(core: Core<Rels>): Promise<void> {
249
- await writeSidecar(core.descriptor.codec, sidecarPath(core), core.chain)
250
- }
251
-
252
- /** Writes the checkpoint store file and fsyncs the file and its parent
253
- * directory — the mdb is durable before the sidecar. */
254
- async function writeCheckpointMdb(dir: string, bytes: Uint8Array): Promise<void> {
255
- const data = path.join(dir, "data.mdb")
256
- const handle = await fs.open(data, "w")
257
- const written = await errors.try(
258
- (async function writeAll() {
259
- await handle.writeFile(bytes)
260
- await handle.sync()
261
- })()
262
- )
263
- await handle.close()
264
- if (written.error) {
265
- throw errors.wrap(written.error, `write checkpoint store ${data}`)
266
- }
267
- const parent = await fs.open(dir, "r")
268
- const synced = await errors.try(parent.sync())
269
- await parent.close()
270
- if (synced.error) {
271
- throw errors.wrap(synced.error, `fsync checkpoint store directory ${dir}`)
272
- }
273
- }
274
-
275
- async function settle<Rels extends SchemaRelations>(core: Core<Rels>): Promise<void> {
276
- settleHeld(core)
277
- await persistSidecar(core)
278
- }
279
-
280
- async function clearPending<Rels extends SchemaRelations>(core: Core<Rels>): Promise<void> {
281
- await settle(core)
282
- }
283
-
284
- function disposed<Rels extends SchemaRelations>(core: Core<Rels>): void {
285
- if (core.closed) {
286
- throw errors.new("replica is disposed")
287
- }
288
- }
289
-
290
- function adoptChain<Rels extends SchemaRelations>(core: Core<Rels>, chain: Chain): void {
291
- const entries = new Map<Braid, ChainEntry>()
292
- for (const [raw, entry] of chain.entries) {
293
- entries.set(braidOf(core.theory, raw), entry)
294
- }
295
- for (const id of core.descriptor.braidMembers.keys()) {
296
- if (!entries.has(id)) {
297
- entries.set(id, { g: generation(0n), prev: ZERO_HASH, ts: 0n })
298
- }
299
- }
300
- if (chain.tag === "pending") {
301
- const held: HeldBatch = {
302
- braid: braidOf(core.theory, chain.batch.braid),
303
- slot: chain.batch.slot,
304
- bytes: chain.batch.bytes,
305
- ops: null,
306
- ts: null
307
- }
308
- core.chain = { tag: "pending", entries, batch: held }
309
- } else {
310
- core.chain = { tag: "settled", entries }
311
- }
312
- }
313
-
314
- /** One `db.write` applying ops in listed order, rows in listed order.
315
- * The positional row → named fact lift is the engine's `factOf` —
316
- * closed-ref handles included. */
317
- function applyOps<Rels extends SchemaRelations>(core: Core<Rels>, ops: readonly Op[]): WriteOutcome<Rels, number> {
318
- return core.db.write(function applyBatch(tx) {
319
- for (const op of ops) {
320
- const member = core.theory.relations[op.relation]
321
- if (member === undefined) {
322
- throw errors.new(`batch op cites unknown relation ${op.relation}`)
323
- }
324
- const relation = member as MemberRelation<Rels>
325
- const facts = op.rows.map(function liftRow(row) {
326
- return factOf(relation, row)
327
- })
328
- if (op.op === "insert") {
329
- tx.insert(relation, facts)
330
- } else {
331
- tx.delete(relation, facts)
332
- }
333
- }
334
- return 0
335
- })
336
- }
337
-
338
- /**
339
- * Decode, verify the chain, `db.write` the batch, then refuse a
340
- * first-applied no-op against the identity. The sidecar advances
341
- * only after that check; a rejected replay is phase-scoped:
342
- * discard before the store has proven itself whole,
343
- * `ErrReplayDiverged` after.
344
- */
345
- async function applySlot<Rels extends SchemaRelations>(
346
- core: Core<Rels>,
347
- braid: Braid,
348
- slot: Generation,
349
- bytes: Uint8Array,
350
- phase: ApplyPhase
351
- ): Promise<SlotApply> {
352
- const decoded = decodeBatch(core.descriptor, bytes)
353
- const entry = chainEntry(core, braid)
354
- verifyChain(decoded.header, braid, slot, { g: entry.g, prev: entry.prev, ts: entry.ts })
355
- const outcome = applyOps(core, decoded.ops)
356
- if (outcome.tag === "rejected") {
357
- if (phase === "open") {
358
- return { tag: "discard" }
359
- }
360
- throw errors.wrap(ErrReplayDiverged, `braid ${braid} slot ${slot} writer ${decoded.header.writer}`)
361
- }
362
- const identity = chainSum(core.chain) - entry.g + slot
363
- if (outcome.value.generation < identity) {
364
- refuse(
365
- { kind: "NoOpSlot", braid, slot, writer: decoded.header.writer },
366
- `braid ${braid} slot ${slot}: a first-applied slot changed nothing — publish-law violation by writer ${decoded.header.writer}`
367
- )
368
- }
369
- entriesOf(core).set(braid, { g: slot, prev: blake3Digest(bytes), ts: decoded.header.timestamp })
370
- await persistSidecar(core)
371
- return { tag: "applied", generation: outcome.value.generation }
372
- }
373
-
374
- /**
375
- * The published checkpoint vector is the one floor: a slot at or
376
- * below it is retired, and a create must not touch the store.
377
- */
378
- function belowFloor<Rels extends SchemaRelations>(core: Core<Rels>, braid: Braid, slot: Generation): boolean {
379
- const id = braidOf(core.theory, braid)
380
- if (core.checkpoint === null) {
381
- return false
382
- }
383
- const floor = core.checkpoint.braids.get(id)
384
- if (floor === undefined) {
385
- return false
386
- }
387
- return slot <= floor.g
388
- }
389
-
390
- /**
391
- * The gc floor rule: below the current checkpoint's vector a 404 is a
392
- * collected hole, at or above it the honest tip.
393
- */
394
- function holeAt<Rels extends SchemaRelations>(core: Core<Rels>, braid: Braid, next: Generation): boolean {
395
- return belowFloor(core, braid, next)
396
- }
397
-
398
- /**
399
- * Classification of a pending batch against occupant, store
400
- * generation, and the floor. BelowFloor is published (`Clear`) — the
401
- * slot is already in the trusted history, not a candidate to re-judge.
402
- */
403
- type PendingFold =
404
- | { readonly tag: "ours" }
405
- | { readonly tag: "theirs-unapplied" }
406
- | { readonly tag: "theirs-applied" }
407
- | { readonly tag: "absent-unapplied" }
408
- | { readonly tag: "absent-applied" }
409
- | { readonly tag: "below-floor" }
410
- | { readonly tag: "phantom" }
411
-
412
- function foldPending(
413
- sum: bigint,
414
- generation: bigint,
415
- occupant: Uint8Array | null,
416
- pendingBytes: Uint8Array,
417
- covered: boolean
418
- ): PendingFold {
419
- if (covered) {
420
- return { tag: "below-floor" }
421
- }
422
- if (occupant !== null && bytesEqual(occupant, pendingBytes)) {
423
- return { tag: "ours" }
424
- }
425
- if (occupant !== null) {
426
- return generation === sum ? { tag: "theirs-unapplied" } : { tag: "theirs-applied" }
427
- }
428
- if (generation === sum) {
429
- return { tag: "absent-unapplied" }
430
- }
431
- if (generation === saturatingAddU64(sum, 1n)) {
432
- return { tag: "absent-applied" }
433
- }
434
- return { tag: "phantom" }
435
- }
436
-
437
- function isCorruption(error: unknown): boolean {
438
- if (!(error instanceof Error)) {
439
- return false
440
- }
441
- return errors.is(error, ErrReplayDiverged) || errors.is(error, ErrRefused)
442
- }
443
-
444
- /**
445
- * One braid, one slot. Catch-up, refresh, waitFor, and open all call
446
- * this; a hot braid cannot drain past one step per round.
447
- */
448
- async function stepBraid<Rels extends SchemaRelations>(
449
- core: Core<Rels>,
450
- braid: Braid,
451
- phase: ApplyPhase
452
- ): Promise<Step> {
453
- disposed(core)
454
- const id = braidOf(core.theory, braid)
455
- const wedged = core.wedged.get(id)
456
- if (wedged !== undefined) {
457
- return { tag: "wedged", braid: id, cause: wedged }
458
- }
459
- const next = generation(chainEntry(core, id).g + 1n)
460
- const fetched = await core.store.get(logKey(core.prefix, id, next))
461
- if (fetched === null) {
462
- if (holeAt(core, id, next)) {
463
- return { tag: "reseed", cause: "gap-below-floor" }
464
- }
465
- return { tag: "tip" }
466
- }
467
- const chain = core.chain
468
- if (chain.tag === "pending" && id === chain.batch.braid && next === chain.batch.slot) {
469
- if (!bytesEqual(fetched.bytes, chain.batch.bytes)) {
470
- return { tag: "reseed", cause: "lost-pending-fork" }
471
- }
472
- await settle(core)
473
- }
474
- const applied = await errors.try(applySlot(core, id, next, fetched.bytes, phase))
475
- if (applied.error) {
476
- if (phase === "steady" && isCorruption(applied.error)) {
477
- core.wedged.set(id, applied.error.message)
478
- return { tag: "wedged", braid: id, cause: applied.error.message }
479
- }
480
- throw applied.error
481
- }
482
- if (applied.data.tag === "discard") {
483
- return { tag: "reseed", cause: "rejected-in-open" }
484
- }
485
- return { tag: "applied" }
486
- }
487
-
488
- /**
489
- * One pass: heartbeat, one slot per braid, wholeness, disposed. The
490
- * same function refresh, waitFor, catch-up, and open execute.
491
- */
492
- async function runPass<Rels extends SchemaRelations>(
493
- core: Core<Rels>,
494
- braids: readonly Braid[],
495
- phase: ApplyPhase
496
- ): Promise<RefreshOutcome> {
497
- disposed(core)
498
- core.passes += 1
499
- if (core.passes % HEARTBEAT_PASSES === 0) {
500
- await refreshManifest(core)
501
- }
502
- const remaining = new Set(braids)
503
- while (remaining.size > 0) {
504
- disposed(core)
505
- let progressed = false
506
- for (const braid of braids) {
507
- if (!remaining.has(braid)) {
508
- continue
509
- }
510
- const step = await stepBraid(core, braid, phase)
511
- if (step.tag === "applied") {
512
- progressed = true
513
- } else if (step.tag === "tip" || step.tag === "wedged") {
514
- remaining.delete(braid)
515
- } else {
516
- return { tag: "reseed", cause: step.cause }
517
- }
518
- }
519
- if (!progressed) {
520
- break
521
- }
522
- }
523
- if (!wholenessHolds(core)) {
524
- return { tag: "reseed", cause: "wholeness" }
525
- }
526
- for (const [braid, cause] of core.wedged) {
527
- return { tag: "wedged", braid, cause }
528
- }
529
- return { tag: "advanced", vector: vectorOf(core) }
530
- }
531
-
532
- function allBraids<Rels extends SchemaRelations>(core: Core<Rels>): Braid[] {
533
- return [...core.descriptor.braidMembers.keys()]
534
- }
535
-
536
- function wholenessHolds<Rels extends SchemaRelations>(core: Core<Rels>): boolean {
537
- return generationOf(core) === chainGeneration(core.chain)
538
- }
539
-
540
- async function adoptManifest<Rels extends SchemaRelations>(
541
- core: Core<Rels>,
542
- bytes: Uint8Array,
543
- etag: Etag
544
- ): Promise<void> {
545
- const manifest = parseManifest(bytes)
546
- if (!bytesEqual(manifest.fingerprint, digest32(core.descriptor.fingerprintBytes))) {
547
- refuse(
548
- {
549
- kind: "FingerprintMismatch",
550
- carried: hex32(manifest.fingerprint),
551
- expected: core.descriptor.fingerprint
552
- },
553
- "the store's manifest names a different theory"
554
- )
555
- }
556
- if (manifest.checkpoint === null) {
557
- core.checkpoint = null
558
- core.checkpointDigest = null
559
- } else if (core.checkpointDigest === null || !bytesEqual(manifest.checkpoint, core.checkpointDigest)) {
560
- const facts = await core.store.get(ckptDocKey(core.prefix, manifest.checkpoint))
561
- if (facts === null) {
562
- throw errors.new(`manifest points at absent checkpoint ${hex32(manifest.checkpoint)}`)
563
- }
564
- // The codec-backed parseCheckpoint judges the braid set against the
565
- // sealed handle — an unknown or drifted braid refuses at parse.
566
- core.checkpoint = parseCheckpoint(core.descriptor.codec, facts.bytes)
567
- core.checkpointDigest = manifest.checkpoint
568
- }
569
- // The pointer is adopted only after the checkpoint it names is in
570
- // hand. A failed fetch leaves the old etag and the old floor (40/67).
571
- core.manifestEtag = etag
572
- }
573
-
574
- /** A replica never births a manifest. Absence is ManifestMissing. */
575
- async function refreshManifest<Rels extends SchemaRelations>(core: Core<Rels>): Promise<void> {
576
- if (core.manifestEtag !== null) {
577
- const poll = await core.store.getIfChanged(manifestKey(core.prefix), core.manifestEtag)
578
- if (poll.tag === "unchanged") {
579
- return
580
- }
581
- await adoptManifest(core, poll.fetched.bytes, poll.fetched.etag)
582
- return
583
- }
584
- const fetched = await core.store.get(manifestKey(core.prefix))
585
- if (fetched === null) {
586
- refuseManifestMissing("the store has no manifest")
587
- }
588
- await adoptManifest(core, fetched.bytes, fetched.etag)
589
- }
590
-
591
- /** Bootstraps a fresh local store: from the current checkpoint when one
592
- * exists, else `Db.create` at the zero vector. Always holds Settled —
593
- * a prior Pending arm cannot survive beside the new vector. */
594
- async function initializeStore<Rels extends SchemaRelations>(core: Core<Rels>): Promise<void> {
595
- core.storeName = freshStoreName()
596
- const target = storePath(core)
597
- await fs.rm(target, { recursive: true, force: true })
598
- await fs.mkdir(core.dir, { recursive: true })
599
- if (core.checkpoint !== null && core.checkpointDigest !== null) {
600
- const mdb = await core.store.get(checkpointMdbKey(core.prefix, core.checkpointDigest))
601
- if (mdb === null) {
602
- throw errors.new(`checkpoint ${hex32(core.checkpointDigest)} names an absent .mdb`)
603
- }
604
- await fs.mkdir(target, { recursive: true })
605
- await writeCheckpointMdb(target, mdb.bytes)
606
- core.db = await SdkDb.open(target, core.theory)
607
- core.chain = {
608
- tag: "settled",
609
- entries: new Map(
610
- [...core.checkpoint.braids.entries()].map(function seed([raw, head]) {
611
- const braid = braidOf(core.theory, raw)
612
- return [braid, { g: head.g, prev: head.hash, ts: head.ts }] as const
613
- })
614
- )
615
- }
616
- const opened = generationOf(core)
617
- if (opened !== chainGeneration(core.chain)) {
618
- throw errors.new(
619
- `checkpoint store opened at generation ${opened}, chain generation is ${chainGeneration(core.chain)}`
620
- )
621
- }
622
- auditCatalog(core.checkpoint, catalogDigestOf(core))
623
- core.provenance = { tag: "checkpoint-seeded", catalog: core.checkpointDigest }
624
- await persistSidecar(core)
625
- return
626
- }
627
- const created = await SdkDb.create(target, core.theory)
628
- if (created.tag === "rejected") {
629
- throw errors.new("the theory's ground axioms were rejected at bootstrap")
630
- }
631
- core.db = created.value
632
- core.chain = { tag: "settled", entries: zeroChain(core.descriptor) }
633
- core.provenance = { tag: "bootstrapped" }
634
- await persistSidecar(core)
635
- }
636
-
637
- /** The scream tracks the set of repair signatures; a recurrence alarms. */
638
- function screamOf(context: string): { attempt(signature: string): void } {
639
- const seen = new Set<string>()
640
- let attempts = 0
641
- return {
642
- attempt(signature: string): void {
643
- attempts += 1
644
- if (seen.has(signature)) {
645
- console.error(`bumbledb-log alarm: ${context} repair signature recurs: ${signature}`)
646
- } else {
647
- seen.add(signature)
648
- }
649
- if (attempts % 8 === 0) {
650
- console.error(`bumbledb-log warning: ${context} repair attempt ${attempts}: ${signature}`)
651
- }
652
- }
653
- }
654
- }
655
-
656
- /** The disposable law: the directory is cache, never truth. The local
657
- * LMDB path rotates because the engine has no close verb — the old
658
- * environment is left for GC while the fresh pull takes a new path.
659
- * initializeStore writes Settled, so the open-identity is
660
- * chainGeneration(core.chain) — there is no addend a reader can skip. */
661
- async function discardAndReopen<Rels extends SchemaRelations>(core: Core<Rels>): Promise<void> {
662
- const scream = screamOf("replica discard-and-re-pull")
663
- for (;;) {
664
- const old = storePath(core)
665
- await fs.rm(old, { recursive: true, force: true })
666
- core.wedged.clear()
667
- await initializeStore(core)
668
- const outcome = await runPass(core, allBraids(core), "open")
669
- if (outcome.tag === "reseed") {
670
- scream.attempt(outcome.cause)
671
- continue
672
- }
673
- if (outcome.tag === "refused") {
674
- throw errors.wrap(ErrRefused, outcome.detail)
675
- }
676
- if (generationOf(core) === chainGeneration(core.chain)) {
677
- return
678
- }
679
- scream.attempt("wholeness")
680
- }
681
- }
682
-
683
- async function newestStoreDir(dir: string): Promise<string | null> {
684
- const listed = await errors.try(fs.readdir(dir))
685
- if (listed.error) {
686
- return null
687
- }
688
- let newest: string | null = null
689
- let newestAt = -1
690
- for (const name of listed.data) {
691
- if (!name.startsWith("store-")) {
692
- continue
693
- }
694
- const stat = await errors.try(fs.stat(path.join(dir, name)))
695
- if (stat.error === undefined && stat.data.mtimeMs > newestAt) {
696
- newestAt = stat.data.mtimeMs
697
- newest = name
698
- }
699
- }
700
- return newest
701
- }
702
-
703
- /**
704
- * Open reclaims the reserved `~tmp`/`~lease` namespace. The known
705
- * document `{dir}/~lease/ckpt-scratch` names a crash-stranded
706
- * candidate; when that digest is not the live head the ckpt pair is
707
- * deleted with the lease.
708
- */
709
- async function sweepReservedKeys<Rels extends SchemaRelations>(core: Core<Rels>): Promise<void> {
710
- const lease = path.join(core.dir, LEASE_NAMESPACE, CKPT_SCRATCH_LEASE)
711
- const read = await errors.try(fs.readFile(lease))
712
- if (read.error === undefined) {
713
- const digest = parseCkptScratch(read.data)
714
- if (digest !== null && (core.checkpointDigest === null || !bytesEqual(digest, core.checkpointDigest))) {
715
- await core.store.delete(ckptDocKey(core.prefix, digest))
716
- await core.store.delete(checkpointMdbKey(core.prefix, digest))
717
- }
718
- }
719
- await fs.rm(path.join(core.dir, TEMP_NAMESPACE), { recursive: true, force: true })
720
- await fs.rm(path.join(core.dir, LEASE_NAMESPACE), { recursive: true, force: true })
721
- }
722
-
723
- /** The disposable law says cache directories do not hoard corpses: every
724
- * rotated `store-*` LMDB dir except the adopted one is dead — left by a
725
- * crashed process or a prior rotation — and is swept at open. */
726
- async function sweepRotations<Rels extends SchemaRelations>(core: Core<Rels>): Promise<void> {
727
- const listed = await errors.try(fs.readdir(core.dir))
728
- if (listed.error) {
729
- return
730
- }
731
- for (const name of listed.data) {
732
- if (!name.startsWith("store-") || name === core.storeName) {
733
- continue
734
- }
735
- await fs.rm(path.join(core.dir, name), { recursive: true, force: true })
736
- }
737
- }
738
-
739
- /**
740
- * Pending recovery: apply the resurrected batch. A batch rejected here
741
- * was never acked; a born-no-op settles; anything else stays Pending
742
- * so generation(chain) accounts for the unpublished apply.
743
- */
744
- async function resolvePendingAtOpen<Rels extends SchemaRelations>(core: Core<Rels>): Promise<void> {
745
- const chain = core.chain
746
- if (chain.tag === "settled") {
747
- return
748
- }
749
- // A slot the floor already covers is published (`Clear`), not re-judged (46).
750
- if (
751
- foldPending(
752
- chainSum(core.chain),
753
- generationOf(core),
754
- null,
755
- chain.batch.bytes,
756
- belowFloor(core, chain.batch.braid, chain.batch.slot)
757
- ).tag === "below-floor"
758
- ) {
759
- await settle(core)
760
- return
761
- }
762
- const decoded = errors.trySync(function decodePending() {
763
- return decodeBatch(core.descriptor, chain.batch.bytes)
764
- })
765
- if (decoded.error) {
766
- await settle(core)
767
- return
768
- }
769
- const outcome = applyOps(core, decoded.data.ops)
770
- if (outcome.tag === "rejected") {
771
- await settle(core)
772
- return
773
- }
774
- // Store matches chainGeneration of the held Pending (sum+1) when the
775
- // apply is in the db — just now, or already. A born-no-op leaves the
776
- // store at the Settled generation, so the identity fails and we settle.
777
- if (generationOf(core) !== chainGeneration(core.chain)) {
778
- await settle(core)
779
- return
780
- }
781
- holdPending(core, chain.batch, decoded.data.ops, decoded.data.header.timestamp)
782
- }
783
-
784
- /**
785
- * Pending resurrection through a cold re-pull: the store directory was
786
- * unopenable (or discarded), so the sidecar's pending is resolved
787
- * against a freshly caught-up store — a byte-equal published slot
788
- * absorbs it (L10's idempotent replay), and everything else takes the
789
- * one loss path's re-judgment at the tip. A slot the floor already
790
- * covers is published (`Clear`), not re-judged (46).
791
- */
792
- async function resolveColdPending<Rels extends SchemaRelations>(core: Core<Rels>, pending: Pending): Promise<void> {
793
- const braid = braidOf(core.theory, pending.braid)
794
- if (
795
- foldPending(chainSum(core.chain), generationOf(core), null, pending.bytes, belowFloor(core, braid, pending.slot))
796
- .tag === "below-floor"
797
- ) {
798
- await settle(core)
799
- return
800
- }
801
- const decoded = errors.trySync(function decodePending() {
802
- return decodeBatch(core.descriptor, pending.bytes)
803
- })
804
- if (decoded.error) {
805
- return
806
- }
807
- const slot = decoded.data.header.braidGen
808
- if (entriesOf(core).has(braid) && chainEntry(core, braid).g >= slot) {
809
- const published = await core.store.get(logKey(core.prefix, braid, slot))
810
- if (published !== null && bytesEqual(published.bytes, pending.bytes)) {
811
- return
812
- }
813
- }
814
- const before = generationOf(core)
815
- const outcome = applyOps(core, decoded.data.ops)
816
- if (outcome.tag === "accepted" && outcome.value.generation > before) {
817
- holdPending(core, { ...pending, braid }, decoded.data.ops, decoded.data.header.timestamp)
818
- await readdressPending(core, decoded.data.ops, decoded.data.header.writer)
819
- }
820
- }
821
-
822
- async function openCore<Rels extends SchemaRelations>(options: OpenReplicaOptions<Rels>): Promise<Core<Rels>> {
823
- const descriptor = descriptorOf(options.theory)
824
- const core: Core<Rels> = {
825
- store: options.store,
826
- prefix: options.prefix,
827
- dir: path.resolve(options.dir),
828
- theory: options.theory,
829
- descriptor,
830
- db: undefined as unknown as Db<Rels>,
831
- chain: { tag: "settled", entries: zeroChain(descriptor) },
832
- manifestEtag: null,
833
- checkpoint: null,
834
- checkpointDigest: null,
835
- passes: 0,
836
- closed: false,
837
- storeName: freshStoreName(),
838
- gate: Promise.resolve(),
839
- wedged: new Map(),
840
- provenance: { tag: "bootstrapped" }
841
- }
842
- await fs.mkdir(core.dir, { recursive: true })
843
- const fetched = await core.store.get(manifestKey(core.prefix))
844
- if (fetched === null) {
845
- refuseManifestMissing("the store has no manifest")
846
- }
847
- await adoptManifest(core, fetched.bytes, fetched.etag)
848
-
849
- const existing = await newestStoreDir(core.dir)
850
- const sidecar = await readSidecar(descriptor.codec, sidecarPath(core))
851
- let opened = false
852
- let coldPending: Pending | null = null
853
- if (sidecar.tag === "fault") {
854
- throw wrapStore(sidecar.io, `read sidecar ${sidecarPath(core)}`)
855
- }
856
- // The codec-backed readSidecar judges every braid against the sealed
857
- // handle — a foreign braid is a corrupt sidecar, and a corrupt
858
- // sidecar is discarded cache (the disposable law), never adopted.
859
- if (existing !== null && sidecar.tag === "read") {
860
- core.storeName = existing
861
- const openedDb = await errors.try(SdkDb.open(storePath(core), options.theory))
862
- if (openedDb.error === undefined) {
863
- core.db = openedDb.data
864
- adoptChain(core, sidecar.chain)
865
- core.provenance = { tag: "sidecar-resumed", floor: vectorOf(core) }
866
- opened = true
867
- }
868
- }
869
- if (!opened) {
870
- if (sidecar.tag === "read" && sidecar.chain.tag === "pending") {
871
- coldPending = sidecar.chain.batch
872
- }
873
- await initializeStore(core)
874
- }
875
-
876
- if (opened) {
877
- await resolvePendingAtOpen(core)
878
- }
879
-
880
- const outcome = await runPass(core, allBraids(core), "open")
881
- if (outcome.tag === "reseed" || outcome.tag === "refused" || !wholenessHolds(core)) {
882
- coldPending = core.chain.tag === "pending" ? core.chain.batch : coldPending
883
- await discardAndReopen(core)
884
- }
885
-
886
- if (coldPending !== null) {
887
- await resolveColdPending(core, coldPending)
888
- }
889
- if (atCheckpointFloor(core)) {
890
- auditCatalog(core.checkpoint, catalogDigestOf(core))
891
- }
892
- await sweepRotations(core)
893
- await sweepReservedKeys(core)
894
- return core
895
- }
896
-
897
- /**
898
- * The steady-state discard route: a contested pending slot surrenders
899
- * the directory. The pending batch rides in memory — the sidecar is
900
- * not settled ahead of the re-judgment — and takes the one loss path
901
- * at the fresh tip.
902
- */
903
- async function repairDiscard<Rels extends SchemaRelations>(core: Core<Rels>): Promise<void> {
904
- const coldPending = core.chain.tag === "pending" ? core.chain.batch : null
905
- await discardAndReopen(core)
906
- if (coldPending !== null) {
907
- await resolveColdPending(core, coldPending)
908
- }
909
- }
910
-
911
- /**
912
- * Re-addresses an applied-but-unpublished batch at the braid's current
913
- * tip: fresh slot, `prev` citing the new predecessor, timestamp
914
- * re-clamped — the ops are exactly the recorded ops the re-judgment
915
- * just accepted. Publication itself retries on the next commit (60).
916
- */
917
- async function readdressPending<Rels extends SchemaRelations>(
918
- core: Core<Rels>,
919
- ops: readonly Op[],
920
- writerId: bigint
921
- ): Promise<void> {
922
- const first = ops[0]
923
- if (first === undefined) {
924
- await settle(core)
925
- return
926
- }
927
- const relation = core.descriptor.relationByName.get(first.relation)
928
- const raw = relation === undefined ? undefined : core.descriptor.braidOfRelation.get(relation.id)
929
- if (raw === undefined) {
930
- await settle(core)
931
- return
932
- }
933
- const braid = braidOf(core.theory, raw)
934
- const entry = chainEntry(core, braid)
935
- const timestamp = maxBigint(BigInt(Date.now()), entry.ts)
936
- const bytes = encodeBatch(
937
- core.descriptor,
938
- {
939
- braid,
940
- braidGen: generation(entry.g + 1n),
941
- prev: entry.prev,
942
- writer: writerId,
943
- timestamp
944
- },
945
- ops
946
- )
947
- holdPending(core, { braid, slot: generation(entry.g + 1n), bytes }, ops, timestamp)
948
- await persistSidecar(core)
949
- }
950
-
951
- function maxBigint(a: bigint, b: bigint): bigint {
952
- return a > b ? a : b
953
- }
954
-
955
- function vectorOf<Rels extends SchemaRelations>(core: Core<Rels>): ReadonlyMap<Braid, Generation> {
956
- const vector = new Map<Braid, Generation>()
957
- for (const [id, entry] of core.chain.entries) {
958
- vector.set(id, entry.g)
959
- }
960
- return vector
961
- }
962
-
963
- async function refreshPass<Rels extends SchemaRelations>(core: Core<Rels>, braid?: Braid): Promise<RefreshOutcome> {
964
- const braids = braid === undefined ? allBraids(core) : [braidOf(core.theory, braid)]
965
- const outcome = await runPass(core, braids, "steady")
966
- if (outcome.tag === "reseed") {
967
- await repairDiscard(core)
968
- return { tag: "reseed", cause: outcome.cause }
969
- }
970
- return outcome
971
- }
972
-
973
- /** waitFor is refresh with a verdict: the same pass, then the full
974
- * Waited sum. A braid the target needs that is wedged below it returns
975
- * Wedged — no refresh will ever reach the target — and a heartbeat
976
- * refusal returns Refused. */
977
- async function waitForVector<Rels extends SchemaRelations>(
978
- core: Core<Rels>,
979
- target: ReadonlyMap<Braid, Generation>
980
- ): Promise<Waited> {
981
- for (;;) {
982
- disposed(core)
983
- const have = vectorOf(core)
984
- for (const [braid, wanted] of target) {
985
- const cause = core.wedged.get(braid)
986
- const at = have.get(braid)
987
- if (cause !== undefined && at !== undefined && at < wanted) {
988
- return { tag: "wedged", braid, cause }
989
- }
990
- }
991
- if (Vector.from(have).dominates(Vector.from(target))) {
992
- return { tag: "reached", vector: have }
993
- }
994
- const outcome = await withGate(core, async function waitPass() {
995
- disposed(core)
996
- return refreshPass(core)
997
- })
998
- if (outcome.tag === "refused") {
999
- return { tag: "refused", detail: outcome.detail }
1000
- }
1001
- await new Promise(function later(resolve) {
1002
- setTimeout(resolve, WAIT_FOR_POLL_MS)
1003
- })
1004
- }
1005
- }
1006
-
1007
- async function openReplica<Rels extends SchemaRelations>(options: OpenReplicaOptions<Rels>): Promise<Replica<Rels>> {
1008
- const core = await openCore(options)
1009
- const replica: Replica<Rels> = {
1010
- get db() {
1011
- disposed(core)
1012
- return core.db
1013
- },
1014
- get vector() {
1015
- return vectorOf(core)
1016
- },
1017
- async refresh(braid?: Braid) {
1018
- return withGate(core, async function refreshBody() {
1019
- disposed(core)
1020
- await refreshPass(core, braid)
1021
- return vectorOf(core)
1022
- })
1023
- },
1024
- async waitFor(vector) {
1025
- const target = new Map<Braid, Generation>()
1026
- for (const [braid, wanted] of vector) {
1027
- target.set(braidOf(core.theory, braid), wanted)
1028
- }
1029
- return waitForVector(core, target)
1030
- },
1031
- async [Symbol.asyncDispose]() {
1032
- await withGate(core, async function disposeBody() {
1033
- core.closed = true
1034
- await persistSidecar(core)
1035
- })
1036
- }
1037
- }
1038
- cores.set(replica, core as unknown as Core<SchemaRelations>)
1039
- return replica
1040
- }
1041
-
1042
- export type { Core, OpenReplicaOptions, RefreshOutcome, Replica, Waited }
1043
- export {
1044
- applyOps,
1045
- belowFloor,
1046
- chainEntry,
1047
- clearPending,
1048
- coreOf,
1049
- discardAndReopen,
1050
- entriesOf,
1051
- foldPending,
1052
- generationOf,
1053
- holdPending,
1054
- maxBigint,
1055
- openReplica,
1056
- pendingOf,
1057
- persistSidecar,
1058
- readdressPending,
1059
- withGate,
1060
- ZERO_HASH
1061
- }