@bjornpagen/bumbledb-log 0.17.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -18
- package/package.json +5 -3
- package/src/braids.ts +19 -7
- package/src/bytes.ts +79 -11
- package/src/chain.ts +179 -63
- package/src/codec.ts +141 -62
- package/src/descriptor.ts +392 -348
- package/src/errors.ts +140 -10
- package/src/index.ts +23 -8
- package/src/keys.ts +212 -15
- package/src/manifest.ts +171 -75
- package/src/replica.ts +606 -295
- package/src/store-s3.ts +377 -0
- package/src/store.ts +524 -105
- package/src/tenants.ts +322 -26
- package/src/value.ts +80 -29
- package/src/vector.ts +173 -0
- package/src/writer.ts +484 -120
package/README.md
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
# @bjornpagen/bumbledb-log
|
|
2
2
|
|
|
3
3
|
Braided object-store replication for [bumbledb](https://github.com/bjornpagen/bumbledb):
|
|
4
|
-
a thin peer of `@bjornpagen/bumbledb` (peer
|
|
5
|
-
|
|
4
|
+
a thin peer of `@bjornpagen/bumbledb` (peer `^0.19.0`). The package is
|
|
5
|
+
three things:
|
|
6
6
|
|
|
7
7
|
1. **The mirrored pure pair**, byte-exact against the Rust driver and
|
|
8
8
|
pinned by cross-language goldens: `encodeBatch`/`decodeBatch` (the
|
|
@@ -10,30 +10,40 @@ The package is three things:
|
|
|
10
10
|
`braidsOf(descriptor)` (the schema's own shard map, as data — with
|
|
11
11
|
`serialAtStatementsOf` naming the degenerate-serial statements beside it).
|
|
12
12
|
2. **The five-verb object store** — `get`, `getIfChanged`, `putCreate`,
|
|
13
|
-
`putSwap`, `delete` —
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
13
|
+
`putSwap`, `delete` — taking a branded `StoreKey` parsed once by
|
|
14
|
+
`storeKey`. `fsStore` is the tier-1 local-directory implementation;
|
|
15
|
+
`memStore` is the same five verbs over one in-process map
|
|
16
|
+
(single-process only; third `Etag` producer, blake3 like `fsStore`);
|
|
17
|
+
`s3Store` is the five verbs over S3-compatible storage (the official
|
|
18
|
+
`@aws-sdk/client-s3` client signs and talks; R2 rides region `auto`).
|
|
17
19
|
3. **Replica and writer** composed from the engine SDK's existing verbs:
|
|
18
20
|
`openReplica` hands out the SDK's own `Db`; `openWriter` adds the
|
|
19
21
|
right to create log objects; `openTenants` is an LRU of per-tenant
|
|
20
22
|
replicas. No engine surface is duplicated.
|
|
21
23
|
|
|
24
|
+
The package `engines` and the `.ts` test runner require Node >=24.
|
|
25
|
+
|
|
26
|
+
The exported vocabulary reads as English at the call site: `Value`,
|
|
27
|
+
`Interval`, `Batch`, `Theory`, `Descriptor`, `Op`, `Pending`,
|
|
28
|
+
`ChainEntry`, plus the branded scalars `StoreKey`, `Generation`,
|
|
29
|
+
`Etag`, and `Braid` (`storeKey`, `generation`, `etag`, `braid` parse
|
|
30
|
+
at the boundary; the verbs take the proof).
|
|
31
|
+
|
|
22
32
|
Async ⟺ network: `openReplica`, `refresh`, `waitFor`, `commit`,
|
|
23
33
|
`commitSplit`, and disposal await store verbs; everything on
|
|
24
34
|
`replica.db`, the `batch.*` recorders, and the pure pair are synchronous.
|
|
25
35
|
|
|
26
|
-
##
|
|
36
|
+
## A Fluid host
|
|
27
37
|
|
|
28
38
|
```ts
|
|
29
39
|
// lib/db.ts — module scope; Fluid shares this across the instance's requests
|
|
30
40
|
import { fsStore, openReplica, openWriter } from "@bjornpagen/bumbledb-log"
|
|
31
41
|
|
|
32
42
|
export const replica = await openReplica({ store: s3(env), prefix: "prod/main", dir: "/tmp/store", theory: Ledger })
|
|
33
|
-
export const writer = openWriter(replica)
|
|
43
|
+
export const writer = await openWriter(replica)
|
|
34
44
|
|
|
35
45
|
// route handler
|
|
36
|
-
const out = await writer.commit((
|
|
46
|
+
const out = await writer.commit((batch) => batch.insert(Booking, [row]))
|
|
37
47
|
if (out.tag === "accepted") ctx.waitUntil(replica.refresh(out.braid))
|
|
38
48
|
```
|
|
39
49
|
|
|
@@ -42,7 +52,7 @@ if (out.tag === "accepted") ctx.waitUntil(replica.refresh(out.braid))
|
|
|
42
52
|
leaf-blob pattern keeps metadata stores in the tens of MB. Per-tenant
|
|
43
53
|
fleets get the same gate through `openTenants({ budgetBytes, maxOpen })`.
|
|
44
54
|
- **Cross-instance read-your-writes**: a commit returns
|
|
45
|
-
`
|
|
55
|
+
`{ braid, generation }`; a session token is the pointwise max of every
|
|
46
56
|
pair a flow has seen; `replica.waitFor(vector)` refreshes until the
|
|
47
57
|
local vector dominates it. The committing instance always reads its
|
|
48
58
|
own writes without waiting. A singleton map is the single-braid form.
|
|
@@ -54,15 +64,12 @@ if (out.tag === "accepted") ctx.waitUntil(replica.refresh(out.braid))
|
|
|
54
64
|
row — the schema idiom) or resident mode. `{ kind: "slot-race", tip }`
|
|
55
65
|
means the terminal losses were accepted but out-raced: an operational
|
|
56
66
|
signal to shard the theory into more braids or move the hot braid to
|
|
57
|
-
a resident Rust writer, whose group commit batches the queue.
|
|
58
|
-
package ships no group commit of its own; the recorded reopen trigger
|
|
59
|
-
is a measured TS deployment at Turso-density write rates where a
|
|
60
|
-
deliberate batching delay would amortize many writers into one PUT.
|
|
67
|
+
a resident Rust writer, whose group commit batches the queue.
|
|
61
68
|
|
|
62
|
-
##
|
|
69
|
+
## A local fleet
|
|
63
70
|
|
|
64
71
|
```ts
|
|
65
|
-
// one process per scope loop; all processes share one
|
|
72
|
+
// one process per scope loop; all processes share one fsStore prefix
|
|
66
73
|
import { fsStore, openReplica, openWriter } from "@bjornpagen/bumbledb-log"
|
|
67
74
|
|
|
68
75
|
const replica = await openReplica({
|
|
@@ -71,7 +78,7 @@ const replica = await openReplica({
|
|
|
71
78
|
dir: `/data/primer/replicas/${scopeName}`, // per-process local dir — never shared
|
|
72
79
|
theory: Explanation
|
|
73
80
|
})
|
|
74
|
-
const writer = openWriter(replica)
|
|
81
|
+
const writer = await openWriter(replica)
|
|
75
82
|
|
|
76
83
|
// one pass = refresh, render, emit, lower, one commit
|
|
77
84
|
await replica.refresh()
|
|
@@ -109,7 +116,8 @@ of the object file and its parent directory.
|
|
|
109
116
|
Exported sentinel values on the SDK idiom, checked with `errors.is`,
|
|
110
117
|
never by message strings: `ErrRefused` (typed per cause — batch shape,
|
|
111
118
|
version, fingerprint, manifest shape, checkpoint braid-set drift),
|
|
112
|
-
`
|
|
119
|
+
`ErrManifestMissing` (a replica found no manifest; only the writer
|
|
120
|
+
births a store), `ErrSpanningCommit`, `ErrGapDetected`, `ErrReplayDiverged`,
|
|
113
121
|
`ErrChainMismatch` (cause `"prev" | "slot" |
|
|
114
122
|
"timestamp"`), `ErrContention` (cause `hot-key` or `slot-race`),
|
|
115
123
|
`ErrStore` (the vendor channel, present in every wrapped store
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bjornpagen/bumbledb-log",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.19.0",
|
|
4
4
|
"description": "Braided object-store replication for bumbledb: the command codec, theory-derived braids, and replica/writer over five store verbs",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -32,10 +32,12 @@
|
|
|
32
32
|
},
|
|
33
33
|
"homepage": "https://github.com/bjornpagen/bumbledb#readme",
|
|
34
34
|
"dependencies": {
|
|
35
|
-
"@
|
|
35
|
+
"@aws-sdk/client-s3": "^3.1116.0",
|
|
36
|
+
"@superbuilders/errors": "^4.0.2",
|
|
37
|
+
"arkregex": "^0.0.8"
|
|
36
38
|
},
|
|
37
39
|
"peerDependencies": {
|
|
38
|
-
"@bjornpagen/bumbledb": "^0.
|
|
40
|
+
"@bjornpagen/bumbledb": "^0.19.0"
|
|
39
41
|
},
|
|
40
42
|
"devDependencies": {
|
|
41
43
|
"@biomejs/biome": "2.5.4",
|
package/src/braids.ts
CHANGED
|
@@ -5,14 +5,13 @@
|
|
|
5
5
|
* the descriptor, pinned cross-language by the codec goldens.
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
-
import type {
|
|
9
|
-
import { descriptorOf } from "#descriptor.ts"
|
|
8
|
+
import type { Braid, SerialStatement, Theory } from "#descriptor.ts"
|
|
9
|
+
import { braid, braidHex, descriptorOf } from "#descriptor.ts"
|
|
10
10
|
|
|
11
|
-
|
|
12
|
-
type Braid = string
|
|
11
|
+
const U32_MAX = 0xffffffff
|
|
13
12
|
|
|
14
13
|
/** The schema's own shard map: ordinary relation name → braid id. */
|
|
15
|
-
function braidsOf(theory:
|
|
14
|
+
function braidsOf(theory: Theory): ReadonlyMap<string, Braid> {
|
|
16
15
|
const descriptor = descriptorOf(theory)
|
|
17
16
|
const out = new Map<string, Braid>()
|
|
18
17
|
for (const relation of descriptor.relations) {
|
|
@@ -30,9 +29,22 @@ function braidsOf(theory: LogTheory): ReadonlyMap<string, Braid> {
|
|
|
30
29
|
* serializes at that statement. Typed data beside the braid map, one
|
|
31
30
|
* question per verb.
|
|
32
31
|
*/
|
|
33
|
-
function serialAtStatementsOf(theory:
|
|
32
|
+
function serialAtStatementsOf(theory: Theory): readonly SerialStatement[] {
|
|
34
33
|
return descriptorOf(theory).serialAtStatements
|
|
35
34
|
}
|
|
36
35
|
|
|
36
|
+
/**
|
|
37
|
+
* Parses a wire u32 into a braid id: valid only when the relation it
|
|
38
|
+
* names is the smallest in its own component. An unknown, closed, or
|
|
39
|
+
* non-head id is not a braid — the caller refuses, it is not ignored.
|
|
40
|
+
*/
|
|
41
|
+
function parse(theory: Theory, raw: number): Braid | undefined {
|
|
42
|
+
if (!Number.isInteger(raw) || raw < 0 || raw > U32_MAX) {
|
|
43
|
+
return undefined
|
|
44
|
+
}
|
|
45
|
+
const name = braidHex(raw)
|
|
46
|
+
return descriptorOf(theory).braidMembers.has(name) ? name : undefined
|
|
47
|
+
}
|
|
48
|
+
|
|
37
49
|
export type { Braid }
|
|
38
|
-
export { braidsOf, serialAtStatementsOf }
|
|
50
|
+
export { braid, braidsOf, parse, serialAtStatementsOf }
|
package/src/bytes.ts
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Little-endian byte primitives shared by the codec
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
2
|
+
* Little-endian byte primitives shared by the codec and the
|
|
3
|
+
* fingerprint mirror, plus the hex grammar every document digest
|
|
4
|
+
* walks. Every multi-byte integer on the batch wire is little-endian;
|
|
5
|
+
* the fingerprint's canonical literal encoding is the engine's
|
|
6
|
+
* big-endian order-preserving form; a digest is 32 bytes, rendered as
|
|
7
|
+
* 64 lowercase hex characters. Integer order and hex width live here
|
|
8
|
+
* so no third spelling can appear.
|
|
7
9
|
*/
|
|
8
10
|
|
|
9
11
|
import * as errors from "@superbuilders/errors"
|
|
@@ -13,6 +15,13 @@ const I64_MIN = -0x8000000000000000n
|
|
|
13
15
|
const I64_MAX = 0x7fffffffffffffffn
|
|
14
16
|
const I64_SIGN_BIT = 0x8000000000000000n
|
|
15
17
|
|
|
18
|
+
const utf8Encoder = new TextEncoder()
|
|
19
|
+
/** Fatal UTF-8. ignoreBOM is true: a leading U+FEFF is a character, not a stripped BOM. */
|
|
20
|
+
const utf8StrictDecoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true })
|
|
21
|
+
|
|
22
|
+
declare const digest32Brand: unique symbol
|
|
23
|
+
type Digest32 = Uint8Array & { readonly [digest32Brand]: typeof digest32Brand }
|
|
24
|
+
|
|
16
25
|
class ByteWriter {
|
|
17
26
|
private buf: Uint8Array
|
|
18
27
|
private len = 0
|
|
@@ -46,6 +55,10 @@ class ByteWriter {
|
|
|
46
55
|
this.len += raw.length
|
|
47
56
|
}
|
|
48
57
|
|
|
58
|
+
array32(value: Digest32): void {
|
|
59
|
+
this.bytes(value)
|
|
60
|
+
}
|
|
61
|
+
|
|
49
62
|
u16le(value: number): void {
|
|
50
63
|
this.grow(2)
|
|
51
64
|
this.buf[this.len] = value & 0xff
|
|
@@ -148,6 +161,10 @@ class ByteReader {
|
|
|
148
161
|
return new Uint8Array(this.take(count, what))
|
|
149
162
|
}
|
|
150
163
|
|
|
164
|
+
array32(what: string): Digest32 {
|
|
165
|
+
return digest32(this.take(32, what))
|
|
166
|
+
}
|
|
167
|
+
|
|
151
168
|
u16le(what: string): number {
|
|
152
169
|
const raw = this.take(2, what)
|
|
153
170
|
return (raw[0] ?? 0) | ((raw[1] ?? 0) << 8)
|
|
@@ -198,6 +215,16 @@ function bytesCompare(a: Uint8Array, b: Uint8Array): number {
|
|
|
198
215
|
|
|
199
216
|
const HEX_DIGITS = "0123456789abcdef"
|
|
200
217
|
|
|
218
|
+
function hexNibble(byte: number): number | undefined {
|
|
219
|
+
if (byte >= 0x30 && byte <= 0x39) {
|
|
220
|
+
return byte - 0x30
|
|
221
|
+
}
|
|
222
|
+
if (byte >= 0x61 && byte <= 0x66) {
|
|
223
|
+
return byte - 0x61 + 10
|
|
224
|
+
}
|
|
225
|
+
return undefined
|
|
226
|
+
}
|
|
227
|
+
|
|
201
228
|
function toHex(bytes: Uint8Array): string {
|
|
202
229
|
let out = ""
|
|
203
230
|
for (const byte of bytes) {
|
|
@@ -208,27 +235,68 @@ function toHex(bytes: Uint8Array): string {
|
|
|
208
235
|
}
|
|
209
236
|
|
|
210
237
|
function fromHex(hex: string): Uint8Array {
|
|
211
|
-
|
|
238
|
+
const raw = utf8Encoder.encode(hex)
|
|
239
|
+
if (raw.length % 2 !== 0) {
|
|
212
240
|
throw errors.new(`not lowercase hex: ${hex}`)
|
|
213
241
|
}
|
|
214
|
-
const out = new Uint8Array(
|
|
215
|
-
for (let i = 0; i <
|
|
216
|
-
|
|
242
|
+
const out = new Uint8Array(raw.length / 2)
|
|
243
|
+
for (let i = 0, j = 0; i < raw.length; i += 2, j++) {
|
|
244
|
+
const hiByte = raw[i]
|
|
245
|
+
const loByte = raw[i + 1]
|
|
246
|
+
if (hiByte === undefined || loByte === undefined) {
|
|
247
|
+
throw errors.new(`not lowercase hex: ${hex}`)
|
|
248
|
+
}
|
|
249
|
+
const hi = hexNibble(hiByte)
|
|
250
|
+
const lo = hexNibble(loByte)
|
|
251
|
+
if (hi === undefined || lo === undefined) {
|
|
252
|
+
throw errors.new(`not lowercase hex: ${hex}`)
|
|
253
|
+
}
|
|
254
|
+
out[j] = (hi << 4) | lo
|
|
217
255
|
}
|
|
218
256
|
return out
|
|
219
257
|
}
|
|
220
258
|
|
|
221
|
-
|
|
222
|
-
|
|
259
|
+
function digest32(bytes: Uint8Array): Digest32 {
|
|
260
|
+
if (bytes.length !== 32) {
|
|
261
|
+
throw errors.new(`digest is not 32 bytes: ${bytes.length}`)
|
|
262
|
+
}
|
|
263
|
+
const out = new Uint8Array(32)
|
|
264
|
+
out.set(bytes)
|
|
265
|
+
return out as Digest32
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
function digest32FromHex(hex: string): Digest32 {
|
|
269
|
+
return digest32(fromHex(hex))
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
function hex32(bytes: Digest32): string {
|
|
273
|
+
return toHex(bytes)
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
function saturatingAddU64(a: bigint, b: bigint): bigint {
|
|
277
|
+
const sum = a + b
|
|
278
|
+
return sum > U64_MAX ? U64_MAX : sum
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
function checkedAddU64(a: bigint, b: bigint): bigint | undefined {
|
|
282
|
+
const sum = a + b
|
|
283
|
+
return sum > U64_MAX ? undefined : sum
|
|
284
|
+
}
|
|
223
285
|
|
|
286
|
+
export type { Digest32 }
|
|
224
287
|
export {
|
|
225
288
|
ByteReader,
|
|
226
289
|
ByteWriter,
|
|
227
290
|
bytesCompare,
|
|
228
291
|
bytesEqual,
|
|
292
|
+
checkedAddU64,
|
|
293
|
+
digest32,
|
|
294
|
+
digest32FromHex,
|
|
229
295
|
fromHex,
|
|
296
|
+
hex32,
|
|
230
297
|
I64_MAX,
|
|
231
298
|
I64_MIN,
|
|
299
|
+
saturatingAddU64,
|
|
232
300
|
toHex,
|
|
233
301
|
U64_MAX,
|
|
234
302
|
utf8Encoder,
|
package/src/chain.ts
CHANGED
|
@@ -1,93 +1,209 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* The chain sidecar
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
2
|
+
* The chain sidecar in `dir/chain`: a floor cache of chain
|
|
3
|
+
* position, written atomically (temp + rename, fsync). The chain is
|
|
4
|
+
* Settled or Pending — generation is the vector sum, plus one exactly
|
|
5
|
+
* when the value is Pending. The document is a binary v:3 record:
|
|
6
|
+
* version byte 3, counted roster of braid / g / prev / ts, pending
|
|
7
|
+
* tag. Wire pending is the batch bytes. The content address is blake3
|
|
8
|
+
* of those bytes. Every integer is little-endian.
|
|
8
9
|
*/
|
|
9
10
|
|
|
10
11
|
import * as crypto from "node:crypto"
|
|
11
12
|
import * as fs from "node:fs/promises"
|
|
12
13
|
import * as path from "node:path"
|
|
13
14
|
import * as errors from "@superbuilders/errors"
|
|
14
|
-
import {
|
|
15
|
+
import { ByteReader, ByteWriter, saturatingAddU64, U64_MAX } from "#bytes.ts"
|
|
16
|
+
import type { ChainEntry } from "#codec.ts"
|
|
17
|
+
import type { Braid } from "#descriptor.ts"
|
|
18
|
+
import { braidHex } from "#descriptor.ts"
|
|
19
|
+
import { refuse } from "#errors.ts"
|
|
20
|
+
import type { Generation } from "#keys.ts"
|
|
21
|
+
import { generation } from "#keys.ts"
|
|
22
|
+
import { Vector } from "#vector.ts"
|
|
15
23
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
24
|
+
/** The sidecar's file name inside a replica directory. */
|
|
25
|
+
const CHAIN_FILE = "chain"
|
|
26
|
+
|
|
27
|
+
const VERSION = 3
|
|
28
|
+
const SETTLED = 0
|
|
29
|
+
const PENDING = 1
|
|
30
|
+
/** u32 braid + u64 g + 32 prev + u64 ts. */
|
|
31
|
+
const ENTRY_BYTES = 52n
|
|
21
32
|
|
|
22
|
-
interface
|
|
23
|
-
readonly braid:
|
|
24
|
-
readonly gen:
|
|
33
|
+
interface Pending {
|
|
34
|
+
readonly braid: Braid
|
|
35
|
+
readonly gen: Generation
|
|
25
36
|
readonly bytes: Uint8Array
|
|
26
37
|
}
|
|
27
38
|
|
|
28
|
-
|
|
29
|
-
readonly
|
|
30
|
-
readonly pending:
|
|
39
|
+
type Chain =
|
|
40
|
+
| { readonly tag: "settled"; readonly entries: ReadonlyMap<Braid, ChainEntry> }
|
|
41
|
+
| { readonly tag: "pending"; readonly entries: ReadonlyMap<Braid, ChainEntry>; readonly batch: Pending }
|
|
42
|
+
|
|
43
|
+
type SidecarRead =
|
|
44
|
+
| { readonly tag: "absent" }
|
|
45
|
+
| { readonly tag: "fault"; readonly io: Error }
|
|
46
|
+
| { readonly tag: "corrupt"; readonly parse: Error }
|
|
47
|
+
| { readonly tag: "read"; readonly chain: Chain }
|
|
48
|
+
|
|
49
|
+
function codeOf(error: Error): string | undefined {
|
|
50
|
+
return (error as NodeJS.ErrnoException).code
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function braidIdOf(id: Braid): number {
|
|
54
|
+
return Number.parseInt(id.slice(1), 16)
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function vectorOf(entries: ReadonlyMap<Braid, { readonly g: bigint }>): Vector {
|
|
58
|
+
const counts = new Map<Braid, bigint>()
|
|
59
|
+
for (const [braid, entry] of entries) {
|
|
60
|
+
counts.set(braid, entry.g)
|
|
61
|
+
}
|
|
62
|
+
return Vector.from(counts)
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function chainSum(chain: Chain): bigint {
|
|
66
|
+
const sum = vectorOf(chain.entries).sum()
|
|
67
|
+
return typeof sum === "bigint" ? sum : U64_MAX
|
|
31
68
|
}
|
|
32
69
|
|
|
33
|
-
function
|
|
34
|
-
const
|
|
35
|
-
|
|
36
|
-
.map(function renderEntry(braid) {
|
|
37
|
-
const entry = sidecar.chain.get(braid)
|
|
38
|
-
if (entry === undefined) {
|
|
39
|
-
throw errors.new(`sidecar chain lost braid ${braid}`)
|
|
40
|
-
}
|
|
41
|
-
return `"${braid}":{"g":${entry.g},"prev":"${entry.prev}","ts":${entry.ts}}`
|
|
42
|
-
})
|
|
43
|
-
.join(",")
|
|
44
|
-
const pending =
|
|
45
|
-
sidecar.pending === null
|
|
46
|
-
? "null"
|
|
47
|
-
: `{"braid":"${sidecar.pending.braid}","gen":${sidecar.pending.gen},"bytes":"${Buffer.from(sidecar.pending.bytes).toString("base64")}"}`
|
|
48
|
-
return `{"v":2,"chain":{${chain}},"pending":${pending}}`
|
|
70
|
+
function chainGeneration(chain: Chain): bigint {
|
|
71
|
+
const sum = chainSum(chain)
|
|
72
|
+
return chain.tag === "settled" ? sum : saturatingAddU64(sum, 1n)
|
|
49
73
|
}
|
|
50
74
|
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
}
|
|
57
|
-
if (
|
|
58
|
-
|
|
59
|
-
}
|
|
60
|
-
const chain = new Map<string, ChainEntry>()
|
|
61
|
-
for (const [braid, entry] of Object.entries(parsed.chain)) {
|
|
62
|
-
chain.set(braid, { g: BigInt(entry.g), prev: entry.prev, ts: BigInt(entry.ts) })
|
|
63
|
-
}
|
|
64
|
-
const pending =
|
|
65
|
-
parsed.pending === null
|
|
66
|
-
? null
|
|
67
|
-
: {
|
|
68
|
-
braid: parsed.pending.braid,
|
|
69
|
-
gen: BigInt(parsed.pending.gen),
|
|
70
|
-
bytes: new Uint8Array(Buffer.from(parsed.pending.bytes, "base64"))
|
|
71
|
-
}
|
|
72
|
-
return { chain, pending }
|
|
75
|
+
/** A declared count the remaining bytes cannot open is Malformed
|
|
76
|
+
* before the loop. */
|
|
77
|
+
function refuseUnbacked(count: bigint, remaining: number, minItem: bigint, at: string): void {
|
|
78
|
+
if (count === 0n) {
|
|
79
|
+
return
|
|
80
|
+
}
|
|
81
|
+
if (minItem === 0n || BigInt(remaining) / minItem < count) {
|
|
82
|
+
refuse({ kind: "Malformed", at: remaining }, `declared ${at} ${count} outruns the remaining ${remaining} bytes`)
|
|
83
|
+
}
|
|
73
84
|
}
|
|
74
85
|
|
|
75
|
-
|
|
86
|
+
function renderSidecar(chain: Chain): Uint8Array {
|
|
87
|
+
const out = new ByteWriter(64)
|
|
88
|
+
out.u8(VERSION)
|
|
89
|
+
const braids = [...chain.entries.keys()].sort()
|
|
90
|
+
if (braids.length > 0xffffffff) {
|
|
91
|
+
throw errors.new("sidecar chain count exceeds u32")
|
|
92
|
+
}
|
|
93
|
+
out.u32le(braids.length)
|
|
94
|
+
for (const id of braids) {
|
|
95
|
+
const entry = chain.entries.get(id)
|
|
96
|
+
if (entry === undefined) {
|
|
97
|
+
throw errors.new(`sidecar chain lost braid ${id}`)
|
|
98
|
+
}
|
|
99
|
+
out.u32le(braidIdOf(id))
|
|
100
|
+
out.u64le(entry.g)
|
|
101
|
+
out.bytes(entry.prev)
|
|
102
|
+
out.u64le(entry.ts)
|
|
103
|
+
}
|
|
104
|
+
if (chain.tag === "settled") {
|
|
105
|
+
out.u8(SETTLED)
|
|
106
|
+
return out.finish()
|
|
107
|
+
}
|
|
108
|
+
out.u8(PENDING)
|
|
109
|
+
out.u32le(braidIdOf(chain.batch.braid))
|
|
110
|
+
out.u64le(chain.batch.gen)
|
|
111
|
+
if (chain.batch.bytes.length > 0xffffffff) {
|
|
112
|
+
throw errors.new("sidecar pending exceeds u32 length")
|
|
113
|
+
}
|
|
114
|
+
out.u32le(chain.batch.bytes.length)
|
|
115
|
+
out.bytes(chain.batch.bytes)
|
|
116
|
+
return out.finish()
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function parseSidecar(bytes: Uint8Array, known?: ReadonlySet<Braid>): Chain {
|
|
120
|
+
const reader = new ByteReader(bytes, {
|
|
121
|
+
fail(what: string): never {
|
|
122
|
+
refuse({ kind: "Malformed", at: bytes.length }, `sidecar truncated at ${what}`)
|
|
123
|
+
}
|
|
124
|
+
})
|
|
125
|
+
const at = function offset(): number {
|
|
126
|
+
return bytes.length - reader.remaining()
|
|
127
|
+
}
|
|
128
|
+
const version = reader.u8("version")
|
|
129
|
+
if (version !== VERSION) {
|
|
130
|
+
refuse({ kind: "Version", version }, `sidecar version ${version}, consumers refuse ≠ ${VERSION}`)
|
|
131
|
+
}
|
|
132
|
+
const count = BigInt(reader.u32le("chain count"))
|
|
133
|
+
refuseUnbacked(count, reader.remaining(), ENTRY_BYTES, "chain count")
|
|
134
|
+
const entries = new Map<Braid, ChainEntry>()
|
|
135
|
+
let last: Braid | undefined
|
|
136
|
+
for (let i = 0n; i < count; i++) {
|
|
137
|
+
const raw = reader.u32le("braid")
|
|
138
|
+
const name = braidHex(raw)
|
|
139
|
+
if (known !== undefined && !known.has(name)) {
|
|
140
|
+
refuse({ kind: "UnknownBraid", braid: raw }, `sidecar cites unknown braid ${name}`)
|
|
141
|
+
}
|
|
142
|
+
const g = reader.u64le("g")
|
|
143
|
+
const prev = reader.array32("prev")
|
|
144
|
+
const ts = reader.u64le("ts")
|
|
145
|
+
if (last !== undefined && last >= name) {
|
|
146
|
+
refuse({ kind: "Malformed", at: at() }, "sidecar chain is not strictly ascending")
|
|
147
|
+
}
|
|
148
|
+
entries.set(name, { g: generation(g), prev, ts })
|
|
149
|
+
last = name
|
|
150
|
+
}
|
|
151
|
+
if (typeof vectorOf(entries).sum() !== "bigint") {
|
|
152
|
+
refuse({ kind: "Overflow" }, "sidecar chain sum overflows u64")
|
|
153
|
+
}
|
|
154
|
+
const tag = reader.u8("pending")
|
|
155
|
+
if (tag === SETTLED) {
|
|
156
|
+
if (reader.remaining() !== 0) {
|
|
157
|
+
refuse({ kind: "Malformed", at: reader.remaining() }, `${reader.remaining()} trailing bytes after the sidecar`)
|
|
158
|
+
}
|
|
159
|
+
return { tag: "settled", entries }
|
|
160
|
+
}
|
|
161
|
+
if (tag !== PENDING) {
|
|
162
|
+
refuse({ kind: "Malformed", at: at() - 1 }, `sidecar pending tag ${tag}`)
|
|
163
|
+
}
|
|
164
|
+
const raw = reader.u32le("pending braid")
|
|
165
|
+
const name = braidHex(raw)
|
|
166
|
+
if (known !== undefined && !known.has(name)) {
|
|
167
|
+
refuse({ kind: "UnknownBraid", braid: raw }, `sidecar pending cites unknown braid ${name}`)
|
|
168
|
+
}
|
|
169
|
+
const slot = reader.u64le("pending generation")
|
|
170
|
+
const length = reader.u32le("pending length")
|
|
171
|
+
const body = reader.bytes(length, "pending bytes")
|
|
172
|
+
if (reader.remaining() !== 0) {
|
|
173
|
+
refuse({ kind: "Malformed", at: reader.remaining() }, `${reader.remaining()} trailing bytes after the sidecar`)
|
|
174
|
+
}
|
|
175
|
+
return {
|
|
176
|
+
tag: "pending",
|
|
177
|
+
entries,
|
|
178
|
+
batch: { braid: name, gen: generation(slot), bytes: body }
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
async function readSidecar(file: string, known?: ReadonlySet<Braid>): Promise<SidecarRead> {
|
|
76
183
|
const read = await errors.try(fs.readFile(file))
|
|
77
184
|
if (read.error) {
|
|
78
|
-
|
|
185
|
+
if (codeOf(read.error) === "ENOENT") {
|
|
186
|
+
return { tag: "absent" }
|
|
187
|
+
}
|
|
188
|
+
return { tag: "fault", io: read.error }
|
|
189
|
+
}
|
|
190
|
+
const parsed = errors.trySync(function parse() {
|
|
191
|
+
return parseSidecar(read.data, known)
|
|
192
|
+
})
|
|
193
|
+
if (parsed.error) {
|
|
194
|
+
return { tag: "corrupt", parse: parsed.error }
|
|
79
195
|
}
|
|
80
|
-
return
|
|
196
|
+
return { tag: "read", chain: parsed.data }
|
|
81
197
|
}
|
|
82
198
|
|
|
83
|
-
async function writeSidecar(file: string,
|
|
199
|
+
async function writeSidecar(file: string, chain: Chain): Promise<void> {
|
|
84
200
|
const dir = path.dirname(file)
|
|
85
201
|
await fs.mkdir(dir, { recursive: true })
|
|
86
202
|
const temp = path.join(dir, `.chain-${process.pid}-${crypto.randomBytes(4).toString("hex")}`)
|
|
87
203
|
const handle = await fs.open(temp, "wx")
|
|
88
204
|
const written = await errors.try(
|
|
89
205
|
(async function writeAll() {
|
|
90
|
-
await handle.writeFile(renderSidecar(
|
|
206
|
+
await handle.writeFile(renderSidecar(chain))
|
|
91
207
|
await handle.sync()
|
|
92
208
|
})()
|
|
93
209
|
)
|
|
@@ -105,5 +221,5 @@ async function writeSidecar(file: string, sidecar: Sidecar): Promise<void> {
|
|
|
105
221
|
}
|
|
106
222
|
}
|
|
107
223
|
|
|
108
|
-
export type { ChainEntry,
|
|
109
|
-
export { readSidecar, renderSidecar, writeSidecar }
|
|
224
|
+
export type { Chain, ChainEntry, Pending, SidecarRead }
|
|
225
|
+
export { CHAIN_FILE, chainGeneration, chainSum, parseSidecar, readSidecar, renderSidecar, writeSidecar }
|