@optimystic/db-core 1.5.0 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. package/README.md +336 -336
  2. package/dist/src/cluster/structs.d.ts +37 -0
  3. package/dist/src/cluster/structs.d.ts.map +1 -1
  4. package/dist/src/cluster/structs.js +13 -0
  5. package/dist/src/cluster/structs.js.map +1 -1
  6. package/dist/src/collection/collection.d.ts +44 -4
  7. package/dist/src/collection/collection.d.ts.map +1 -1
  8. package/dist/src/collection/collection.js +95 -14
  9. package/dist/src/collection/collection.js.map +1 -1
  10. package/dist/src/log/log.d.ts +41 -8
  11. package/dist/src/log/log.d.ts.map +1 -1
  12. package/dist/src/log/log.js +15 -10
  13. package/dist/src/log/log.js.map +1 -1
  14. package/dist/src/testing/test-transactor.d.ts +36 -0
  15. package/dist/src/testing/test-transactor.d.ts.map +1 -1
  16. package/dist/src/testing/test-transactor.js +49 -0
  17. package/dist/src/testing/test-transactor.js.map +1 -1
  18. package/dist/src/transaction/coordinator.d.ts.map +1 -1
  19. package/dist/src/transaction/coordinator.js +26 -9
  20. package/dist/src/transaction/coordinator.js.map +1 -1
  21. package/package.json +1 -1
  22. package/src/blocks/block-store.ts +31 -31
  23. package/src/blocks/block-types.ts +11 -11
  24. package/src/blocks/index.ts +5 -5
  25. package/src/blocks/structs.ts +17 -17
  26. package/src/btree/btree.ts +872 -872
  27. package/src/btree/independent-trunk.ts +53 -53
  28. package/src/btree/index.ts +5 -5
  29. package/src/btree/key-range.ts +15 -15
  30. package/src/btree/keyset.ts +6 -6
  31. package/src/btree/nodes.ts +25 -25
  32. package/src/btree/path.ts +37 -37
  33. package/src/btree/tree-block.ts +11 -11
  34. package/src/btree/trunk.ts +14 -14
  35. package/src/chain/chain-nodes.ts +26 -26
  36. package/src/chain/chain.ts +349 -349
  37. package/src/chain/index.ts +3 -3
  38. package/src/cluster/i-cluster.ts +6 -6
  39. package/src/cluster/index.ts +3 -3
  40. package/src/cluster/structs.ts +38 -0
  41. package/src/cohort-topic/addressing.ts +120 -120
  42. package/src/cohort-topic/antidos/bootstrap-evidence.ts +106 -106
  43. package/src/cohort-topic/antidos/index.ts +5 -5
  44. package/src/cohort-topic/antidos/rate-limiter.ts +210 -210
  45. package/src/cohort-topic/antidos/replay-guard.ts +146 -146
  46. package/src/cohort-topic/antidos/topic-budget.ts +160 -160
  47. package/src/cohort-topic/antiflood/index.ts +2 -2
  48. package/src/cohort-topic/antiflood/invariants.ts +108 -108
  49. package/src/cohort-topic/antiflood/jitter.ts +117 -117
  50. package/src/cohort-topic/coldstart.ts +237 -237
  51. package/src/cohort-topic/dmax.ts +88 -88
  52. package/src/cohort-topic/gossip/bus.ts +254 -254
  53. package/src/cohort-topic/gossip/index.ts +3 -3
  54. package/src/cohort-topic/gossip/records.ts +45 -45
  55. package/src/cohort-topic/gossip/view.ts +91 -91
  56. package/src/cohort-topic/index.ts +20 -20
  57. package/src/cohort-topic/load/barometer.ts +134 -134
  58. package/src/cohort-topic/load/index.ts +1 -1
  59. package/src/cohort-topic/member-engine.ts +430 -430
  60. package/src/cohort-topic/membership/index.ts +3 -3
  61. package/src/cohort-topic/membership/publisher.ts +163 -163
  62. package/src/cohort-topic/membership/source.ts +41 -41
  63. package/src/cohort-topic/ports.ts +157 -157
  64. package/src/cohort-topic/promotion.ts +477 -477
  65. package/src/cohort-topic/registration/bytes.ts +37 -37
  66. package/src/cohort-topic/registration/handoff.ts +154 -154
  67. package/src/cohort-topic/registration/index.ts +6 -6
  68. package/src/cohort-topic/registration/renewal.ts +495 -495
  69. package/src/cohort-topic/registration/sharding.ts +61 -61
  70. package/src/cohort-topic/registration/store.ts +81 -81
  71. package/src/cohort-topic/registration/types.ts +91 -91
  72. package/src/cohort-topic/ring-hash.ts +50 -50
  73. package/src/cohort-topic/service.ts +416 -416
  74. package/src/cohort-topic/sig/index.ts +2 -2
  75. package/src/cohort-topic/sig/payloads.ts +59 -59
  76. package/src/cohort-topic/sig/threshold.ts +64 -64
  77. package/src/cohort-topic/tiers.ts +74 -74
  78. package/src/cohort-topic/traffic.ts +233 -233
  79. package/src/cohort-topic/walk.ts +326 -326
  80. package/src/cohort-topic/willingness.ts +237 -237
  81. package/src/cohort-topic/wire/index.ts +18 -18
  82. package/src/cohort-topic/wire/payloads.ts +126 -126
  83. package/src/collection/action.ts +48 -48
  84. package/src/collection/collection-type-registry.ts +37 -37
  85. package/src/collection/collection.ts +2203 -2118
  86. package/src/collection/index.ts +4 -4
  87. package/src/collections/diary/index.ts +2 -2
  88. package/src/collections/diary/struct.ts +3 -3
  89. package/src/collections/index.ts +2 -2
  90. package/src/collections/tree/collection-trunk.ts +25 -25
  91. package/src/collections/tree/index.ts +2 -2
  92. package/src/collections/tree/readme.md +55 -55
  93. package/src/log/index.ts +2 -2
  94. package/src/log/log.ts +304 -268
  95. package/src/log/struct.ts +165 -165
  96. package/src/matchmaking/capability-filter.ts +45 -45
  97. package/src/matchmaking/config.ts +98 -98
  98. package/src/matchmaking/index.ts +21 -21
  99. package/src/matchmaking/multi-cohort-seeker.ts +234 -234
  100. package/src/matchmaking/provider.ts +123 -123
  101. package/src/matchmaking/query-eval.ts +105 -105
  102. package/src/matchmaking/seeker-walk.ts +127 -127
  103. package/src/matchmaking/seeker.ts +86 -86
  104. package/src/matchmaking/topic-anchor.ts +90 -90
  105. package/src/matchmaking/voting-quorum.ts +394 -394
  106. package/src/network/i-peer-network.ts +25 -25
  107. package/src/network/repo-protocol.ts +13 -13
  108. package/src/network/types.ts +37 -37
  109. package/src/reactivity/backfill.ts +220 -220
  110. package/src/reactivity/backpressure.ts +191 -191
  111. package/src/reactivity/checkpoint.ts +308 -308
  112. package/src/reactivity/config.ts +172 -172
  113. package/src/reactivity/dedupe.ts +132 -132
  114. package/src/reactivity/forwarder.ts +87 -87
  115. package/src/reactivity/index.ts +34 -34
  116. package/src/reactivity/notification.ts +123 -123
  117. package/src/reactivity/policy.ts +79 -79
  118. package/src/reactivity/push-state.ts +310 -310
  119. package/src/reactivity/recover.ts +153 -153
  120. package/src/reactivity/replay-buffer.ts +141 -141
  121. package/src/reactivity/resume.ts +549 -549
  122. package/src/reactivity/rotation.ts +415 -415
  123. package/src/reactivity/subscriber.ts +132 -132
  124. package/src/reactivity/subscription.ts +66 -66
  125. package/src/reactivity/topic-anchor.ts +71 -71
  126. package/src/reactivity/verify.ts +73 -73
  127. package/src/reactivity/wire-validate.ts +13 -13
  128. package/src/testing/async-wait.ts +65 -65
  129. package/src/testing/index.ts +2 -2
  130. package/src/testing/test-transactor.ts +62 -0
  131. package/src/transaction/actions-engine.ts +68 -68
  132. package/src/transaction/coordinator.ts +28 -11
  133. package/src/transaction/index.ts +52 -52
  134. package/src/transaction/operations-hash.ts +196 -196
  135. package/src/transaction/read-dependency-collector.ts +78 -78
  136. package/src/transaction/session.ts +238 -238
  137. package/src/transaction/transaction.ts +442 -442
  138. package/src/transaction/validator.ts +197 -197
  139. package/src/transactor/block-floors.ts +126 -126
  140. package/src/transactor/index.ts +6 -6
  141. package/src/transactor/network-transactor.ts +1545 -1545
  142. package/src/transform/atomic-proxy.ts +92 -92
  143. package/src/transform/cache-source.ts +370 -370
  144. package/src/transform/helpers.ts +164 -164
  145. package/src/transform/index.ts +7 -7
  146. package/src/transform/struct.ts +21 -21
  147. package/src/utility/actor.ts +62 -62
  148. package/src/utility/backoff.ts +95 -95
  149. package/src/utility/batch-coordinator.ts +191 -191
  150. package/src/utility/ensured.ts +32 -32
  151. package/src/utility/groupby.ts +18 -18
  152. package/src/utility/hash-string.ts +14 -14
  153. package/src/utility/is-record-empty.ts +5 -5
  154. package/src/utility/latches.ts +42 -42
  155. package/src/utility/lru-map.ts +60 -60
  156. package/src/utility/nameof.ts +7 -7
  157. package/src/utility/pending.ts +43 -43
@@ -1,25 +1,25 @@
1
- import type { AbortOptions, PeerId, Stream } from "./types.js";
2
-
3
- export type IPeerNetwork = {
4
- /**
5
- * Dial a peer and establish a protocol stream
6
- */
7
- connect(peerId: PeerId, protocol: string, options?: AbortOptions): Promise<Stream>;
8
-
9
- /**
10
- * Optionally teach the dialer how to reach `peerId`, from addresses carried by an
11
- * application-level message (a cluster record's peer map, a redirect payload).
12
- *
13
- * This exists because libp2p only propagates a peer's addresses to peers it is
14
- * DIRECTLY connected to. A cohort here is chosen by key position, so a member
15
- * routinely shares a cohort with a relay-only peer it has never met and holds an
16
- * empty address list for it — a dial by peer id alone then fails immediately. Our
17
- * own protocol messages already carry the addresses; this is the seam that lets a
18
- * recipient keep them.
19
- *
20
- * Addresses are multiaddr strings (matching `ClusterPeers` / `RedirectPayload`), so
21
- * db-core needs no multiaddr dependency. Implementations without an address book
22
- * omit this method.
23
- */
24
- recordPeerAddresses?(peerId: PeerId, multiaddrs: string[]): void;
25
- }
1
+ import type { AbortOptions, PeerId, Stream } from "./types.js";
2
+
3
+ export type IPeerNetwork = {
4
+ /**
5
+ * Dial a peer and establish a protocol stream
6
+ */
7
+ connect(peerId: PeerId, protocol: string, options?: AbortOptions): Promise<Stream>;
8
+
9
+ /**
10
+ * Optionally teach the dialer how to reach `peerId`, from addresses carried by an
11
+ * application-level message (a cluster record's peer map, a redirect payload).
12
+ *
13
+ * This exists because libp2p only propagates a peer's addresses to peers it is
14
+ * DIRECTLY connected to. A cohort here is chosen by key position, so a member
15
+ * routinely shares a cohort with a relay-only peer it has never met and holds an
16
+ * empty address list for it — a dial by peer id alone then fails immediately. Our
17
+ * own protocol messages already carry the addresses; this is the seam that lets a
18
+ * recipient keep them.
19
+ *
20
+ * Addresses are multiaddr strings (matching `ClusterPeers` / `RedirectPayload`), so
21
+ * db-core needs no multiaddr dependency. Implementations without an address book
22
+ * omit this method.
23
+ */
24
+ recordPeerAddresses?(peerId: PeerId, multiaddrs: string[]): void;
25
+ }
@@ -1,13 +1,13 @@
1
- import type { ActionBlocks, PendRequest, CommitRequest, BlockGets, InvalidateRequest } from "../index.js";
2
-
3
- export type RepoMessage = {
4
- operations: [
5
- { get: BlockGets } |
6
- { pend: PendRequest } |
7
- { cancel: { actionRef: ActionBlocks } } |
8
- { commit: CommitRequest } |
9
- { invalidate: InvalidateRequest }
10
- ],
11
- expiration?: number,
12
- coordinatingBlockIds?: string[],
13
- };
1
+ import type { ActionBlocks, PendRequest, CommitRequest, BlockGets, InvalidateRequest } from "../index.js";
2
+
3
+ export type RepoMessage = {
4
+ operations: [
5
+ { get: BlockGets } |
6
+ { pend: PendRequest } |
7
+ { cancel: { actionRef: ActionBlocks } } |
8
+ { commit: CommitRequest } |
9
+ { invalidate: InvalidateRequest }
10
+ ],
11
+ expiration?: number,
12
+ coordinatingBlockIds?: string[],
13
+ };
@@ -1,37 +1,37 @@
1
- /**
2
- * Portable type aliases for peer networking.
3
- *
4
- * These minimal structural types decouple db-core from any concrete
5
- * networking library (e.g. libp2p). Concrete implementations in
6
- * transport packages (db-p2p) satisfy these structurally.
7
- */
8
-
9
- /** Minimal peer identifier — structurally compatible with libp2p's PeerId. */
10
- export type PeerId = {
11
- toString(): string;
12
- equals(other: unknown): boolean;
13
- };
14
-
15
- /** Opaque network stream — db-core never accesses stream internals. */
16
- export type Stream = {
17
- close(): Promise<void>;
18
- };
19
-
20
- /** Options for abortable operations. */
21
- export type AbortOptions = {
22
- signal?: AbortSignal;
23
- };
24
-
25
- /** Create a lightweight PeerId from its string representation. */
26
- export function peerIdFromString(id: string): PeerId {
27
- return {
28
- toString: () => id,
29
- equals: (other: unknown) =>
30
- other != null
31
- && typeof other === 'object'
32
- && 'toString' in other
33
- && typeof (other as PeerId).toString === 'function'
34
- && (other as PeerId).toString() === id,
35
- };
36
- }
37
-
1
+ /**
2
+ * Portable type aliases for peer networking.
3
+ *
4
+ * These minimal structural types decouple db-core from any concrete
5
+ * networking library (e.g. libp2p). Concrete implementations in
6
+ * transport packages (db-p2p) satisfy these structurally.
7
+ */
8
+
9
+ /** Minimal peer identifier — structurally compatible with libp2p's PeerId. */
10
+ export type PeerId = {
11
+ toString(): string;
12
+ equals(other: unknown): boolean;
13
+ };
14
+
15
+ /** Opaque network stream — db-core never accesses stream internals. */
16
+ export type Stream = {
17
+ close(): Promise<void>;
18
+ };
19
+
20
+ /** Options for abortable operations. */
21
+ export type AbortOptions = {
22
+ signal?: AbortSignal;
23
+ };
24
+
25
+ /** Create a lightweight PeerId from its string representation. */
26
+ export function peerIdFromString(id: string): PeerId {
27
+ return {
28
+ toString: () => id,
29
+ equals: (other: unknown) =>
30
+ other != null
31
+ && typeof other === 'object'
32
+ && 'toString' in other
33
+ && typeof (other as PeerId).toString === 'function'
34
+ && (other as PeerId).toString() === id,
35
+ };
36
+ }
37
+
@@ -1,220 +1,220 @@
1
- /**
2
- * Reactivity — Backfill RPC (`docs/reactivity.md` §Backfill RPC, §Wire formats).
3
- *
4
- * A subscriber that detects a revision gap (slow-subscriber drop, brief sleep) requests the missing
5
- * range from its serving cohort; the cohort answers from its {@link ReplayBuffer} ring. The reply
6
- * carries the **intersection** of the requested range with the buffer plus the `available` window the
7
- * cohort actually holds, so a subscriber whose lag fell past the ring's low edge learns it must fall
8
- * back further (to a checkpoint resume or a chain read) rather than silently receiving a short answer.
9
- *
10
- * Entries retain the original threshold signature (the buffer stores full {@link NotificationV1}s), so a
11
- * backfill reply is verifiable **end-to-end** — the subscriber re-runs the same {@link NotificationVerifier}
12
- * over each backfilled entry it runs over a live notification. Any cohort member can serve, because the
13
- * replay buffer is gossiped across the cohort (origination ticket).
14
- *
15
- * Wire conventions match the rest of reactivity: JSON, byte fields base64url, unix-ms timestamps, `v: 1`,
16
- * per-message structural validation on decode. `BackfillV1` is signed by the subscriber's peer key
17
- * (`signature`); replay protection rides the request envelope as elsewhere in the substrate.
18
- */
19
-
20
- import {
21
- decodeCohortMessage,
22
- encodeCohortMessage,
23
- DEFAULT_MAX_MESSAGE_BYTES,
24
- } from "../cohort-topic/wire/codec.js";
25
- import {
26
- asObject,
27
- b64urlField,
28
- failWire,
29
- reqFiniteNumber,
30
- reqIntInRange,
31
- reqString,
32
- requireV1,
33
- } from "./wire-validate.js";
34
- import type { ReplayBuffer } from "./replay-buffer.js";
35
- import type { ReactivitySubscriber } from "./subscriber.js";
36
- import { validateNotificationArray, validateNotificationV1, type NotificationV1 } from "./wire.js";
37
-
38
- /** A subscriber's request to replay a contiguous revision range from a cohort's buffer. */
39
- export interface BackfillV1 {
40
- v: 1;
41
- /** Collection id, base64url. */
42
- collectionId: string;
43
- /** Inclusive low edge of the requested range. */
44
- fromRevision: number;
45
- /** Inclusive high edge of the requested range. */
46
- toRevision: number;
47
- /**
48
- * Unix ms — bound into {@link backfillSigningPayload} so the request's freshness is authenticated.
49
- * The serving handler's replay guard keys on the signature + this timestamp, so a captured request
50
- * cannot be replayed with a forged-fresh timestamp (the forged value invalidates the signature).
51
- */
52
- timestamp: number;
53
- /** Subscriber peer-key signature over the request. */
54
- signature: string;
55
- }
56
-
57
- /** The cohort's reply: the held intersection of the request, plus the window it actually has. */
58
- export interface BackfillReplyV1 {
59
- v: 1;
60
- /** The signed notifications in `[from, to] ∩ buffer`, ascending by revision. */
61
- entries: NotificationV1[];
62
- /** The revision window the serving cohort actually holds (so the subscriber can fall back further). */
63
- available: {
64
- fromRevision: number;
65
- toRevision: number;
66
- };
67
- }
68
-
69
- /** Narrow an already-parsed value to {@link BackfillV1}, throwing on any defect. */
70
- export function validateBackfillV1(value: unknown): BackfillV1 {
71
- const what = "BackfillV1";
72
- const obj = asObject(value, what);
73
- requireV1(obj, what);
74
- const fromRevision = reqIntInRange(obj, "fromRevision", what, 0);
75
- const toRevision = reqIntInRange(obj, "toRevision", what, fromRevision);
76
- return {
77
- v: 1,
78
- collectionId: b64urlField(reqString(obj, "collectionId", what), "collectionId", what),
79
- fromRevision,
80
- toRevision,
81
- timestamp: reqFiniteNumber(obj, "timestamp", what),
82
- signature: b64urlField(reqString(obj, "signature", what), "signature", what),
83
- };
84
- }
85
-
86
- const utf8 = new TextEncoder();
87
-
88
- /** A {@link BackfillV1} minus its `signature` — the canonical bytes the subscriber peer-key-signs. */
89
- export type BackfillSignable = Omit<BackfillV1, "signature">;
90
-
91
- /**
92
- * Canonical signed byte image of a {@link BackfillV1} (every field except `signature`). Mirrors the
93
- * cohort-topic `registerSigningPayload` pattern: determinism comes from encoding an explicitly-ordered,
94
- * type-tagged JSON array (array order is stable; object key order is not) as UTF-8. The leading
95
- * `"BackfillV1"` tag means a backfill image can never collide with the `"ResumeV1"` image even when the
96
- * shared fields are identical. Signer and verifier must agree byte-for-byte — the array order is the
97
- * contract (pinned by the determinism test).
98
- */
99
- export function backfillSigningPayload(body: BackfillSignable): Uint8Array {
100
- return utf8.encode(JSON.stringify([
101
- "BackfillV1",
102
- body.v,
103
- body.collectionId,
104
- body.fromRevision,
105
- body.toRevision,
106
- body.timestamp,
107
- ]));
108
- }
109
-
110
- /** Narrow an already-parsed value to {@link BackfillReplyV1}, throwing on any defect. */
111
- export function validateBackfillReplyV1(value: unknown): BackfillReplyV1 {
112
- const what = "BackfillReplyV1";
113
- const obj = asObject(value, what);
114
- requireV1(obj, what);
115
- const available = asObject(obj["available"], `${what}.available`);
116
- const availFrom = reqIntInRange(available, "fromRevision", `${what}.available`, 0);
117
- return {
118
- v: 1,
119
- entries: validateNotificationArray(obj["entries"], what),
120
- available: {
121
- fromRevision: availFrom,
122
- toRevision: reqIntInRange(available, "toRevision", `${what}.available`, availFrom),
123
- },
124
- };
125
- }
126
-
127
- /** Encode a {@link BackfillV1} as a length-prefixed UTF-8 JSON frame. */
128
- export function encodeBackfillV1(msg: BackfillV1, maxMessageBytes: number = DEFAULT_MAX_MESSAGE_BYTES): Uint8Array {
129
- return encodeCohortMessage(validateBackfillV1(msg), maxMessageBytes);
130
- }
131
-
132
- /** Decode a length-prefixed {@link BackfillV1} frame. */
133
- export function decodeBackfillV1(bytes: Uint8Array, maxMessageBytes?: number): BackfillV1 {
134
- return validateBackfillV1(decodeCohortMessage(bytes, maxMessageBytes));
135
- }
136
-
137
- /** Encode a {@link BackfillReplyV1} as a length-prefixed UTF-8 JSON frame. */
138
- export function encodeBackfillReplyV1(msg: BackfillReplyV1, maxMessageBytes: number = DEFAULT_MAX_MESSAGE_BYTES): Uint8Array {
139
- return encodeCohortMessage(validateBackfillReplyV1(msg), maxMessageBytes);
140
- }
141
-
142
- /** Decode a length-prefixed {@link BackfillReplyV1} frame. */
143
- export function decodeBackfillReplyV1(bytes: Uint8Array, maxMessageBytes?: number): BackfillReplyV1 {
144
- return validateBackfillReplyV1(decodeCohortMessage(bytes, maxMessageBytes));
145
- }
146
-
147
- /**
148
- * Serve a {@link BackfillV1} from a cohort's replay buffer: return the intersection of the requested
149
- * range with the buffer and report the buffer's actual `available` window. A request whose `collectionId`
150
- * does not match `expectedCollectionId` is rejected (a backfill is collection-scoped). When the buffer is
151
- * empty, `available` collapses to the request's `fromRevision` (an empty window) and no entries are returned.
152
- */
153
- export function serveBackfill(buffer: ReplayBuffer, req: BackfillV1, expectedCollectionId: string): BackfillReplyV1 {
154
- if (req.collectionId !== expectedCollectionId) {
155
- failWire(`serveBackfill: request collectionId does not match this cohort's collection`);
156
- }
157
- const entries = buffer.range(req.fromRevision, req.toRevision).map((e) => validateNotificationV1(e.payload));
158
- const low = buffer.lowRevision;
159
- const high = buffer.highRevision;
160
- const available = low === undefined || high === undefined
161
- ? { fromRevision: req.fromRevision, toRevision: req.fromRevision }
162
- : { fromRevision: low, toRevision: high };
163
- return { v: 1, entries, available };
164
- }
165
-
166
- // --- subscriber-side backfill requester (the `requestBackfill` seam ↔ this RPC) ---
167
-
168
- /** Sends a signed {@link BackfillV1} to the serving cohort and awaits its {@link BackfillReplyV1}. */
169
- export type BackfillTransport = (req: BackfillV1) => Promise<BackfillReplyV1>;
170
-
171
- /** Construction inputs for {@link createBackfillRequester}. */
172
- export interface BackfillRequesterDeps {
173
- /** Collection this subscription tracks, base64url. */
174
- readonly collectionId: string;
175
- /** Sign the request over its unsigned image (the subscriber's peer key); returns the base64url sig. */
176
- readonly sign: (req: Omit<BackfillV1, "signature">) => string;
177
- /** Send the request and await the reply (the db-p2p reactivity application protocol supplies this). */
178
- readonly transport: BackfillTransport;
179
- /** Re-enter each backfilled entry through the delivery path (verify → contiguity → deliver → dedupe). */
180
- readonly subscriber: ReactivitySubscriber;
181
- /** Unix-ms clock stamped into the signed `timestamp` (injected — db-core proper never reads an ambient clock). */
182
- readonly clock: () => number;
183
- /**
184
- * Called when the cohort's `available` window does not reach the gap's low edge (`available.fromRevision
185
- * > from`): the buffer no longer covers the gap, so the subscriber must escalate to a checkpoint resume
186
- * or a chain read (`docs/reactivity.md` §Backfill RPC, "fall back further").
187
- */
188
- readonly onUnderflow?: (requested: { from: number; to: number }, available: { fromRevision: number; toRevision: number }) => void;
189
- }
190
-
191
- /**
192
- * Build the `requestBackfill(from, to)` driver that connects the subscriber delivery path's backfill seam
193
- * ([reactivity-origination-replay-delivery]) to the {@link BackfillV1} RPC. It signs and sends the request,
194
- * replays the returned entries through {@link ReactivitySubscriber.onNotification} (so they close the gap,
195
- * verified and deduped), and reports an underflow when the cohort's `available` window fell past the gap's
196
- * low edge so the caller can fall back to a checkpoint resume or a chain read.
197
- *
198
- * Returns a `Promise`; the synchronous `(from, to) => void` seam wraps a call to this with `void`.
199
- */
200
- export function createBackfillRequester(deps: BackfillRequesterDeps): (from: number, to: number) => Promise<BackfillReplyV1> {
201
- return async (from: number, to: number): Promise<BackfillReplyV1> => {
202
- const unsigned: Omit<BackfillV1, "signature"> = { v: 1, collectionId: deps.collectionId, fromRevision: from, toRevision: to, timestamp: deps.clock() };
203
- const req: BackfillV1 = { ...unsigned, signature: deps.sign(unsigned) };
204
- const reply = await deps.transport(req);
205
- // Underflow: the cohort's held window does not reach the gap's low edge (`docs/reactivity.md`
206
- // §Backfill RPC, "fall back further"). The returned entries all sit *above* the still-unfillable
207
- // low range, so none can apply contiguously from `from`. Replaying them would only re-fire the
208
- // subscriber's gap → `requestBackfill` seam (which, when wired back to this driver, recurses
209
- // without bound). Escalate to a checkpoint resume / chain read instead and leave the replay to the
210
- // recovery path that can actually close the low range.
211
- if (reply.available.fromRevision > from) {
212
- deps.onUnderflow?.({ from, to }, reply.available);
213
- return reply;
214
- }
215
- for (const entry of reply.entries) {
216
- await deps.subscriber.onNotification(entry);
217
- }
218
- return reply;
219
- };
220
- }
1
+ /**
2
+ * Reactivity — Backfill RPC (`docs/reactivity.md` §Backfill RPC, §Wire formats).
3
+ *
4
+ * A subscriber that detects a revision gap (slow-subscriber drop, brief sleep) requests the missing
5
+ * range from its serving cohort; the cohort answers from its {@link ReplayBuffer} ring. The reply
6
+ * carries the **intersection** of the requested range with the buffer plus the `available` window the
7
+ * cohort actually holds, so a subscriber whose lag fell past the ring's low edge learns it must fall
8
+ * back further (to a checkpoint resume or a chain read) rather than silently receiving a short answer.
9
+ *
10
+ * Entries retain the original threshold signature (the buffer stores full {@link NotificationV1}s), so a
11
+ * backfill reply is verifiable **end-to-end** — the subscriber re-runs the same {@link NotificationVerifier}
12
+ * over each backfilled entry it runs over a live notification. Any cohort member can serve, because the
13
+ * replay buffer is gossiped across the cohort (origination ticket).
14
+ *
15
+ * Wire conventions match the rest of reactivity: JSON, byte fields base64url, unix-ms timestamps, `v: 1`,
16
+ * per-message structural validation on decode. `BackfillV1` is signed by the subscriber's peer key
17
+ * (`signature`); replay protection rides the request envelope as elsewhere in the substrate.
18
+ */
19
+
20
+ import {
21
+ decodeCohortMessage,
22
+ encodeCohortMessage,
23
+ DEFAULT_MAX_MESSAGE_BYTES,
24
+ } from "../cohort-topic/wire/codec.js";
25
+ import {
26
+ asObject,
27
+ b64urlField,
28
+ failWire,
29
+ reqFiniteNumber,
30
+ reqIntInRange,
31
+ reqString,
32
+ requireV1,
33
+ } from "./wire-validate.js";
34
+ import type { ReplayBuffer } from "./replay-buffer.js";
35
+ import type { ReactivitySubscriber } from "./subscriber.js";
36
+ import { validateNotificationArray, validateNotificationV1, type NotificationV1 } from "./wire.js";
37
+
38
+ /** A subscriber's request to replay a contiguous revision range from a cohort's buffer. */
39
+ export interface BackfillV1 {
40
+ v: 1;
41
+ /** Collection id, base64url. */
42
+ collectionId: string;
43
+ /** Inclusive low edge of the requested range. */
44
+ fromRevision: number;
45
+ /** Inclusive high edge of the requested range. */
46
+ toRevision: number;
47
+ /**
48
+ * Unix ms — bound into {@link backfillSigningPayload} so the request's freshness is authenticated.
49
+ * The serving handler's replay guard keys on the signature + this timestamp, so a captured request
50
+ * cannot be replayed with a forged-fresh timestamp (the forged value invalidates the signature).
51
+ */
52
+ timestamp: number;
53
+ /** Subscriber peer-key signature over the request. */
54
+ signature: string;
55
+ }
56
+
57
+ /** The cohort's reply: the held intersection of the request, plus the window it actually has. */
58
+ export interface BackfillReplyV1 {
59
+ v: 1;
60
+ /** The signed notifications in `[from, to] ∩ buffer`, ascending by revision. */
61
+ entries: NotificationV1[];
62
+ /** The revision window the serving cohort actually holds (so the subscriber can fall back further). */
63
+ available: {
64
+ fromRevision: number;
65
+ toRevision: number;
66
+ };
67
+ }
68
+
69
+ /** Narrow an already-parsed value to {@link BackfillV1}, throwing on any defect. */
70
+ export function validateBackfillV1(value: unknown): BackfillV1 {
71
+ const what = "BackfillV1";
72
+ const obj = asObject(value, what);
73
+ requireV1(obj, what);
74
+ const fromRevision = reqIntInRange(obj, "fromRevision", what, 0);
75
+ const toRevision = reqIntInRange(obj, "toRevision", what, fromRevision);
76
+ return {
77
+ v: 1,
78
+ collectionId: b64urlField(reqString(obj, "collectionId", what), "collectionId", what),
79
+ fromRevision,
80
+ toRevision,
81
+ timestamp: reqFiniteNumber(obj, "timestamp", what),
82
+ signature: b64urlField(reqString(obj, "signature", what), "signature", what),
83
+ };
84
+ }
85
+
86
+ const utf8 = new TextEncoder();
87
+
88
+ /** A {@link BackfillV1} minus its `signature` — the canonical bytes the subscriber peer-key-signs. */
89
+ export type BackfillSignable = Omit<BackfillV1, "signature">;
90
+
91
+ /**
92
+ * Canonical signed byte image of a {@link BackfillV1} (every field except `signature`). Mirrors the
93
+ * cohort-topic `registerSigningPayload` pattern: determinism comes from encoding an explicitly-ordered,
94
+ * type-tagged JSON array (array order is stable; object key order is not) as UTF-8. The leading
95
+ * `"BackfillV1"` tag means a backfill image can never collide with the `"ResumeV1"` image even when the
96
+ * shared fields are identical. Signer and verifier must agree byte-for-byte — the array order is the
97
+ * contract (pinned by the determinism test).
98
+ */
99
+ export function backfillSigningPayload(body: BackfillSignable): Uint8Array {
100
+ return utf8.encode(JSON.stringify([
101
+ "BackfillV1",
102
+ body.v,
103
+ body.collectionId,
104
+ body.fromRevision,
105
+ body.toRevision,
106
+ body.timestamp,
107
+ ]));
108
+ }
109
+
110
+ /** Narrow an already-parsed value to {@link BackfillReplyV1}, throwing on any defect. */
111
+ export function validateBackfillReplyV1(value: unknown): BackfillReplyV1 {
112
+ const what = "BackfillReplyV1";
113
+ const obj = asObject(value, what);
114
+ requireV1(obj, what);
115
+ const available = asObject(obj["available"], `${what}.available`);
116
+ const availFrom = reqIntInRange(available, "fromRevision", `${what}.available`, 0);
117
+ return {
118
+ v: 1,
119
+ entries: validateNotificationArray(obj["entries"], what),
120
+ available: {
121
+ fromRevision: availFrom,
122
+ toRevision: reqIntInRange(available, "toRevision", `${what}.available`, availFrom),
123
+ },
124
+ };
125
+ }
126
+
127
+ /** Encode a {@link BackfillV1} as a length-prefixed UTF-8 JSON frame. */
128
+ export function encodeBackfillV1(msg: BackfillV1, maxMessageBytes: number = DEFAULT_MAX_MESSAGE_BYTES): Uint8Array {
129
+ return encodeCohortMessage(validateBackfillV1(msg), maxMessageBytes);
130
+ }
131
+
132
+ /** Decode a length-prefixed {@link BackfillV1} frame. */
133
+ export function decodeBackfillV1(bytes: Uint8Array, maxMessageBytes?: number): BackfillV1 {
134
+ return validateBackfillV1(decodeCohortMessage(bytes, maxMessageBytes));
135
+ }
136
+
137
+ /** Encode a {@link BackfillReplyV1} as a length-prefixed UTF-8 JSON frame. */
138
+ export function encodeBackfillReplyV1(msg: BackfillReplyV1, maxMessageBytes: number = DEFAULT_MAX_MESSAGE_BYTES): Uint8Array {
139
+ return encodeCohortMessage(validateBackfillReplyV1(msg), maxMessageBytes);
140
+ }
141
+
142
+ /** Decode a length-prefixed {@link BackfillReplyV1} frame. */
143
+ export function decodeBackfillReplyV1(bytes: Uint8Array, maxMessageBytes?: number): BackfillReplyV1 {
144
+ return validateBackfillReplyV1(decodeCohortMessage(bytes, maxMessageBytes));
145
+ }
146
+
147
+ /**
148
+ * Serve a {@link BackfillV1} from a cohort's replay buffer: return the intersection of the requested
149
+ * range with the buffer and report the buffer's actual `available` window. A request whose `collectionId`
150
+ * does not match `expectedCollectionId` is rejected (a backfill is collection-scoped). When the buffer is
151
+ * empty, `available` collapses to the request's `fromRevision` (an empty window) and no entries are returned.
152
+ */
153
+ export function serveBackfill(buffer: ReplayBuffer, req: BackfillV1, expectedCollectionId: string): BackfillReplyV1 {
154
+ if (req.collectionId !== expectedCollectionId) {
155
+ failWire(`serveBackfill: request collectionId does not match this cohort's collection`);
156
+ }
157
+ const entries = buffer.range(req.fromRevision, req.toRevision).map((e) => validateNotificationV1(e.payload));
158
+ const low = buffer.lowRevision;
159
+ const high = buffer.highRevision;
160
+ const available = low === undefined || high === undefined
161
+ ? { fromRevision: req.fromRevision, toRevision: req.fromRevision }
162
+ : { fromRevision: low, toRevision: high };
163
+ return { v: 1, entries, available };
164
+ }
165
+
166
+ // --- subscriber-side backfill requester (the `requestBackfill` seam ↔ this RPC) ---
167
+
168
+ /** Sends a signed {@link BackfillV1} to the serving cohort and awaits its {@link BackfillReplyV1}. */
169
+ export type BackfillTransport = (req: BackfillV1) => Promise<BackfillReplyV1>;
170
+
171
+ /** Construction inputs for {@link createBackfillRequester}. */
172
+ export interface BackfillRequesterDeps {
173
+ /** Collection this subscription tracks, base64url. */
174
+ readonly collectionId: string;
175
+ /** Sign the request over its unsigned image (the subscriber's peer key); returns the base64url sig. */
176
+ readonly sign: (req: Omit<BackfillV1, "signature">) => string;
177
+ /** Send the request and await the reply (the db-p2p reactivity application protocol supplies this). */
178
+ readonly transport: BackfillTransport;
179
+ /** Re-enter each backfilled entry through the delivery path (verify → contiguity → deliver → dedupe). */
180
+ readonly subscriber: ReactivitySubscriber;
181
+ /** Unix-ms clock stamped into the signed `timestamp` (injected — db-core proper never reads an ambient clock). */
182
+ readonly clock: () => number;
183
+ /**
184
+ * Called when the cohort's `available` window does not reach the gap's low edge (`available.fromRevision
185
+ * > from`): the buffer no longer covers the gap, so the subscriber must escalate to a checkpoint resume
186
+ * or a chain read (`docs/reactivity.md` §Backfill RPC, "fall back further").
187
+ */
188
+ readonly onUnderflow?: (requested: { from: number; to: number }, available: { fromRevision: number; toRevision: number }) => void;
189
+ }
190
+
191
+ /**
192
+ * Build the `requestBackfill(from, to)` driver that connects the subscriber delivery path's backfill seam
193
+ * ([reactivity-origination-replay-delivery]) to the {@link BackfillV1} RPC. It signs and sends the request,
194
+ * replays the returned entries through {@link ReactivitySubscriber.onNotification} (so they close the gap,
195
+ * verified and deduped), and reports an underflow when the cohort's `available` window fell past the gap's
196
+ * low edge so the caller can fall back to a checkpoint resume or a chain read.
197
+ *
198
+ * Returns a `Promise`; the synchronous `(from, to) => void` seam wraps a call to this with `void`.
199
+ */
200
+ export function createBackfillRequester(deps: BackfillRequesterDeps): (from: number, to: number) => Promise<BackfillReplyV1> {
201
+ return async (from: number, to: number): Promise<BackfillReplyV1> => {
202
+ const unsigned: Omit<BackfillV1, "signature"> = { v: 1, collectionId: deps.collectionId, fromRevision: from, toRevision: to, timestamp: deps.clock() };
203
+ const req: BackfillV1 = { ...unsigned, signature: deps.sign(unsigned) };
204
+ const reply = await deps.transport(req);
205
+ // Underflow: the cohort's held window does not reach the gap's low edge (`docs/reactivity.md`
206
+ // §Backfill RPC, "fall back further"). The returned entries all sit *above* the still-unfillable
207
+ // low range, so none can apply contiguously from `from`. Replaying them would only re-fire the
208
+ // subscriber's gap → `requestBackfill` seam (which, when wired back to this driver, recurses
209
+ // without bound). Escalate to a checkpoint resume / chain read instead and leave the replay to the
210
+ // recovery path that can actually close the low range.
211
+ if (reply.available.fromRevision > from) {
212
+ deps.onUnderflow?.({ from, to }, reply.available);
213
+ return reply;
214
+ }
215
+ for (const entry of reply.entries) {
216
+ await deps.subscriber.onNotification(entry);
217
+ }
218
+ return reply;
219
+ };
220
+ }