@optimystic/db-p2p 0.22.0 → 0.24.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (194) hide show
  1. package/dist/src/cluster/client.d.ts +10 -0
  2. package/dist/src/cluster/client.d.ts.map +1 -1
  3. package/dist/src/cluster/client.js +30 -1
  4. package/dist/src/cluster/client.js.map +1 -1
  5. package/dist/src/cluster/cluster-repo.d.ts +39 -14
  6. package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
  7. package/dist/src/cluster/cluster-repo.js +226 -119
  8. package/dist/src/cluster/cluster-repo.js.map +1 -1
  9. package/dist/src/cluster/service.d.ts +32 -1
  10. package/dist/src/cluster/service.d.ts.map +1 -1
  11. package/dist/src/cluster/service.js +43 -2
  12. package/dist/src/cluster/service.js.map +1 -1
  13. package/dist/src/cohort-topic/host.js +34 -11
  14. package/dist/src/cohort-topic/host.js.map +1 -1
  15. package/dist/src/cohort-topic/stream-util.d.ts +37 -7
  16. package/dist/src/cohort-topic/stream-util.d.ts.map +1 -1
  17. package/dist/src/cohort-topic/stream-util.js +77 -19
  18. package/dist/src/cohort-topic/stream-util.js.map +1 -1
  19. package/dist/src/dispute/dispute-service.d.ts.map +1 -1
  20. package/dist/src/dispute/dispute-service.js +9 -3
  21. package/dist/src/dispute/dispute-service.js.map +1 -1
  22. package/dist/src/index.d.ts +3 -0
  23. package/dist/src/index.d.ts.map +1 -1
  24. package/dist/src/index.js +3 -0
  25. package/dist/src/index.js.map +1 -1
  26. package/dist/src/libp2p-key-network.d.ts +88 -2
  27. package/dist/src/libp2p-key-network.d.ts.map +1 -1
  28. package/dist/src/libp2p-key-network.js +134 -28
  29. package/dist/src/libp2p-key-network.js.map +1 -1
  30. package/dist/src/libp2p-node-base.d.ts.map +1 -1
  31. package/dist/src/libp2p-node-base.js +25 -1
  32. package/dist/src/libp2p-node-base.js.map +1 -1
  33. package/dist/src/logger.d.ts +17 -1
  34. package/dist/src/logger.d.ts.map +1 -1
  35. package/dist/src/logger.js +19 -2
  36. package/dist/src/logger.js.map +1 -1
  37. package/dist/src/matchmaking/query-transport.js +3 -3
  38. package/dist/src/matchmaking/query-transport.js.map +1 -1
  39. package/dist/src/owned-block-seed.d.ts +6 -3
  40. package/dist/src/owned-block-seed.d.ts.map +1 -1
  41. package/dist/src/owned-block-seed.js +16 -3
  42. package/dist/src/owned-block-seed.js.map +1 -1
  43. package/dist/src/peer-address-book.d.ts +72 -0
  44. package/dist/src/peer-address-book.d.ts.map +1 -0
  45. package/dist/src/peer-address-book.js +123 -0
  46. package/dist/src/peer-address-book.js.map +1 -0
  47. package/dist/src/reactivity/notify-transport.d.ts +4 -4
  48. package/dist/src/reactivity/notify-transport.js +6 -6
  49. package/dist/src/reactivity/notify-transport.js.map +1 -1
  50. package/dist/src/reactivity/push-state-gossip.js +2 -2
  51. package/dist/src/reactivity/push-state-gossip.js.map +1 -1
  52. package/dist/src/reactivity/recover-transport.d.ts +6 -2
  53. package/dist/src/reactivity/recover-transport.d.ts.map +1 -1
  54. package/dist/src/reactivity/recover-transport.js +7 -3
  55. package/dist/src/reactivity/recover-transport.js.map +1 -1
  56. package/dist/src/repo/client.d.ts.map +1 -1
  57. package/dist/src/repo/client.js +11 -2
  58. package/dist/src/repo/client.js.map +1 -1
  59. package/dist/src/repo/cluster-coordinator.d.ts +30 -0
  60. package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
  61. package/dist/src/repo/cluster-coordinator.js +95 -3
  62. package/dist/src/repo/cluster-coordinator.js.map +1 -1
  63. package/dist/src/repo/coordinator-repo.d.ts +62 -9
  64. package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
  65. package/dist/src/repo/coordinator-repo.js +242 -73
  66. package/dist/src/repo/coordinator-repo.js.map +1 -1
  67. package/dist/src/rn.d.ts +3 -0
  68. package/dist/src/rn.d.ts.map +1 -1
  69. package/dist/src/rn.js +3 -0
  70. package/dist/src/rn.js.map +1 -1
  71. package/dist/src/storage/cached-raw-storage.d.ts +83 -0
  72. package/dist/src/storage/cached-raw-storage.d.ts.map +1 -0
  73. package/dist/src/storage/cached-raw-storage.js +152 -0
  74. package/dist/src/storage/cached-raw-storage.js.map +1 -0
  75. package/dist/src/storage/cached-store-driver.d.ts +186 -0
  76. package/dist/src/storage/cached-store-driver.d.ts.map +1 -0
  77. package/dist/src/storage/cached-store-driver.js +775 -0
  78. package/dist/src/storage/cached-store-driver.js.map +1 -0
  79. package/dist/src/storage/i-raw-storage.d.ts +12 -5
  80. package/dist/src/storage/i-raw-storage.d.ts.map +1 -1
  81. package/dist/src/storage/shared-cache-pool.d.ts +234 -0
  82. package/dist/src/storage/shared-cache-pool.d.ts.map +1 -0
  83. package/dist/src/storage/shared-cache-pool.js +354 -0
  84. package/dist/src/storage/shared-cache-pool.js.map +1 -0
  85. package/dist/src/testing/cohort-topic-mesh-harness.d.ts +13 -6
  86. package/dist/src/testing/cohort-topic-mesh-harness.d.ts.map +1 -1
  87. package/dist/src/testing/cohort-topic-mesh-harness.js +15 -6
  88. package/dist/src/testing/cohort-topic-mesh-harness.js.map +1 -1
  89. package/dist/src/testing/raw-storage-conformance.d.ts +2 -1
  90. package/dist/src/testing/raw-storage-conformance.d.ts.map +1 -1
  91. package/dist/src/testing/raw-storage-conformance.js +35 -2
  92. package/dist/src/testing/raw-storage-conformance.js.map +1 -1
  93. package/package.json +3 -3
  94. package/readme.md +668 -668
  95. package/src/cluster/block-transfer.ts +424 -424
  96. package/src/cluster/client.ts +119 -88
  97. package/src/cluster/cluster-error.ts +64 -64
  98. package/src/cluster/cluster-policy.ts +203 -203
  99. package/src/cluster/cluster-repo.ts +242 -122
  100. package/src/cluster/cluster-size-coupling.ts +45 -45
  101. package/src/cluster/commit-cert.ts +139 -139
  102. package/src/cluster/i-transaction-state-store.ts +43 -43
  103. package/src/cluster/memory-transaction-state-store.ts +56 -56
  104. package/src/cluster/peer-key-binding.ts +37 -37
  105. package/src/cluster/persistent-transaction-state-store.ts +92 -92
  106. package/src/cluster/quorum-restore.ts +223 -223
  107. package/src/cluster/reconcile-block.ts +203 -203
  108. package/src/cluster/service.ts +293 -241
  109. package/src/cluster/supermajority-coupling.ts +37 -37
  110. package/src/cohort-topic/bootstrap-evidence-builder.ts +122 -122
  111. package/src/cohort-topic/bootstrap-evidence-verifiers.ts +132 -132
  112. package/src/cohort-topic/bootstrap-parent-reference.ts +159 -159
  113. package/src/cohort-topic/change-bridge.ts +109 -109
  114. package/src/cohort-topic/cohort-gossip-driver.ts +231 -231
  115. package/src/cohort-topic/cohort-gossip-transport.ts +84 -84
  116. package/src/cohort-topic/fret-trust-anchor.ts +153 -153
  117. package/src/cohort-topic/host.ts +42 -11
  118. package/src/cohort-topic/index.ts +13 -13
  119. package/src/cohort-topic/membership-publish-sink.ts +20 -20
  120. package/src/cohort-topic/membership-source.ts +68 -68
  121. package/src/cohort-topic/peer-codec.ts +31 -31
  122. package/src/cohort-topic/peer-sig.ts +86 -86
  123. package/src/cohort-topic/protocols.ts +71 -71
  124. package/src/cohort-topic/reactivity-membership-gate.ts +77 -77
  125. package/src/cohort-topic/size-estimator.ts +16 -16
  126. package/src/cohort-topic/stream-util.ts +79 -19
  127. package/src/cohort-topic/threshold-crypto.ts +239 -239
  128. package/src/cohort-topic/topic-router.ts +77 -77
  129. package/src/dispute/arbitrator-selection.ts +138 -138
  130. package/src/dispute/cascade.ts +524 -524
  131. package/src/dispute/dispute-service.ts +11 -5
  132. package/src/dispute/invalidation.ts +625 -625
  133. package/src/inbound-authorization.ts +190 -190
  134. package/src/index.ts +52 -49
  135. package/src/libp2p-key-network.ts +1120 -990
  136. package/src/libp2p-node-base.ts +1675 -1651
  137. package/src/libp2p-node-rn.ts +30 -30
  138. package/src/libp2p-node.ts +36 -36
  139. package/src/logger.ts +19 -2
  140. package/src/matchmaking/aggregate-counts.ts +104 -104
  141. package/src/matchmaking/index.ts +20 -20
  142. package/src/matchmaking/module.ts +363 -363
  143. package/src/matchmaking/protocols.ts +51 -51
  144. package/src/matchmaking/provider-manager.ts +95 -95
  145. package/src/matchmaking/query-handler.ts +88 -88
  146. package/src/matchmaking/query-transport.ts +3 -3
  147. package/src/matchmaking/seeker-manager.ts +64 -64
  148. package/src/matchmaking/seeker-walk-client.ts +293 -293
  149. package/src/matchmaking/traffic-validation.ts +195 -195
  150. package/src/optimystic-node.ts +36 -36
  151. package/src/owned-block-seed.ts +53 -40
  152. package/src/peer-address-book.ts +149 -0
  153. package/src/protocol-limits.ts +33 -33
  154. package/src/reactivity/forwarder-host.ts +438 -438
  155. package/src/reactivity/index.ts +19 -19
  156. package/src/reactivity/notify-transport.ts +144 -144
  157. package/src/reactivity/origination-manager.ts +192 -192
  158. package/src/reactivity/protocols.ts +61 -61
  159. package/src/reactivity/push-state-gossip.ts +291 -291
  160. package/src/reactivity/recover-transport.ts +7 -3
  161. package/src/reactivity/rotation-rereg-scheduler.ts +256 -256
  162. package/src/reactivity/subscriber-registry.ts +96 -96
  163. package/src/reactivity/subscription-manager.ts +450 -450
  164. package/src/reactivity/topic-bytes.ts +37 -37
  165. package/src/repo/client.ts +12 -2
  166. package/src/repo/cluster-coordinator.ts +99 -3
  167. package/src/repo/coordinator-repo.ts +281 -74
  168. package/src/repo/types.ts +7 -7
  169. package/src/rn.ts +39 -36
  170. package/src/rpc-deadline.ts +45 -45
  171. package/src/storage/arachnode-partition.ts +74 -74
  172. package/src/storage/cached-raw-storage.ts +180 -0
  173. package/src/storage/cached-store-driver.ts +859 -0
  174. package/src/storage/i-kv-store.ts +8 -8
  175. package/src/storage/i-raw-storage.ts +12 -5
  176. package/src/storage/kv-raw-storage.ts +135 -135
  177. package/src/storage/memory-kv-store.ts +28 -28
  178. package/src/storage/memory-storage.ts +25 -25
  179. package/src/storage/memory-store-driver.ts +157 -157
  180. package/src/storage/raw-store-codec.ts +42 -42
  181. package/src/storage/raw-store-driver.ts +80 -80
  182. package/src/storage/ring-selector.ts +317 -317
  183. package/src/storage/ring-shift-coordinator.ts +271 -271
  184. package/src/storage/shared-cache-pool.ts +452 -0
  185. package/src/storage/storage-repo.ts +1014 -1014
  186. package/src/testing/cohort-topic-mesh-harness.ts +673 -663
  187. package/src/testing/index.ts +8 -8
  188. package/src/testing/matchmaking-mesh-harness.ts +475 -475
  189. package/src/testing/raw-storage-conformance.ts +453 -417
  190. package/src/testing/reactivity-mesh-harness.ts +922 -922
  191. package/dist/src/storage/restoration-coordinator-v2.d.ts +0 -67
  192. package/dist/src/storage/restoration-coordinator-v2.d.ts.map +0 -1
  193. package/dist/src/storage/restoration-coordinator-v2.js +0 -172
  194. package/dist/src/storage/restoration-coordinator-v2.js.map +0 -1
@@ -1,231 +1,231 @@
1
- /**
2
- * Cohort-topic gossip-cadence driver helpers (host-side, gap 5).
3
- *
4
- * The {@link import("./host.js").CohortTopicHost} owns a single repeating timer; on each tick it drives
5
- * every live `CoordEngine`'s gossip round. The per-round work this module factors out — assembling a
6
- * `CohortGossipV1` and accumulating/draining the registration-record deltas the renewal `touch`/`evicted`
7
- * hooks emit — is split here so it is unit-testable without the timer or libp2p.
8
- *
9
- * db-p2p (not db-core) owns the cadence because the round interval, the peer-key signature on the gossip
10
- * envelope, and the per-coord inbound routing all live at the FRET/libp2p boundary; db-core supplies the
11
- * pure substrate (`selfWillingnessBits`, `toCohortTopicSummary`, `toGossipRecord`, the bus merge).
12
- */
13
-
14
- import {
15
- bytesToB64url,
16
- toGossipRecord,
17
- willingnessBitsHex,
18
- selfWillingnessBits,
19
- type ChildLinkRefV1,
20
- type CohortGossipV1,
21
- type CohortTopicSummary,
22
- type GossipRecordRefV1,
23
- type GossipRecordV1,
24
- type LoadBarometerState,
25
- type NodeProfile,
26
- type RegistrationRecord,
27
- } from "@optimystic/db-core";
28
-
29
- /**
30
- * Default gossip-round cadence in ms. There is no dedicated `gossip_round` constant in
31
- * `docs/cohort-topic.md` §Configuration, so this is a derived, injectable default on the order of one
32
- * round — a few seconds, aligned with the traffic window / ping cadence (`ping_interval = ttl/3` is
33
- * 30 s Core, so a sub-round of that). The membership refresh (`T_membership_refresh`, 5 min) and the
34
- * demotion hysteresis (`T_demote`, 5 min) are gated by elapsed-time **inside** their modules, so the
35
- * driver can tick fast and let those modules decide when to act.
36
- */
37
- export const DEFAULT_GOSSIP_INTERVAL_MS = 5_000;
38
-
39
- /**
40
- * Default `T_willingness_heartbeat` (ms): how often a genuinely-**idle** but **willing** engine re-broadcasts
41
- * a willingness-only heartbeat so a cold cohort can bootstrap (siblings hear it, instantiate, and reciprocate
42
- * their own willingness) without waiting on a first registration that can never be admitted while the view is
43
- * empty. See `docs/cohort-topic.md` §Cold-start instantiation / §Configuration.
44
- *
45
- * On the order of the ping interval (~30 s, `ttl/3` Core) — a few gossip rounds at the 5 s cadence. A
46
- * record-carrying (non-idle) round already ships willingness every round and resets this clock, so the
47
- * throttle governs only engines with nothing else to say. The very first idle round after an engine is
48
- * created emits immediately (no wait), so bootstrap converges in ~2 rounds; the throttle only paces the
49
- * steady-state re-broadcast of an idle willing cohort.
50
- */
51
- export const DEFAULT_WILLINGNESS_HEARTBEAT_MS = 30_000;
52
-
53
- /**
54
- * Per-`CoordEngine` queue of registration-record deltas accumulated between gossip rounds. The renewal
55
- * cohort side calls {@link PendingDeltas.touch} on every served ping/re-attach and
56
- * {@link PendingDeltas.evicted} on every TTL sweep; the next round {@link PendingDeltas.drain}s the
57
- * batch into the gossip frame and clears it — one broadcast per round rather than one per ping.
58
- */
59
- export interface PendingDeltas {
60
- /** Upsert a fresh/touched record (keyed by `(topicId, participantId)`; last write wins on `lastPing`). */
61
- touch(rec: RegistrationRecord): void;
62
- /** Queue an eviction ref and drop any pending record for the same key (a stale record can't also re-advertise). */
63
- evicted(rec: RegistrationRecord): void;
64
- /**
65
- * Queue a child-cohort **link** for replication (keyed by `(topicId, childCohortCoord)`; last write wins on
66
- * `effectiveAt`). Enqueued by the parent engine only when the local child registry actually changed, so a
67
- * stale/no-op record is not re-gossiped. A link and a later unlink for the same child in one round collapse
68
- * to whichever carries the newer `effectiveAt`.
69
- */
70
- childLink(topicId: Uint8Array, childCohortCoord: Uint8Array, effectiveAt: number): void;
71
- /** Queue a child-cohort **unlink** (a released/demoted child) for replication; same key + last-writer-wins as {@link childLink}. */
72
- childUnlink(topicId: Uint8Array, childCohortCoord: Uint8Array, effectiveAt: number): void;
73
- /** True iff nothing is queued. */
74
- isEmpty(): boolean;
75
- /** Drain the queue into wire-shaped deltas, clearing it. */
76
- drain(): { records: GossipRecordV1[]; evicted: GossipRecordRefV1[]; childLinks: ChildLinkRefV1[]; childUnlinks: ChildLinkRefV1[] };
77
- }
78
-
79
- /** A queued child link/unlink: the wire ref plus whether it is a link (`true`) or an unlink (`false`). */
80
- interface PendingChildDelta {
81
- ref: ChildLinkRefV1;
82
- linked: boolean;
83
- }
84
-
85
- /** Build an empty {@link PendingDeltas} queue. */
86
- export function createPendingDeltas(): PendingDeltas {
87
- const records = new Map<string, RegistrationRecord>();
88
- const evicted = new Map<string, GossipRecordRefV1>();
89
- const childDeltas = new Map<string, PendingChildDelta>();
90
- const keyOf = (topicId: Uint8Array, participantId: Uint8Array): string =>
91
- `${bytesToB64url(topicId)}|${bytesToB64url(participantId)}`;
92
- // A child delta and its later opposite (link→unlink) share one key so the round drains only the newest.
93
- const childKeyOf = (topicId: Uint8Array, childCohortCoord: Uint8Array): string =>
94
- `${bytesToB64url(topicId)}|${bytesToB64url(childCohortCoord)}`;
95
- const queueChild = (topicId: Uint8Array, childCohortCoord: Uint8Array, effectiveAt: number, linked: boolean): void => {
96
- const key = childKeyOf(topicId, childCohortCoord);
97
- const held = childDeltas.get(key);
98
- // Last-writer-wins on effectiveAt: never let an older link/unlink shadow a newer one queued the same round.
99
- if (held !== undefined && effectiveAt < held.ref.effectiveAt) {
100
- return;
101
- }
102
- childDeltas.set(key, { ref: { topicId: bytesToB64url(topicId), childCohortCoord: bytesToB64url(childCohortCoord), effectiveAt }, linked });
103
- };
104
- return {
105
- touch(rec: RegistrationRecord): void {
106
- const key = keyOf(rec.topicId, rec.participantId);
107
- const held = records.get(key);
108
- // Last-writer-wins on lastPing: never let an older touch shadow a newer one queued the same round.
109
- if (held === undefined || rec.lastPing >= held.lastPing) {
110
- records.set(key, rec);
111
- }
112
- evicted.delete(key); // a live touch supersedes a pending eviction (eviction-vs-late-touch)
113
- },
114
- evicted(rec: RegistrationRecord): void {
115
- const key = keyOf(rec.topicId, rec.participantId);
116
- // Stamp the evicted record's lastPing so the receiver can gate the delete on freshness (a stale
117
- // eviction must not delete a record the participant has since re-registered — see bus.mergeRecords).
118
- evicted.set(key, { topicId: bytesToB64url(rec.topicId), participantId: bytesToB64url(rec.participantId), lastPing: rec.lastPing });
119
- records.delete(key);
120
- },
121
- childLink(topicId: Uint8Array, childCohortCoord: Uint8Array, effectiveAt: number): void {
122
- queueChild(topicId, childCohortCoord, effectiveAt, true);
123
- },
124
- childUnlink(topicId: Uint8Array, childCohortCoord: Uint8Array, effectiveAt: number): void {
125
- queueChild(topicId, childCohortCoord, effectiveAt, false);
126
- },
127
- isEmpty(): boolean {
128
- return records.size === 0 && evicted.size === 0 && childDeltas.size === 0;
129
- },
130
- drain(): { records: GossipRecordV1[]; evicted: GossipRecordRefV1[]; childLinks: ChildLinkRefV1[]; childUnlinks: ChildLinkRefV1[] } {
131
- const childLinks: ChildLinkRefV1[] = [];
132
- const childUnlinks: ChildLinkRefV1[] = [];
133
- for (const delta of childDeltas.values()) {
134
- (delta.linked ? childLinks : childUnlinks).push(delta.ref);
135
- }
136
- const out = {
137
- records: [...records.values()].map(toGossipRecord),
138
- evicted: [...evicted.values()],
139
- childLinks,
140
- childUnlinks,
141
- };
142
- records.clear();
143
- evicted.clear();
144
- childDeltas.clear();
145
- return out;
146
- },
147
- };
148
- }
149
-
150
- /** Inputs to {@link buildCohortGossip} — the per-round signals already gathered by the engine. */
151
- export interface GossipFrameInputs {
152
- /** This member's dialable id, base64url (the `fromMember` / view key). */
153
- readonly fromMember: string;
154
- /** This cohort's served coord, base64url (the inbound routing key). */
155
- readonly coord: string;
156
- /** This cohort's epoch, base64url. */
157
- readonly cohortEpoch: string;
158
- /** This cohort's tree tier `d` — carried on the frame so a cold sibling instantiates at the right tier. */
159
- readonly treeTier: number;
160
- /**
161
- * True to emit a **willingness-only heartbeat** even when this engine is idle (no resident topics, no
162
- * pending deltas) — provided the node is actually willing for some tier. Lets an idle-but-willing engine
163
- * still tell siblings it will serve, so a cold cohort can bootstrap (§Cold-start instantiation). The caller
164
- * (the gossip round) sets this from the per-engine heartbeat clock; a non-idle round ignores it (it already
165
- * ships willingness).
166
- */
167
- readonly heartbeat: boolean;
168
- /** This node's tier profile (the `∧` half of the willingness vector). */
169
- readonly profile: NodeProfile;
170
- /** This member's load barometer (load buckets + the load-shed half of willingness). */
171
- readonly barometer: LoadBarometerState;
172
- /** Cohort-wide observation window (seconds) for the rate fields in `topicSummaries`. */
173
- readonly windowSeconds: number;
174
- /** Per-resident-topic summaries (own published counts), already frozen for this round. */
175
- readonly topicSummaries: CohortTopicSummary[];
176
- /** Drained fresh/touched record deltas for this round. */
177
- readonly records: GossipRecordV1[];
178
- /** Drained eviction refs for this round. */
179
- readonly evicted: GossipRecordRefV1[];
180
- /** Drained child-cohort link refs for this round (cross-member child-set convergence). */
181
- readonly childLinks: ChildLinkRefV1[];
182
- /** Drained child-cohort unlink refs for this round (a released/demoted child). */
183
- readonly childUnlinks: ChildLinkRefV1[];
184
- /** Round timestamp, unix ms. */
185
- readonly timestamp: number;
186
- }
187
-
188
- /**
189
- * Assemble one round's {@link CohortGossipV1}, or `undefined` when this engine has nothing to say. An engine
190
- * is **idle** when it holds no resident topics and no pending deltas. An idle engine normally builds no frame
191
- * (idle empty engines cost no gossip) — **except** on a willingness heartbeat ({@link GossipFrameInputs.heartbeat}),
192
- * where an idle engine that is willing for at least one tier (`selfWillingnessBits !== 0`) still emits a
193
- * willingness/load-only frame (empty `topicSummaries`, no `records`/`evicted`) so siblings can hear it and a
194
- * cold cohort can bootstrap (§Cold-start instantiation). An idle-and-unwilling engine stays silent even on a
195
- * heartbeat (nothing to bootstrap). `willingnessBits` is `profile ∧ load` ({@link selfWillingnessBits}); the
196
- * `signature` slot is left empty for the host's peer-key signer to fill before broadcast.
197
- */
198
- export function buildCohortGossip(i: GossipFrameInputs): CohortGossipV1 | undefined {
199
- const willingness = selfWillingnessBits(i.profile, i.barometer);
200
- const idle = i.topicSummaries.length === 0 && i.records.length === 0 && i.evicted.length === 0
201
- && i.childLinks.length === 0 && i.childUnlinks.length === 0;
202
- if (idle && !(i.heartbeat && willingness !== 0)) {
203
- return undefined;
204
- }
205
- const g: CohortGossipV1 = {
206
- v: 1,
207
- fromMember: i.fromMember,
208
- coord: i.coord,
209
- cohortEpoch: i.cohortEpoch,
210
- treeTier: i.treeTier,
211
- willingnessBits: willingnessBitsHex(willingness),
212
- loadBuckets: i.barometer.loadBuckets(),
213
- windowSeconds: i.windowSeconds,
214
- topicSummaries: i.topicSummaries,
215
- timestamp: i.timestamp,
216
- signature: "",
217
- };
218
- if (i.records.length > 0) {
219
- g.records = i.records;
220
- }
221
- if (i.evicted.length > 0) {
222
- g.evicted = i.evicted;
223
- }
224
- if (i.childLinks.length > 0) {
225
- g.childLinks = i.childLinks;
226
- }
227
- if (i.childUnlinks.length > 0) {
228
- g.childUnlinks = i.childUnlinks;
229
- }
230
- return g;
231
- }
1
+ /**
2
+ * Cohort-topic gossip-cadence driver helpers (host-side, gap 5).
3
+ *
4
+ * The {@link import("./host.js").CohortTopicHost} owns a single repeating timer; on each tick it drives
5
+ * every live `CoordEngine`'s gossip round. The per-round work this module factors out — assembling a
6
+ * `CohortGossipV1` and accumulating/draining the registration-record deltas the renewal `touch`/`evicted`
7
+ * hooks emit — is split here so it is unit-testable without the timer or libp2p.
8
+ *
9
+ * db-p2p (not db-core) owns the cadence because the round interval, the peer-key signature on the gossip
10
+ * envelope, and the per-coord inbound routing all live at the FRET/libp2p boundary; db-core supplies the
11
+ * pure substrate (`selfWillingnessBits`, `toCohortTopicSummary`, `toGossipRecord`, the bus merge).
12
+ */
13
+
14
+ import {
15
+ bytesToB64url,
16
+ toGossipRecord,
17
+ willingnessBitsHex,
18
+ selfWillingnessBits,
19
+ type ChildLinkRefV1,
20
+ type CohortGossipV1,
21
+ type CohortTopicSummary,
22
+ type GossipRecordRefV1,
23
+ type GossipRecordV1,
24
+ type LoadBarometerState,
25
+ type NodeProfile,
26
+ type RegistrationRecord,
27
+ } from "@optimystic/db-core";
28
+
29
+ /**
30
+ * Default gossip-round cadence in ms. There is no dedicated `gossip_round` constant in
31
+ * `docs/cohort-topic.md` §Configuration, so this is a derived, injectable default on the order of one
32
+ * round — a few seconds, aligned with the traffic window / ping cadence (`ping_interval = ttl/3` is
33
+ * 30 s Core, so a sub-round of that). The membership refresh (`T_membership_refresh`, 5 min) and the
34
+ * demotion hysteresis (`T_demote`, 5 min) are gated by elapsed-time **inside** their modules, so the
35
+ * driver can tick fast and let those modules decide when to act.
36
+ */
37
+ export const DEFAULT_GOSSIP_INTERVAL_MS = 5_000;
38
+
39
+ /**
40
+ * Default `T_willingness_heartbeat` (ms): how often a genuinely-**idle** but **willing** engine re-broadcasts
41
+ * a willingness-only heartbeat so a cold cohort can bootstrap (siblings hear it, instantiate, and reciprocate
42
+ * their own willingness) without waiting on a first registration that can never be admitted while the view is
43
+ * empty. See `docs/cohort-topic.md` §Cold-start instantiation / §Configuration.
44
+ *
45
+ * On the order of the ping interval (~30 s, `ttl/3` Core) — a few gossip rounds at the 5 s cadence. A
46
+ * record-carrying (non-idle) round already ships willingness every round and resets this clock, so the
47
+ * throttle governs only engines with nothing else to say. The very first idle round after an engine is
48
+ * created emits immediately (no wait), so bootstrap converges in ~2 rounds; the throttle only paces the
49
+ * steady-state re-broadcast of an idle willing cohort.
50
+ */
51
+ export const DEFAULT_WILLINGNESS_HEARTBEAT_MS = 30_000;
52
+
53
+ /**
54
+ * Per-`CoordEngine` queue of registration-record deltas accumulated between gossip rounds. The renewal
55
+ * cohort side calls {@link PendingDeltas.touch} on every served ping/re-attach and
56
+ * {@link PendingDeltas.evicted} on every TTL sweep; the next round {@link PendingDeltas.drain}s the
57
+ * batch into the gossip frame and clears it — one broadcast per round rather than one per ping.
58
+ */
59
+ export interface PendingDeltas {
60
+ /** Upsert a fresh/touched record (keyed by `(topicId, participantId)`; last write wins on `lastPing`). */
61
+ touch(rec: RegistrationRecord): void;
62
+ /** Queue an eviction ref and drop any pending record for the same key (a stale record can't also re-advertise). */
63
+ evicted(rec: RegistrationRecord): void;
64
+ /**
65
+ * Queue a child-cohort **link** for replication (keyed by `(topicId, childCohortCoord)`; last write wins on
66
+ * `effectiveAt`). Enqueued by the parent engine only when the local child registry actually changed, so a
67
+ * stale/no-op record is not re-gossiped. A link and a later unlink for the same child in one round collapse
68
+ * to whichever carries the newer `effectiveAt`.
69
+ */
70
+ childLink(topicId: Uint8Array, childCohortCoord: Uint8Array, effectiveAt: number): void;
71
+ /** Queue a child-cohort **unlink** (a released/demoted child) for replication; same key + last-writer-wins as {@link childLink}. */
72
+ childUnlink(topicId: Uint8Array, childCohortCoord: Uint8Array, effectiveAt: number): void;
73
+ /** True iff nothing is queued. */
74
+ isEmpty(): boolean;
75
+ /** Drain the queue into wire-shaped deltas, clearing it. */
76
+ drain(): { records: GossipRecordV1[]; evicted: GossipRecordRefV1[]; childLinks: ChildLinkRefV1[]; childUnlinks: ChildLinkRefV1[] };
77
+ }
78
+
79
+ /** A queued child link/unlink: the wire ref plus whether it is a link (`true`) or an unlink (`false`). */
80
+ interface PendingChildDelta {
81
+ ref: ChildLinkRefV1;
82
+ linked: boolean;
83
+ }
84
+
85
+ /** Build an empty {@link PendingDeltas} queue. */
86
+ export function createPendingDeltas(): PendingDeltas {
87
+ const records = new Map<string, RegistrationRecord>();
88
+ const evicted = new Map<string, GossipRecordRefV1>();
89
+ const childDeltas = new Map<string, PendingChildDelta>();
90
+ const keyOf = (topicId: Uint8Array, participantId: Uint8Array): string =>
91
+ `${bytesToB64url(topicId)}|${bytesToB64url(participantId)}`;
92
+ // A child delta and its later opposite (link→unlink) share one key so the round drains only the newest.
93
+ const childKeyOf = (topicId: Uint8Array, childCohortCoord: Uint8Array): string =>
94
+ `${bytesToB64url(topicId)}|${bytesToB64url(childCohortCoord)}`;
95
+ const queueChild = (topicId: Uint8Array, childCohortCoord: Uint8Array, effectiveAt: number, linked: boolean): void => {
96
+ const key = childKeyOf(topicId, childCohortCoord);
97
+ const held = childDeltas.get(key);
98
+ // Last-writer-wins on effectiveAt: never let an older link/unlink shadow a newer one queued the same round.
99
+ if (held !== undefined && effectiveAt < held.ref.effectiveAt) {
100
+ return;
101
+ }
102
+ childDeltas.set(key, { ref: { topicId: bytesToB64url(topicId), childCohortCoord: bytesToB64url(childCohortCoord), effectiveAt }, linked });
103
+ };
104
+ return {
105
+ touch(rec: RegistrationRecord): void {
106
+ const key = keyOf(rec.topicId, rec.participantId);
107
+ const held = records.get(key);
108
+ // Last-writer-wins on lastPing: never let an older touch shadow a newer one queued the same round.
109
+ if (held === undefined || rec.lastPing >= held.lastPing) {
110
+ records.set(key, rec);
111
+ }
112
+ evicted.delete(key); // a live touch supersedes a pending eviction (eviction-vs-late-touch)
113
+ },
114
+ evicted(rec: RegistrationRecord): void {
115
+ const key = keyOf(rec.topicId, rec.participantId);
116
+ // Stamp the evicted record's lastPing so the receiver can gate the delete on freshness (a stale
117
+ // eviction must not delete a record the participant has since re-registered — see bus.mergeRecords).
118
+ evicted.set(key, { topicId: bytesToB64url(rec.topicId), participantId: bytesToB64url(rec.participantId), lastPing: rec.lastPing });
119
+ records.delete(key);
120
+ },
121
+ childLink(topicId: Uint8Array, childCohortCoord: Uint8Array, effectiveAt: number): void {
122
+ queueChild(topicId, childCohortCoord, effectiveAt, true);
123
+ },
124
+ childUnlink(topicId: Uint8Array, childCohortCoord: Uint8Array, effectiveAt: number): void {
125
+ queueChild(topicId, childCohortCoord, effectiveAt, false);
126
+ },
127
+ isEmpty(): boolean {
128
+ return records.size === 0 && evicted.size === 0 && childDeltas.size === 0;
129
+ },
130
+ drain(): { records: GossipRecordV1[]; evicted: GossipRecordRefV1[]; childLinks: ChildLinkRefV1[]; childUnlinks: ChildLinkRefV1[] } {
131
+ const childLinks: ChildLinkRefV1[] = [];
132
+ const childUnlinks: ChildLinkRefV1[] = [];
133
+ for (const delta of childDeltas.values()) {
134
+ (delta.linked ? childLinks : childUnlinks).push(delta.ref);
135
+ }
136
+ const out = {
137
+ records: [...records.values()].map(toGossipRecord),
138
+ evicted: [...evicted.values()],
139
+ childLinks,
140
+ childUnlinks,
141
+ };
142
+ records.clear();
143
+ evicted.clear();
144
+ childDeltas.clear();
145
+ return out;
146
+ },
147
+ };
148
+ }
149
+
150
+ /** Inputs to {@link buildCohortGossip} — the per-round signals already gathered by the engine. */
151
+ export interface GossipFrameInputs {
152
+ /** This member's dialable id, base64url (the `fromMember` / view key). */
153
+ readonly fromMember: string;
154
+ /** This cohort's served coord, base64url (the inbound routing key). */
155
+ readonly coord: string;
156
+ /** This cohort's epoch, base64url. */
157
+ readonly cohortEpoch: string;
158
+ /** This cohort's tree tier `d` — carried on the frame so a cold sibling instantiates at the right tier. */
159
+ readonly treeTier: number;
160
+ /**
161
+ * True to emit a **willingness-only heartbeat** even when this engine is idle (no resident topics, no
162
+ * pending deltas) — provided the node is actually willing for some tier. Lets an idle-but-willing engine
163
+ * still tell siblings it will serve, so a cold cohort can bootstrap (§Cold-start instantiation). The caller
164
+ * (the gossip round) sets this from the per-engine heartbeat clock; a non-idle round ignores it (it already
165
+ * ships willingness).
166
+ */
167
+ readonly heartbeat: boolean;
168
+ /** This node's tier profile (the `∧` half of the willingness vector). */
169
+ readonly profile: NodeProfile;
170
+ /** This member's load barometer (load buckets + the load-shed half of willingness). */
171
+ readonly barometer: LoadBarometerState;
172
+ /** Cohort-wide observation window (seconds) for the rate fields in `topicSummaries`. */
173
+ readonly windowSeconds: number;
174
+ /** Per-resident-topic summaries (own published counts), already frozen for this round. */
175
+ readonly topicSummaries: CohortTopicSummary[];
176
+ /** Drained fresh/touched record deltas for this round. */
177
+ readonly records: GossipRecordV1[];
178
+ /** Drained eviction refs for this round. */
179
+ readonly evicted: GossipRecordRefV1[];
180
+ /** Drained child-cohort link refs for this round (cross-member child-set convergence). */
181
+ readonly childLinks: ChildLinkRefV1[];
182
+ /** Drained child-cohort unlink refs for this round (a released/demoted child). */
183
+ readonly childUnlinks: ChildLinkRefV1[];
184
+ /** Round timestamp, unix ms. */
185
+ readonly timestamp: number;
186
+ }
187
+
188
+ /**
189
+ * Assemble one round's {@link CohortGossipV1}, or `undefined` when this engine has nothing to say. An engine
190
+ * is **idle** when it holds no resident topics and no pending deltas. An idle engine normally builds no frame
191
+ * (idle empty engines cost no gossip) — **except** on a willingness heartbeat ({@link GossipFrameInputs.heartbeat}),
192
+ * where an idle engine that is willing for at least one tier (`selfWillingnessBits !== 0`) still emits a
193
+ * willingness/load-only frame (empty `topicSummaries`, no `records`/`evicted`) so siblings can hear it and a
194
+ * cold cohort can bootstrap (§Cold-start instantiation). An idle-and-unwilling engine stays silent even on a
195
+ * heartbeat (nothing to bootstrap). `willingnessBits` is `profile ∧ load` ({@link selfWillingnessBits}); the
196
+ * `signature` slot is left empty for the host's peer-key signer to fill before broadcast.
197
+ */
198
+ export function buildCohortGossip(i: GossipFrameInputs): CohortGossipV1 | undefined {
199
+ const willingness = selfWillingnessBits(i.profile, i.barometer);
200
+ const idle = i.topicSummaries.length === 0 && i.records.length === 0 && i.evicted.length === 0
201
+ && i.childLinks.length === 0 && i.childUnlinks.length === 0;
202
+ if (idle && !(i.heartbeat && willingness !== 0)) {
203
+ return undefined;
204
+ }
205
+ const g: CohortGossipV1 = {
206
+ v: 1,
207
+ fromMember: i.fromMember,
208
+ coord: i.coord,
209
+ cohortEpoch: i.cohortEpoch,
210
+ treeTier: i.treeTier,
211
+ willingnessBits: willingnessBitsHex(willingness),
212
+ loadBuckets: i.barometer.loadBuckets(),
213
+ windowSeconds: i.windowSeconds,
214
+ topicSummaries: i.topicSummaries,
215
+ timestamp: i.timestamp,
216
+ signature: "",
217
+ };
218
+ if (i.records.length > 0) {
219
+ g.records = i.records;
220
+ }
221
+ if (i.evicted.length > 0) {
222
+ g.evicted = i.evicted;
223
+ }
224
+ if (i.childLinks.length > 0) {
225
+ g.childLinks = i.childLinks;
226
+ }
227
+ if (i.childUnlinks.length > 0) {
228
+ g.childUnlinks = i.childUnlinks;
229
+ }
230
+ return g;
231
+ }
@@ -1,84 +1,84 @@
1
- import type { ICohortGossipTransport, PeerRef, RingCoord } from "@optimystic/db-core";
2
- import type { Libp2p } from "libp2p";
3
- import { peerIdFromString } from "@libp2p/peer-id";
4
- import { peerIdToBytes } from "./peer-codec.js";
5
- import { sendOneWay } from "./stream-util.js";
6
- import { PROTOCOL_COHORT_GOSSIP } from "./protocols.js";
7
-
8
- /** Resolves the cohort owning a coord to its libp2p peer-id strings (FRET two-sided assembly). */
9
- export interface CohortPeerResolver {
10
- /** Peer-id strings of the cohort owning `coord`, up to `wants` members. */
11
- cohortPeers(coord: RingCoord, wants: number): string[];
12
- }
13
-
14
- export interface FretCohortGossipTransportOptions {
15
- readonly gossipProtocol?: string;
16
- /** Cohort fan-out for a broadcast. Default 16. */
17
- readonly wants?: number;
18
- /** This node's peer-id string — excluded from its own broadcasts. */
19
- readonly selfPeerId: string;
20
- }
21
-
22
- /**
23
- * FRET-backed {@link ICohortGossipTransport}: intra-cohort gossip over
24
- * `/optimystic/cohort-topic/1.0.0/cohort-gossip`. `broadcast` fans a frame out to the FRET-assembled
25
- * cohort for `coord` (fire-and-forget, self excluded); inbound frames arrive through the host's gossip
26
- * protocol handler, which calls {@link deliver}. Subscribers registered via `onMessage` see every
27
- * delivered frame.
28
- */
29
- export class FretCohortGossipTransport implements ICohortGossipTransport {
30
- private readonly handlers = new Set<(from: PeerRef, msg: Uint8Array) => void>();
31
- private readonly gossipProtocol: string;
32
- private readonly wants: number;
33
- private readonly selfPeerId: string;
34
-
35
- constructor(private readonly node: Libp2p, private readonly resolver: CohortPeerResolver, options: FretCohortGossipTransportOptions) {
36
- this.gossipProtocol = options.gossipProtocol ?? PROTOCOL_COHORT_GOSSIP;
37
- this.wants = options.wants ?? 16;
38
- this.selfPeerId = options.selfPeerId;
39
- }
40
-
41
- broadcast(coord: RingCoord, msg: Uint8Array): void {
42
- this.broadcastOver(this.gossipProtocol, coord, msg);
43
- }
44
-
45
- /**
46
- * Fan `msg` out to the FRET-assembled cohort for `coord` over an arbitrary one-way `protocol` (self
47
- * excluded, fire-and-forget). Reuses the cohort peer resolution so the `promote` protocol's
48
- * promotion/demotion notice broadcast shares the gossip transport's wiring; a single unreachable
49
- * member is recovered by the cohort's next convergence round, so per-peer failures are swallowed.
50
- */
51
- broadcastOver(protocol: string, coord: RingCoord, msg: Uint8Array): void {
52
- for (const peerStr of this.resolver.cohortPeers(coord, this.wants)) {
53
- if (peerStr === this.selfPeerId) {
54
- continue;
55
- }
56
- void sendOneWay(this.node, peerIdFromString(peerStr), protocol, msg).catch(() => {
57
- // Best-effort: a single unreachable member is recovered by the next round.
58
- });
59
- }
60
- }
61
-
62
- onMessage(handler: (from: PeerRef, msg: Uint8Array) => void): () => void {
63
- this.handlers.add(handler);
64
- return () => this.handlers.delete(handler);
65
- }
66
-
67
- /**
68
- * Number of live inbound subscriptions (one per subscribed gossip bus). Test/diagnostic introspection:
69
- * each {@link CoordEngine}'s bus subscribes via {@link onMessage} and drops its subscription on `close()`,
70
- * so this tracks the number of live coord engines (+ the node-level participant bus) — used to observe that
71
- * the coord-engine registry's LRU eviction actually tears evicted engines down (no leaked subscriptions).
72
- */
73
- get subscriberCount(): number {
74
- return this.handlers.size;
75
- }
76
-
77
- /** Feed an inbound gossip frame (called by the host's `/cohort-gossip` protocol handler). */
78
- deliver(fromPeerId: string, msg: Uint8Array): void {
79
- const from: PeerRef = { id: peerIdToBytes(fromPeerId) };
80
- for (const handler of this.handlers) {
81
- handler(from, msg);
82
- }
83
- }
84
- }
1
+ import type { ICohortGossipTransport, PeerRef, RingCoord } from "@optimystic/db-core";
2
+ import type { Libp2p } from "libp2p";
3
+ import { peerIdFromString } from "@libp2p/peer-id";
4
+ import { peerIdToBytes } from "./peer-codec.js";
5
+ import { sendOneWay } from "./stream-util.js";
6
+ import { PROTOCOL_COHORT_GOSSIP } from "./protocols.js";
7
+
8
+ /** Resolves the cohort owning a coord to its libp2p peer-id strings (FRET two-sided assembly). */
9
+ export interface CohortPeerResolver {
10
+ /** Peer-id strings of the cohort owning `coord`, up to `wants` members. */
11
+ cohortPeers(coord: RingCoord, wants: number): string[];
12
+ }
13
+
14
+ export interface FretCohortGossipTransportOptions {
15
+ readonly gossipProtocol?: string;
16
+ /** Cohort fan-out for a broadcast. Default 16. */
17
+ readonly wants?: number;
18
+ /** This node's peer-id string — excluded from its own broadcasts. */
19
+ readonly selfPeerId: string;
20
+ }
21
+
22
+ /**
23
+ * FRET-backed {@link ICohortGossipTransport}: intra-cohort gossip over
24
+ * `/optimystic/cohort-topic/1.0.0/cohort-gossip`. `broadcast` fans a frame out to the FRET-assembled
25
+ * cohort for `coord` (fire-and-forget, self excluded); inbound frames arrive through the host's gossip
26
+ * protocol handler, which calls {@link deliver}. Subscribers registered via `onMessage` see every
27
+ * delivered frame.
28
+ */
29
+ export class FretCohortGossipTransport implements ICohortGossipTransport {
30
+ private readonly handlers = new Set<(from: PeerRef, msg: Uint8Array) => void>();
31
+ private readonly gossipProtocol: string;
32
+ private readonly wants: number;
33
+ private readonly selfPeerId: string;
34
+
35
+ constructor(private readonly node: Libp2p, private readonly resolver: CohortPeerResolver, options: FretCohortGossipTransportOptions) {
36
+ this.gossipProtocol = options.gossipProtocol ?? PROTOCOL_COHORT_GOSSIP;
37
+ this.wants = options.wants ?? 16;
38
+ this.selfPeerId = options.selfPeerId;
39
+ }
40
+
41
+ broadcast(coord: RingCoord, msg: Uint8Array): void {
42
+ this.broadcastOver(this.gossipProtocol, coord, msg);
43
+ }
44
+
45
+ /**
46
+ * Fan `msg` out to the FRET-assembled cohort for `coord` over an arbitrary one-way `protocol` (self
47
+ * excluded, fire-and-forget). Reuses the cohort peer resolution so the `promote` protocol's
48
+ * promotion/demotion notice broadcast shares the gossip transport's wiring; a single unreachable
49
+ * member is recovered by the cohort's next convergence round, so per-peer failures are swallowed.
50
+ */
51
+ broadcastOver(protocol: string, coord: RingCoord, msg: Uint8Array): void {
52
+ for (const peerStr of this.resolver.cohortPeers(coord, this.wants)) {
53
+ if (peerStr === this.selfPeerId) {
54
+ continue;
55
+ }
56
+ void sendOneWay(this.node, peerIdFromString(peerStr), protocol, msg).catch(() => {
57
+ // Best-effort: a single unreachable member is recovered by the next round.
58
+ });
59
+ }
60
+ }
61
+
62
+ onMessage(handler: (from: PeerRef, msg: Uint8Array) => void): () => void {
63
+ this.handlers.add(handler);
64
+ return () => this.handlers.delete(handler);
65
+ }
66
+
67
+ /**
68
+ * Number of live inbound subscriptions (one per subscribed gossip bus). Test/diagnostic introspection:
69
+ * each {@link CoordEngine}'s bus subscribes via {@link onMessage} and drops its subscription on `close()`,
70
+ * so this tracks the number of live coord engines (+ the node-level participant bus) — used to observe that
71
+ * the coord-engine registry's LRU eviction actually tears evicted engines down (no leaked subscriptions).
72
+ */
73
+ get subscriberCount(): number {
74
+ return this.handlers.size;
75
+ }
76
+
77
+ /** Feed an inbound gossip frame (called by the host's `/cohort-gossip` protocol handler). */
78
+ deliver(fromPeerId: string, msg: Uint8Array): void {
79
+ const from: PeerRef = { id: peerIdToBytes(fromPeerId) };
80
+ for (const handler of this.handlers) {
81
+ handler(from, msg);
82
+ }
83
+ }
84
+ }