@optimystic/db-p2p 0.25.1 → 0.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/dist/src/cluster/certified-claims.d.ts +17 -3
  2. package/dist/src/cluster/certified-claims.d.ts.map +1 -1
  3. package/dist/src/cluster/certified-claims.js +5 -3
  4. package/dist/src/cluster/certified-claims.js.map +1 -1
  5. package/dist/src/cluster/cluster-repo.d.ts +57 -92
  6. package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
  7. package/dist/src/cluster/cluster-repo.js +149 -201
  8. package/dist/src/cluster/cluster-repo.js.map +1 -1
  9. package/dist/src/cluster/commit-proof.d.ts +16 -0
  10. package/dist/src/cluster/commit-proof.d.ts.map +1 -1
  11. package/dist/src/cluster/commit-proof.js +32 -1
  12. package/dist/src/cluster/commit-proof.js.map +1 -1
  13. package/dist/src/cluster/quorum-restore.d.ts +81 -28
  14. package/dist/src/cluster/quorum-restore.d.ts.map +1 -1
  15. package/dist/src/cluster/quorum-restore.js +148 -51
  16. package/dist/src/cluster/quorum-restore.js.map +1 -1
  17. package/dist/src/cluster/race-resolution.d.ts +89 -0
  18. package/dist/src/cluster/race-resolution.d.ts.map +1 -0
  19. package/dist/src/cluster/race-resolution.js +140 -0
  20. package/dist/src/cluster/race-resolution.js.map +1 -0
  21. package/dist/src/cluster/reconcile-block.d.ts +9 -4
  22. package/dist/src/cluster/reconcile-block.d.ts.map +1 -1
  23. package/dist/src/cluster/reconcile-block.js +28 -11
  24. package/dist/src/cluster/reconcile-block.js.map +1 -1
  25. package/dist/src/cluster/record-operations.d.ts +25 -0
  26. package/dist/src/cluster/record-operations.d.ts.map +1 -0
  27. package/dist/src/cluster/record-operations.js +56 -0
  28. package/dist/src/cluster/record-operations.js.map +1 -0
  29. package/dist/src/cohort-topic/host.d.ts +133 -51
  30. package/dist/src/cohort-topic/host.d.ts.map +1 -1
  31. package/dist/src/cohort-topic/host.js +290 -83
  32. package/dist/src/cohort-topic/host.js.map +1 -1
  33. package/dist/src/libp2p-node-base.d.ts +13 -4
  34. package/dist/src/libp2p-node-base.d.ts.map +1 -1
  35. package/dist/src/libp2p-node-base.js +5 -5
  36. package/dist/src/libp2p-node-base.js.map +1 -1
  37. package/dist/src/repo/cluster-coordinator.d.ts +9 -0
  38. package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
  39. package/dist/src/repo/cluster-coordinator.js +13 -2
  40. package/dist/src/repo/cluster-coordinator.js.map +1 -1
  41. package/dist/src/repo/coordinator-repo.d.ts +34 -2
  42. package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
  43. package/dist/src/repo/coordinator-repo.js +57 -3
  44. package/dist/src/repo/coordinator-repo.js.map +1 -1
  45. package/dist/src/storage/block-latch.d.ts +5 -4
  46. package/dist/src/storage/block-latch.d.ts.map +1 -1
  47. package/dist/src/storage/block-latch.js +5 -4
  48. package/dist/src/storage/block-latch.js.map +1 -1
  49. package/dist/src/storage/block-storage.d.ts +1 -1
  50. package/dist/src/storage/block-storage.d.ts.map +1 -1
  51. package/dist/src/storage/block-storage.js +11 -3
  52. package/dist/src/storage/block-storage.js.map +1 -1
  53. package/dist/src/storage/cached-raw-storage.d.ts +17 -1
  54. package/dist/src/storage/cached-raw-storage.d.ts.map +1 -1
  55. package/dist/src/storage/cached-raw-storage.js +8 -1
  56. package/dist/src/storage/cached-raw-storage.js.map +1 -1
  57. package/dist/src/storage/cached-store-driver.d.ts +7 -0
  58. package/dist/src/storage/cached-store-driver.d.ts.map +1 -1
  59. package/dist/src/storage/cached-store-driver.js +20 -0
  60. package/dist/src/storage/cached-store-driver.js.map +1 -1
  61. package/dist/src/storage/i-block-storage.d.ts +45 -2
  62. package/dist/src/storage/i-block-storage.d.ts.map +1 -1
  63. package/dist/src/storage/i-block-storage.js +29 -0
  64. package/dist/src/storage/i-block-storage.js.map +1 -1
  65. package/dist/src/storage/i-raw-storage.d.ts +16 -0
  66. package/dist/src/storage/i-raw-storage.d.ts.map +1 -1
  67. package/dist/src/storage/kv-raw-storage.d.ts +5 -1
  68. package/dist/src/storage/kv-raw-storage.d.ts.map +1 -1
  69. package/dist/src/storage/kv-raw-storage.js +8 -1
  70. package/dist/src/storage/kv-raw-storage.js.map +1 -1
  71. package/dist/src/storage/raw-store-driver.d.ts +15 -0
  72. package/dist/src/storage/raw-store-driver.d.ts.map +1 -1
  73. package/dist/src/storage/shared-cache-pool.d.ts +5 -0
  74. package/dist/src/storage/shared-cache-pool.d.ts.map +1 -1
  75. package/dist/src/storage/shared-cache-pool.js +6 -1
  76. package/dist/src/storage/shared-cache-pool.js.map +1 -1
  77. package/dist/src/storage/storage-repo.d.ts.map +1 -1
  78. package/dist/src/storage/storage-repo.js +141 -99
  79. package/dist/src/storage/storage-repo.js.map +1 -1
  80. package/dist/src/storage/with-read-cache.d.ts +9 -5
  81. package/dist/src/storage/with-read-cache.d.ts.map +1 -1
  82. package/dist/src/storage/with-read-cache.js +16 -6
  83. package/dist/src/storage/with-read-cache.js.map +1 -1
  84. package/package.json +2 -2
  85. package/{README.md → readme.md} +2 -2
  86. package/src/cluster/certified-claims.ts +22 -9
  87. package/src/cluster/cluster-repo.ts +199 -212
  88. package/src/cluster/commit-proof.ts +38 -2
  89. package/src/cluster/quorum-restore.ts +183 -56
  90. package/src/cluster/race-resolution.ts +158 -0
  91. package/src/cluster/reconcile-block.ts +34 -11
  92. package/src/cluster/record-operations.ts +59 -0
  93. package/src/cohort-topic/host.ts +423 -118
  94. package/src/libp2p-node-base.ts +18 -9
  95. package/src/repo/cluster-coordinator.ts +1039 -1027
  96. package/src/repo/coordinator-repo.ts +1937 -1855
  97. package/src/storage/block-latch.ts +5 -4
  98. package/src/storage/block-storage.ts +11 -3
  99. package/src/storage/cached-raw-storage.ts +21 -1
  100. package/src/storage/cached-store-driver.ts +23 -0
  101. package/src/storage/i-block-storage.ts +47 -2
  102. package/src/storage/i-raw-storage.ts +17 -0
  103. package/src/storage/kv-raw-storage.ts +8 -1
  104. package/src/storage/raw-store-driver.ts +26 -0
  105. package/src/storage/shared-cache-pool.ts +6 -1
  106. package/src/storage/storage-repo.ts +143 -103
  107. package/src/storage/with-read-cache.ts +16 -6
@@ -285,9 +285,10 @@ export interface CohortTopicAntiDosOptions {
285
285
  * Hard cap on the number of live per-coord cohort engines this node keeps. The served coord is a hash
286
286
  * over attacker-chosen `(treeTier, participantCoord, topicId)`, and an engine is created **before** the
287
287
  * per-coord anti-DoS gates run, so without this cap one peer spraying distinct coords forces unbounded
288
- * engine allocation. On overflow the registry evicts the least-recently-used **idle** engine (no records,
289
- * no cold-start forwarder) and tears it down; if every slot holds a live cohort it refuses the new coord.
290
- * Default {@link DEFAULT_COORD_ENGINES_MAX}.
288
+ * engine allocation. On overflow the registry evicts the lowest-ranked, least-recently-used candidate
289
+ * engine (see {@link EVICTION_RANK}: engines holding nothing go first, child-/promotion-holding engines
290
+ * last) and tears it down; if every slot holds a pinned engine (records or a cold-start forwarder) it
291
+ * refuses the new coord. Default {@link DEFAULT_COORD_ENGINES_MAX}.
291
292
  */
292
293
  readonly coordEnginesMax?: number;
293
294
  /**
@@ -335,6 +336,18 @@ export interface CohortTopicAntiDosOptions {
335
336
  readonly parentTopicView?: BootstrapParentTopicView;
336
337
  }
337
338
 
339
+ /**
340
+ * The classes of engine-owned state that evicting a {@link CoordEngine} destroys. **Exhaustive by
341
+ * contract**: every piece of state a `CoordEngine` owns and eviction would lose must appear here, and
342
+ * every consumer indexes it as a total `Record` ({@link EngineLiveness}, {@link EVICTION_RANK}), so
343
+ * adding a member without declaring it everywhere is a compile error — never a silently-evictable
344
+ * fifth piece of state.
345
+ */
346
+ export type EngineStateKind = "records" | "forwarders" | "children" | "promotion";
347
+
348
+ /** Which state classes an engine currently holds. Total — one boolean per kind, never partial. */
349
+ export type EngineLiveness = Readonly<Record<EngineStateKind, boolean>>;
350
+
338
351
  /**
339
352
  * One cohort the node serves, bound to a FRET-routed coordinate. Owns the per-coord slice of cohort
340
353
  * state; the node-wide collaborators are injected (see {@link CoordEngineContext}).
@@ -356,13 +369,19 @@ export interface CoordEngine {
356
369
  * the requester then re-anchors). See `cohort-topic-trust-anchor-rotation-production`.
357
370
  */
358
371
  cohortIdentityAt(epoch: Uint8Array): readonly string[] | undefined;
359
- /** True iff this engine currently holds any registration record (a cold probe leaves it empty). */
372
+ /**
373
+ * Which classes of engine-owned state this engine currently holds — the single, exhaustive answer to
374
+ * "does evicting this engine destroy state it cannot cheaply reconstruct?". The registry's eviction
375
+ * ranking consumes it ({@link EVICTION_RANK}); {@link hasState} / {@link hasForwarders} are
376
+ * derivations of it, so the two views can never disagree.
377
+ */
378
+ liveness(): EngineLiveness;
379
+ /** True iff this engine currently holds any registration record (= `liveness().records`). */
360
380
  hasState(): boolean;
361
381
  /**
362
- * True iff this engine currently holds any cold-start forwarder (possibly `awaiting_parent`). Together
363
- * with {@link hasState}, this is the "engine is idle / safe to reclaim" predicate the coord-engine
364
- * registry's LRU eviction reads: an engine with neither a record nor a forwarder is a throwaway (an
365
- * attacker-sprayed cold coord), an engine with either holds genuine cohort state and is never evicted.
382
+ * True iff this engine currently holds any cold-start forwarder (possibly `awaiting_parent`)
383
+ * (= `liveness().forwarders`). Records and forwarders are the *pinned* state classes: an engine holding
384
+ * either is never an eviction candidate at all (see {@link EVICTION_RANK}).
366
385
  */
367
386
  hasForwarders(): boolean;
368
387
  /** True iff this engine holds the record for `(topicId, participantId)` — the renewal lookup key. */
@@ -475,9 +494,11 @@ export interface CoordRegistry {
475
494
  *
476
495
  * **Capacity.** The registry is hard-capped (see {@link CohortTopicAntiDosOptions.coordEnginesMax}). A
477
496
  * lookup that returns an already-resident engine always succeeds. A *creation* over a full registry
478
- * first evicts the least-recently-used idle engine; when every slot holds a live cohort it throws
479
- * {@link CoordEngineRegistryFullError} rather than growing unbounded callers on the register /
480
- * child-link / cold-sibling paths catch it and answer a clean capacity refusal.
497
+ * first evicts the lowest-ranked, least-recently-used candidate engine ({@link EVICTION_RANK}: engines
498
+ * holding nothing before child-/promotion-holding ones); when every slot holds a pinned engine (records
499
+ * or a cold-start forwarder) it throws {@link CoordEngineRegistryFullError} rather than growing
500
+ * unbounded — callers on the register / child-link / cold-sibling paths catch it and answer a clean
501
+ * capacity refusal.
481
502
  */
482
503
  forCoord(coord: RingCoord, treeTier: number, participantCoord: Uint8Array): CoordEngine;
483
504
  /** The engine holding the record for `(topicId, participantId)`, or `undefined` (renewal dispatch). */
@@ -528,9 +549,10 @@ export interface CohortTopicHost {
528
549
  */
529
550
  readonly gossipTransport: FretCohortGossipTransport;
530
551
  /**
531
- * The node-level `promote`-handler anti-abuse gate (per-`(peer, topic)` rate limiter + per-`(topic, tier)`
532
- * `effectiveAt` high-water). Exposed for test/diagnostic introspection over its bounded-memory state — the
533
- * limiter's `size` and the `highWater` `LruMap` — and so the gossip-cadence sweep wiring is observable.
552
+ * The node-level `promote`-handler anti-abuse gate (per-`(peer, topic)` rate limiter + the
553
+ * per-`(coord, tier, topic)` adopted-transition record). Exposed for test/diagnostic introspection over
554
+ * its bounded-memory state — the limiter's `size` and the `transitions` `LruMap` — and so the
555
+ * gossip-cadence sweep wiring is observable.
534
556
  */
535
557
  readonly promoteGate: PromoteGate;
536
558
  /**
@@ -611,6 +633,26 @@ interface CoordEngineContext {
611
633
  * notices signed by its own cohort without a network refetch. Absent in unit composition.
612
634
  */
613
635
  readonly onCertPublished?: (cert: MembershipCertV1) => void;
636
+ /**
637
+ * Hook fired by the coord-engine registry AFTER it evicts (and closes) an engine, with the evicted
638
+ * engine's served coord. The host wires it to `verifier.forget(coord)`: on a keyed node the
639
+ * gossip-cadence driver calls {@link CoordEngine.pumpMembership} for EVERY engine — record-less ones
640
+ * included — so any evicted engine may have published a cert and trust-locked its coord in the
641
+ * verifier ({@link onCertPublished} → `verifier.cache`). Eviction therefore drops the lock
642
+ * unconditionally (a cheap no-op for a coord that never published) rather than trying to track "did
643
+ * this engine publish" — that would be another declare-yourself-or-be-forgotten trap. Absent in
644
+ * key-less / unit composition.
645
+ */
646
+ readonly onEngineEvicted?: (coord: RingCoord) => void;
647
+ /**
648
+ * Read the node-level record of the last adopted promotion/demotion transition for one
649
+ * `(coord, tier, topic)` — the host's {@link PromoteGate.transitions} map, which outlives every
650
+ * engine. A freshly created {@link CoordEngine} seeds its {@link PromotionLifecycle}'s per-topic
651
+ * ordering and direction from it (via `PromotionDeps.seedTransition`), so eviction + recreation
652
+ * neither reopens the replay window nor forgets a correct promoted mode. Wired unconditionally by the
653
+ * host (key-less nodes also adopt verified inbound notices); optional only for unit composition.
654
+ */
655
+ readonly adoptedTransition?: (coord: RingCoord, tier: number, topicId: Uint8Array) => AdoptedTransition | undefined;
614
656
  }
615
657
 
616
658
  /**
@@ -713,12 +755,26 @@ export async function createCohortTopicHost(node: Libp2p, fret: FretService, opt
713
755
  return members.includes(bytesToPeerIdString(fromBytes));
714
756
  };
715
757
 
758
+ // Node-level `promote`-handler anti-abuse gate (`cohort-topic-promote-handler-verify-amplification`):
759
+ // a per-(peer, topic) rate limiter (own instance — the register-path limiter is per-coord inside each
760
+ // engine; this handler is node-level) plus the per-(coord, tier, topic) adopted-transition record.
761
+ // Defaults to `register_rate_per_peer` (4 / min / peer / topic) with exponential back-off. Declared here
762
+ // rather than with the other anti-DoS wiring below because `broadcastNotice` writes the record on the
763
+ // origination path.
764
+ const promoteGate = createPromoteGate(options.antiDos?.rateLimiter);
765
+
716
766
  // --- outbound notice broadcast (gap 4) ---
717
767
  // A coord engine that threshold-signs a promotion/demotion notice hands it here; we fan it over the
718
768
  // `promote` protocol to the cohort around the served coord (siblings adopt the state) and, for a
719
769
  // demotion, additionally to the parent coord (childCohortCount bookkeeping). Reuses the gossip
720
770
  // transport's cohort peer resolution.
721
771
  const broadcastNotice = (notice: PromotionNoticeV1 | DemotionNoticeV1, servedCoord: RingCoord): void => {
772
+ // Record the origination in the node-level adopted-transition map BEFORE fanning out: `broadcastOver`
773
+ // excludes self, so an originated notice never arrives back on the inbound path — this is the ONLY
774
+ // write for locally-originated transitions. Keyed off the notice's own `cohortCoord` only; the parent
775
+ // coord a demotion also fans to is deliberately outside this ordering (the parent-unlink is ordered by
776
+ // the child registry's per-child `lastEffectiveAt` — see `applyDemotionUnlinkAtParent`).
777
+ recordAdoptedTransition(promoteGate, notice);
722
778
  const frame = encodeCohortMessage(notice, maxBytes);
723
779
  // A demotion fans to BOTH the demoting child's served coord (siblings adopt `promoted = false` via the
724
780
  // `cohortCoord`-routed apply) and the parent coord (the parent unrecords the child). `handleInboundNotice`
@@ -755,12 +811,6 @@ export async function createCohortTopicHost(node: Libp2p, fret: FretService, opt
755
811
  options.antiDos?.parentTopicView !== undefined || options.committedParentTopicReader !== undefined;
756
812
  const bootstrapEvidence = createBootstrapEvidencePolicy(options.antiDos, hash, log, parentTopicView, hasCommittedParentBacking);
757
813
 
758
- // Node-level `promote`-handler anti-abuse gate (`cohort-topic-promote-handler-verify-amplification`):
759
- // a per-(peer, topic) rate limiter (own instance — the register-path limiter is per-coord inside each
760
- // engine; this handler is node-level) plus the per-(topic, tier) effectiveAt high-water. Defaults to
761
- // `register_rate_per_peer` (4 / min / peer / topic) with exponential back-off.
762
- const promoteGate = createPromoteGate(options.antiDos?.rateLimiter);
763
-
764
814
  const ctx: CoordEngineContext = {
765
815
  hash,
766
816
  addressing,
@@ -799,13 +849,20 @@ export async function createCohortTopicHost(node: Libp2p, fret: FretService, opt
799
849
  // cohort it stops republishing and its anchor goes `"unknown"`, so it can be stranded distrusting the
800
850
  // coord's later-epoch messages. The verifier self-heals via bounded re-TOFU on a demonstrated chain gap
801
851
  // (`staleGapRecoveryStrikes`, see `db-core/.../membership/verifier.ts` + `docs/cohort-topic.md`
802
- // §Bootstrapping trust). The *root-cause* fix is for the host to drop the lock here on demotion, but
803
- // that needs an engine-reclaim / demotion signal the host does not emit today. `createCoordRegistry`
804
- // now evicts, but only IDLE engines (no records → never published a cert), so it never strands a
805
- // trust-lock and does not resolve this on its own (see the NOTE at `evictOneIdle`). When a demotion /
806
- // cert-publishing-engine reclaim signal lands, add a `verifier.forget(coord)` / downgrade call on
807
- // demotion and prefer it over (or alongside) the strike-counter heuristic.
852
+ // §Bootstrapping trust). Engine EVICTION drops the lock for the evicted coord via `onEngineEvicted`
853
+ // below on a keyed node `pumpMembership` publishes for every engine (record-less ones included), so
854
+ // eviction cannot assume "no records → never published". Demotion, however, does NOT release the engine
855
+ // (a demoted engine keeps its records/forwarder and stays resident), so the demotion-side lock drop
856
+ // still has no signal to hang off; the strike-counter heuristic remains the recovery there.
808
857
  onCertPublished: (cert: MembershipCertV1): void => verifier.cache(cert),
858
+ // Drop the verifier trust-lock for an evicted engine's coord — unconditional, because on a keyed node
859
+ // ANY engine (even one holding no records) may have published a cert via the gossip-cadence
860
+ // `pumpMembership` sweep. A no-op for a coord the verifier holds nothing for.
861
+ onEngineEvicted: (coord: RingCoord): void => verifier.forget(bytesToB64url(coord)),
862
+ // Node-level adopted-transition reader (engine seeding). Wired unconditionally — key-less hosts also
863
+ // adopt verified inbound notices, so their recreated engines need the seed just the same.
864
+ adoptedTransition: (coord: RingCoord, tier: number, topicId: Uint8Array): AdoptedTransition | undefined =>
865
+ promoteGate.transitions.get(transitionKey(bytesToB64url(coord), tier, bytesToB64url(topicId))),
809
866
  };
810
867
  const registry = createCoordRegistry(ctx, options.antiDos?.coordEnginesMax);
811
868
 
@@ -829,10 +886,11 @@ export async function createCohortTopicHost(node: Libp2p, fret: FretService, opt
829
886
  // link work — so a tier-`d > 0` frame for an unknown coord falls through to today's drop (the bus has no
830
887
  // engine subscribed to it). See `docs/cohort-topic.md` §Cold-start instantiation.
831
888
  //
832
- // The registry is hard-capped with LRU eviction of idle engines (`createCoordRegistry`), so a
833
- // gossip-instantiated cold sibling is no longer a permanent per-co-member-coord cost: an idle one is
834
- // reclaimed under memory pressure like any other cold engine, and a creation over a full-of-live registry
835
- // is refused (`CoordEngineRegistryFullError`) and dropped here rather than crashing the gossip handler.
889
+ // The registry is hard-capped with ranked LRU eviction (`createCoordRegistry`), so a
890
+ // gossip-instantiated cold sibling is no longer a permanent per-co-member-coord cost: one holding no
891
+ // state is reclaimed under memory pressure like any other cold engine, and a creation over a
892
+ // full-of-pinned registry is refused (`CoordEngineRegistryFullError`) and dropped here rather than
893
+ // crashing the gossip handler.
836
894
  const maybeInstantiateColdSibling = (frame: Uint8Array): void => {
837
895
  if (verifyGossip === undefined) {
838
896
  return; // key-less / interim mode: no co-member gate, so never auto-instantiate
@@ -1347,7 +1405,7 @@ export const DEFAULT_COORD_ENGINES_MAX = 2048;
1347
1405
 
1348
1406
  /**
1349
1407
  * Thrown by {@link CoordRegistry.forCoord} when it must create a new engine but the registry is full of
1350
- * **live** cohorts (every slot holds records or a cold-start forwarder, so nothing is idle-evictable).
1408
+ * **pinned** engines (every slot holds records or a cold-start forwarder, so nothing is evictable).
1351
1409
  * Signals a capacity refusal, not a bug — the register / child-link / cold-sibling dispatch paths catch it
1352
1410
  * and answer a clean refusal (`unwilling_cohort` / `rejected` / drop) rather than letting it escape.
1353
1411
  */
@@ -1358,20 +1416,47 @@ export class CoordEngineRegistryFullError extends Error {
1358
1416
  }
1359
1417
  }
1360
1418
 
1419
+ /**
1420
+ * Eviction rank per {@link EngineStateKind}. `"pinned"` — an engine holding it is not an eviction
1421
+ * candidate at all. A number — the engine IS a candidate, but only after every candidate whose rank is
1422
+ * strictly lower; higher = evicted later. An engine holding nothing is rank 0: the genuinely-cold
1423
+ * attacker-sprayed coord the cap exists for.
1424
+ *
1425
+ * `children` / `promotion` are deliberately NOT pinned. A child link is peer-supplied input —
1426
+ * key-less-permissive mode records one without any signature check — so pinning on it would let any peer
1427
+ * make all `coordEnginesMax` slots un-evictable and drive {@link CoordEngineRegistryFullError} for
1428
+ * legitimate coords, reopening the exact spray vector the cap closes. Ranking keeps the hard guarantee
1429
+ * (some engine is always evictable while any unpinned one exists) while making the realistic loss — a
1430
+ * handful of real parent/promoted engines among ~2000 attacker-cold ones — the *last* to go instead of a
1431
+ * pure LRU-age pick.
1432
+ *
1433
+ * EXHAUSTIVE: a new {@link EngineStateKind} with no rank here does not typecheck, and the spec's
1434
+ * exhaustiveness guard cross-checks this table's keys against a `liveness()` result at runtime.
1435
+ */
1436
+ export const EVICTION_RANK: Record<EngineStateKind, "pinned" | number> = {
1437
+ records: "pinned",
1438
+ forwarders: "pinned",
1439
+ children: 1,
1440
+ promotion: 1,
1441
+ };
1442
+
1361
1443
  /**
1362
1444
  * Build the lazy `servedCoord → CoordEngine` registry over the shared collaborators, hard-capped at
1363
- * `maxEngines` with least-recently-used eviction of **idle** engines.
1445
+ * `maxEngines` with ranked least-recently-used eviction.
1364
1446
  *
1365
1447
  * The served coord is a hash over attacker-chosen `(treeTier, participantCoord, topicId)`, and `forCoord`
1366
1448
  * runs on the register hot path **before** the per-coord anti-DoS gates — so, uncapped, one peer spraying
1367
1449
  * distinct coords drives unbounded engine allocation (each engine owns a store, gossip bus, rate limiter,
1368
1450
  * replay guard, topic budget, …). The cap bounds that: on a creation over a full registry we evict the
1369
- * least-recently-used **idle** engine (no records, no cold-start forwarder — a throwaway cold coord) and
1370
- * tear it down; when every slot holds a live cohort we refuse the new coord ({@link CoordEngineRegistryFullError})
1371
- * so a legitimate multi-cohort node keeps working while attacker-driven cold engines cannot pile up.
1451
+ * `(rank, recency)`-least candidate engine ({@link EVICTION_RANK} over {@link CoordEngine.liveness}) and
1452
+ * tear it down; when every slot holds a pinned engine (records or a forwarder) we refuse the new coord
1453
+ * ({@link CoordEngineRegistryFullError}) so a legitimate multi-cohort node keeps working while
1454
+ * attacker-driven cold engines cannot pile up.
1372
1455
  *
1373
1456
  * Recency is bumped on every lookup that hands back an engine (`forCoord` / `findByCoord` / `findHolder` /
1374
- * `findServing`), so a hot cohort under load is never the eviction victim.
1457
+ * `findServing`) and breaks ties *within* a rank, so a hot cohort under load loses only to a colder engine
1458
+ * of the same rank — but rank is applied first, so even a hot rank-0 engine is evicted before any rank-1
1459
+ * one (intended: a child-/promotion-holding engine survives a spray of freshly-touched cold coords).
1375
1460
  */
1376
1461
  function createCoordRegistry(ctx: CoordEngineContext, maxEngines: number = DEFAULT_COORD_ENGINES_MAX): CoordRegistry {
1377
1462
  if (!Number.isInteger(maxEngines) || maxEngines <= 0) {
@@ -1384,28 +1469,49 @@ function createCoordRegistry(ctx: CoordEngineContext, maxEngines: number = DEFAU
1384
1469
  let seq = 0;
1385
1470
  const touch = (key: string): void => { recency.set(key, ++seq); };
1386
1471
 
1387
- // An engine is idle-evictable iff it holds no registration record AND no cold-start forwarder i.e. no
1388
- // genuine cohort state to lose. A live engine (records or a forwarder) is never a throwaway.
1389
- const isIdle = (engine: CoordEngine): boolean => !engine.hasState() && !engine.hasForwarders();
1472
+ // An engine's eviction rank: `undefined` when it holds any pinned state class (not a candidate at all),
1473
+ // else the maximum numeric rank over the classes it holds (0 when it holds nothing). Iterates the rank
1474
+ // table, whose keys the type system pins to exactly the EngineStateKind set the liveness record carries.
1475
+ const evictionRank = (engine: CoordEngine): number | undefined => {
1476
+ const liveness = engine.liveness();
1477
+ let rank = 0;
1478
+ for (const kind of Object.keys(EVICTION_RANK) as EngineStateKind[]) {
1479
+ if (!liveness[kind]) {
1480
+ continue;
1481
+ }
1482
+ const r = EVICTION_RANK[kind];
1483
+ if (r === "pinned") {
1484
+ return undefined;
1485
+ }
1486
+ if (r > rank) {
1487
+ rank = r;
1488
+ }
1489
+ }
1490
+ return rank;
1491
+ };
1390
1492
 
1391
- // Evict the least-recently-used idle engine to free a slot; returns true iff one was freed. Tears the
1392
- // victim down (`close()` drops its gossip-bus subscription) so eviction does not leak the subscription.
1493
+ // Evict the `(rank, recency)`-lexicographically-least candidate engine to free a slot; returns true iff
1494
+ // one was freed. Tears the victim down (`close()` drops its gossip-bus subscription) so eviction does not
1495
+ // leak the subscription, and fires `ctx.onEngineEvicted` so the host drops the coord's verifier
1496
+ // trust-lock.
1393
1497
  //
1394
- // NOTE (verifier trust-lock, cohort-topic-treetier-bound-engine-cap): only IDLE engines are evicted here,
1395
- // and an idle engine (`hasState() === false`) has never published a membership cert — so there is no
1396
- // verifier trust-lock (`onCertPublished` `verifier.cache`, above) to drop for its coord. If this policy
1397
- // is ever widened to evict a cert-publishing engine, add a `verifier.forget(coord)` / downgrade here:
1398
- // otherwise the stale trust-lock strands the coord's later-epoch messages (the drop-the-lock tripwire the
1399
- // `onCertPublished` NOTE describes). Do NOT widen without that.
1400
- const evictOneIdle = (): boolean => {
1498
+ // NOTE (verifier trust-lock): eviction can reclaim an engine that HAS published a membership cert — on a
1499
+ // keyed node the gossip-cadence driver calls `pumpMembership` for every engine in `registry.all()`,
1500
+ // record-less ones included, so "no records" never implied "never published". `onEngineEvicted` is
1501
+ // therefore called for EVERY victim, unconditionally (the host wires it to `verifier.forget(coord)`, a
1502
+ // cheap no-op for a coord that never published); do not try to track which engines published.
1503
+ const evictOne = (): boolean => {
1401
1504
  let victimKey: string | undefined;
1505
+ let victimRank = Infinity;
1402
1506
  let victimSeq = Infinity;
1403
1507
  for (const [key, engine] of engines) {
1404
- if (!isIdle(engine)) {
1405
- continue; // live cohort (records) or mid-link cold-start forwarder — never evicted
1508
+ const rank = evictionRank(engine);
1509
+ if (rank === undefined) {
1510
+ continue; // pinned: holds records or a mid-link cold-start forwarder — never evicted
1406
1511
  }
1407
1512
  const s = recency.get(key) ?? 0;
1408
- if (s < victimSeq) {
1513
+ if (rank < victimRank || (rank === victimRank && s < victimSeq)) {
1514
+ victimRank = rank;
1409
1515
  victimSeq = s;
1410
1516
  victimKey = key;
1411
1517
  }
@@ -1413,9 +1519,11 @@ function createCoordRegistry(ctx: CoordEngineContext, maxEngines: number = DEFAU
1413
1519
  if (victimKey === undefined) {
1414
1520
  return false;
1415
1521
  }
1416
- engines.get(victimKey)!.close();
1522
+ const victim = engines.get(victimKey)!;
1523
+ victim.close();
1417
1524
  engines.delete(victimKey);
1418
1525
  recency.delete(victimKey);
1526
+ ctx.onEngineEvicted?.(victim.servedCoord);
1419
1527
  return true;
1420
1528
  };
1421
1529
 
@@ -1426,8 +1534,8 @@ function createCoordRegistry(ctx: CoordEngineContext, maxEngines: number = DEFAU
1426
1534
  // share one engine rather than racing to construct a second.
1427
1535
  let engine = engines.get(key);
1428
1536
  if (engine === undefined) {
1429
- if (engines.size >= maxEngines && !evictOneIdle()) {
1430
- // Full of live cohorts — refuse rather than grow unbounded. Callers turn this into a clean
1537
+ if (engines.size >= maxEngines && !evictOne()) {
1538
+ // Full of pinned engines — refuse rather than grow unbounded. Callers turn this into a clean
1431
1539
  // capacity reply/drop (see the paths listed on `CoordEngineRegistryFullError`).
1432
1540
  log("cohort-topic: coord-engine registry full (max=%d) — refusing new coord %s", maxEngines, key);
1433
1541
  throw new CoordEngineRegistryFullError(maxEngines);
@@ -1560,9 +1668,10 @@ class RotationState {
1560
1668
  * cohort-wide. Because merge is last-writer-wins by `effectiveAt`, a link and a later unlink converge in any
1561
1669
  * arrival order, and a never-seen unlink writes a `linked = false` tombstone that a subsequently-arriving stale
1562
1670
  * link cannot resurrect. The union is keyed by child coord, never the parent epoch, so a parent membership
1563
- * rotation does not reset it. NOTE: a link/unlink is broadcast once (drained from the pending delta queue);
1564
- * a parent member that joins the cohort *after* the delta drained (rotation cold-start) learns the child set
1565
- * only on the next local record/unrecord for that child — see `debt-cohort-topic-child-set-late-joiner-resync`.
1671
+ * rotation does not reset it. A link/unlink is broadcast once (drained from the pending delta queue), so a
1672
+ * member that missed that one frame a rotation cold-start, most of all — is healed by the throttled
1673
+ * re-advertisement of {@link ChildRegistry.linkedChildren} in `gossipRound` (one frame per
1674
+ * `T_willingness_heartbeat`, see there).
1566
1675
  */
1567
1676
  interface ChildRegistry {
1568
1677
  /** Link the child; returns true iff this advanced the state (a fresh link or a strictly-newer effectiveAt). */
@@ -1570,13 +1679,45 @@ interface ChildRegistry {
1570
1679
  /** Release the child (linked = false); returns true iff this advanced the state. A never-seen child writes a tombstone. */
1571
1680
  unrecordChild(topicId: Uint8Array, childCohortCoord: Uint8Array, effectiveAt: number): boolean;
1572
1681
  count(topicId: Uint8Array): number;
1682
+ /**
1683
+ * Every currently-linked child across **all** topics (released children — tombstones — excluded), each at
1684
+ * the `effectiveAt` that linked it. Deliberately not per-topic: `gossipRound` iterates only *resident*
1685
+ * topics (ones this engine holds records or forwarders for), and a parent can hold child links for a topic
1686
+ * whose participants sharded entirely down to the children, so a per-topic accessor would miss exactly the
1687
+ * parents that matter.
1688
+ */
1689
+ linkedChildren(): readonly LinkedChild[];
1690
+ /**
1691
+ * Whether any child is currently linked (tombstones excluded) — the same predicate as
1692
+ * `linkedChildren().length > 0`, but short-circuiting and allocation-free. The eviction census
1693
+ * ({@link CoordEngine.liveness}) runs it once per resident engine per eviction, where materializing every
1694
+ * linked child only to read `.length` is pure garbage.
1695
+ */
1696
+ hasLinkedChildren(): boolean;
1697
+ }
1698
+
1699
+ /**
1700
+ * One linked child cohort, as re-advertised by the throttled child-set resync round. The two byte fields
1701
+ * alias the registry's own copies (never clone-on-read), so a consumer must treat them as read-only.
1702
+ */
1703
+ interface LinkedChild {
1704
+ readonly topicId: Uint8Array;
1705
+ readonly childCohortCoord: Uint8Array;
1706
+ readonly effectiveAt: number;
1573
1707
  }
1574
1708
 
1575
1709
  interface ChildEntry {
1576
1710
  linked: boolean;
1577
1711
  lastEffectiveAt: number;
1712
+ /** The key's original bytes, retained on first insert (both are in hand in `apply`) so `linkedChildren` needs no key decode. */
1713
+ topicId: Uint8Array;
1714
+ childCohortCoord: Uint8Array;
1578
1715
  }
1579
1716
 
1717
+ // NOTE: released children are kept forever as `linked = false` tombstones (the high-water that stops a stale
1718
+ // link resurrecting them), so a long-lived engine's map grows with child churn. They cost memory only, never
1719
+ // traffic — `linkedChildren` excludes them, so re-advertisement volume is bounded by the *live* child count.
1720
+ // If child churn on a long-lived parent ever makes this map large, age tombstones out past a demotion horizon.
1580
1721
  function createChildRegistry(): ChildRegistry {
1581
1722
  const byTopic = new Map<string, Map<string, ChildEntry>>();
1582
1723
  // Freshness-ordered write shared by link/unlink: apply `linked` only if `effectiveAt` is strictly newer than
@@ -1594,7 +1735,7 @@ function createChildRegistry(): ChildRegistry {
1594
1735
  if (existing === undefined) {
1595
1736
  // A never-seen unlink writes a `linked = false` tombstone (never a negative count), so a later stale
1596
1737
  // link with an earlier effectiveAt cannot resurrect a demoted child.
1597
- children.set(childKey, { linked, lastEffectiveAt: effectiveAt });
1738
+ children.set(childKey, { linked, lastEffectiveAt: effectiveAt, topicId, childCohortCoord });
1598
1739
  return true;
1599
1740
  }
1600
1741
  if (effectiveAt <= existing.lastEffectiveAt) {
@@ -1625,6 +1766,27 @@ function createChildRegistry(): ChildRegistry {
1625
1766
  }
1626
1767
  return n;
1627
1768
  },
1769
+ hasLinkedChildren(): boolean {
1770
+ for (const children of byTopic.values()) {
1771
+ for (const entry of children.values()) {
1772
+ if (entry.linked) {
1773
+ return true;
1774
+ }
1775
+ }
1776
+ }
1777
+ return false;
1778
+ },
1779
+ linkedChildren(): readonly LinkedChild[] {
1780
+ const out: LinkedChild[] = [];
1781
+ for (const children of byTopic.values()) {
1782
+ for (const entry of children.values()) {
1783
+ if (entry.linked) {
1784
+ out.push({ topicId: entry.topicId, childCohortCoord: entry.childCohortCoord, effectiveAt: entry.lastEffectiveAt });
1785
+ }
1786
+ }
1787
+ }
1788
+ return out;
1789
+ },
1628
1790
  };
1629
1791
  }
1630
1792
 
@@ -1788,6 +1950,16 @@ function createCoordEngine(ctx: CoordEngineContext, servedCoord: RingCoord, tree
1788
1950
  }
1789
1951
  };
1790
1952
 
1953
+ // Set by `close()` (registry eviction, or host `stop()`); makes every time-driven entry point below inert.
1954
+ // The gossip-cadence driver iterates a `registry.all()` SNAPSHOT and awaits between engines, so an inbound
1955
+ // register / child-link / cold-sibling frame landing in one of those awaits can evict — and close — an
1956
+ // engine the tick is still walking. Driving a closed engine afterwards is not merely wasted work: a
1957
+ // `pumpMembership` would publish a cert and re-run `onCertPublished` → `verifier.cache`, re-locking the very
1958
+ // coord `onEngineEvicted` → `verifier.forget` had just released, for a coord this node no longer serves.
1959
+ // Guarding here rather than in the driver keeps the invariant total: a closed engine does nothing, whoever
1960
+ // still holds a reference to it.
1961
+ let closed = false;
1962
+
1791
1963
  /**
1792
1964
  * Publish (or refresh) this cohort's membership cert, attaching a rotation attestation when the cohort
1793
1965
  * identity (epoch) changed since the last publish. `refresh` selects the publisher path: `false` for a
@@ -1798,7 +1970,7 @@ function createCoordEngine(ctx: CoordEngineContext, servedCoord: RingCoord, tree
1798
1970
  * tick. Key-less interim mode no-ops (the verify-only signer cannot assemble).
1799
1971
  */
1800
1972
  const publishMembership = async (now: number, refresh: boolean): Promise<MembershipCertV1 | undefined> => {
1801
- if (!canPublish) {
1973
+ if (closed || !canPublish) {
1802
1974
  return undefined;
1803
1975
  }
1804
1976
  const snapshot = snapshotAt(now); // also observes the current identity (snapshotAt → cohort())
@@ -1845,6 +2017,11 @@ function createCoordEngine(ctx: CoordEngineContext, servedCoord: RingCoord, tree
1845
2017
  cohortCoord: (): Uint8Array => servedCoord,
1846
2018
  cohortEpoch: localEpoch,
1847
2019
  signer: noticeSigner,
2020
+ // Seed a topic's replay ordering + direction from the node-level adopted-transition record (the
2021
+ // promote gate's map), which outlives this engine — so recreating an evicted engine neither reopens
2022
+ // the replay window nor forgets a correct promoted mode. For a notice at this served coord both a
2023
+ // promotion's `fromTier` and a demotion's `tier` equal `treeTier`, so this key matches both writes.
2024
+ seedTransition: (topicId: Uint8Array): AdoptedTransition | undefined => ctx.adoptedTransition?.(servedCoord, treeTier, topicId),
1848
2025
  // Production defaults (cap_promote = 64, …) unless the host was given a promotion override — the
1849
2026
  // live-tier e2e lowers `capPromote` to drive promotion with a small participant count. The
1850
2027
  // coord-derived inputs above (treeTier / childCohortCount / parentCoord) are never overridden.
@@ -1969,11 +2146,19 @@ function createCoordEngine(ctx: CoordEngineContext, servedCoord: RingCoord, tree
1969
2146
  // node ever serves very many idle cohorts, batch the heartbeats or lengthen the interval.
1970
2147
  let lastGossipAt: number | undefined;
1971
2148
 
2149
+ // Timestamp of the last round in which this engine re-advertised its linked child set, or `undefined` if it
2150
+ // never has. Bounds child-set resync to one frame per `T_willingness_heartbeat`, not one per gossip round.
2151
+ let lastChildReadvertAt: number | undefined;
2152
+
1972
2153
  // One gossip round: sweep stale records (firing the `evicted` deltas), freeze each resident topic's
1973
- // traffic summary, drain the touch/evicted deltas, then assemble + sign + broadcast the frame. An idle
1974
- // engine (no topics, no deltas) normally builds no frame — except a willingness heartbeat, where an idle
1975
- // but willing engine still emits a willingness/load-only frame so a cold cohort can bootstrap.
2154
+ // traffic summary, re-advertise the linked child set on its throttle, drain the touch/evicted/child deltas,
2155
+ // then assemble + sign + broadcast the frame. An idle engine (no topics, no deltas) normally builds no
2156
+ // frame — except a willingness heartbeat, where an idle but willing engine still emits a willingness/load-
2157
+ // only frame so a cold cohort can bootstrap.
1976
2158
  const gossipRound = async (now: number): Promise<CohortGossipV1 | undefined> => {
2159
+ if (closed) {
2160
+ return undefined;
2161
+ }
1977
2162
  engine.sweepStale(now);
1978
2163
  const topicSummaries = residentTopics().map((topicId) =>
1979
2164
  toCohortTopicSummary(topicId, traffic.publish(topicId, now), {
@@ -1986,6 +2171,34 @@ function createCoordEngine(ctx: CoordEngineContext, servedCoord: RingCoord, tree
1986
2171
  childCohortCount: childRegistry.count(topicId),
1987
2172
  }),
1988
2173
  );
2174
+ // Child-set resync. A link/unlink delta is broadcast exactly once, so a member whose engine was
2175
+ // instantiated after a link drained (a parent rotation, or any FRET reshuffle adding a node to the
2176
+ // parent cohort) would read `childCohortCount == 0` for children that plainly exist — wrongly clearing
2177
+ // the demotion gate and telling seekers the topic is not promoted. So every `willingnessHeartbeatMs`
2178
+ // re-emit the *currently-linked* set at each child's original `effectiveAt`. Safe by construction: the
2179
+ // receiving merge is last-writer-wins on `effectiveAt`, so a re-advertisement is a no-op on a member
2180
+ // that already holds it and cannot resurrect a child released by a newer unlink. Queued through
2181
+ // `pending` (not straight into the frame) so a same-round *local* link for the same child collapses to
2182
+ // one ref instead of two (`queueChild` is last-writer-wins per `(topic, child)`). Sourcing from the
2183
+ // registry rather than the queue is what makes the unlink direction safe: an already-released child is
2184
+ // simply absent from `linkedChildren()`, and nothing between here and `drain()` awaits, so a
2185
+ // re-advertisement can never race a same-round unlink. Bounded to one extra frame per engine per
2186
+ // heartbeat, and the round is deliberately non-idle — the frame is the carrier.
2187
+ // NOTE: this heals a missed *link*, never a missed *unlink* — absence is not advertised, so a member
2188
+ // that dropped an unlink frame over-counts until it hears another delta for that child. Both consumers
2189
+ // fail conservatively there (demotion stays blocked; a seeker sweeps a no-longer-promoted tier —
2190
+ // wasteful, not wrong). If over-counting ever needs healing too, advertise the tombstones as well.
2191
+ if (lastChildReadvertAt === undefined || now - lastChildReadvertAt >= ctx.willingnessHeartbeatMs) {
2192
+ const linked = childRegistry.linkedChildren();
2193
+ for (const child of linked) {
2194
+ pending.childLink(child.topicId, child.childCohortCoord, child.effectiveAt);
2195
+ }
2196
+ // Only start the clock once something was actually enqueued: an engine that parents nothing stays
2197
+ // silent and re-advertises immediately when it first does hold a child.
2198
+ if (linked.length > 0) {
2199
+ lastChildReadvertAt = now;
2200
+ }
2201
+ }
1989
2202
  const { records, evicted, childLinks, childUnlinks } = pending.drain();
1990
2203
  const idle = topicSummaries.length === 0 && records.length === 0 && evicted.length === 0
1991
2204
  && childLinks.length === 0 && childUnlinks.length === 0;
@@ -2021,7 +2234,7 @@ function createCoordEngine(ctx: CoordEngineContext, servedCoord: RingCoord, tree
2021
2234
  // parent coord) via the same path a promotion uses. Skipped without a key (verify-only signer can't
2022
2235
  // assemble); for the single-cohort tier-0 milestone the lifecycle never demotes (the root has no parent).
2023
2236
  const demotionTick = async (now: number): Promise<void> => {
2024
- if (!canPublish) {
2237
+ if (closed || !canPublish) {
2025
2238
  return;
2026
2239
  }
2027
2240
  for (const topicId of residentTopics()) {
@@ -2038,14 +2251,40 @@ function createCoordEngine(ctx: CoordEngineContext, servedCoord: RingCoord, tree
2038
2251
  }
2039
2252
  };
2040
2253
 
2254
+ // The exhaustive engine-owned-state census the registry's eviction ranking consults. One entry per
2255
+ // EngineStateKind — the total-Record type makes forgetting a new kind here a compile error. `children`
2256
+ // derives from the *linked* set (tombstones excluded), so a parent whose every child demoted away is
2257
+ // reclaimable again; `promotion` counts only adopted transition state (a `promoted` flag or a
2258
+ // `lastEffectiveAt` high-water — growth samples alone are rebuilt from the store and do not count).
2259
+ // NOTE: `evictOne` runs this once per resident engine per eviction — up to `coordEnginesMax` (2048)
2260
+ // censuses for one reclaim. Three of the four arms short-circuit and allocate nothing; `records` still
2261
+ // materializes every registration via `store.listAll()` only to read `.length` (pre-dating this census —
2262
+ // the old idle predicate did the same). Unmeasured, and bounded by a registry that is full of *cold*
2263
+ // engines in the case that matters. If eviction ever shows up in a profile, give `RegistrationStore` an
2264
+ // `isEmpty()` (its `byTopic` map already knows) and read that here.
2265
+ const liveness = (): EngineLiveness => ({
2266
+ records: store.listAll().length > 0,
2267
+ forwarders: coldStart.hasForwarders(),
2268
+ children: childRegistry.hasLinkedChildren(),
2269
+ promotion: promotion.hasAdoptedState(),
2270
+ });
2271
+
2041
2272
  return {
2042
2273
  servedCoord,
2043
2274
  treeTier,
2044
2275
  engine,
2045
2276
  cohort,
2046
2277
  cohortIdentityAt: (epoch: Uint8Array): readonly string[] | undefined => rotationState.membersAt(bytesToB64url(epoch)),
2047
- hasState: (): boolean => store.listAll().length > 0,
2048
- hasForwarders: (): boolean => coldStart.hasForwarders(),
2278
+ liveness,
2279
+ // Derivations of `liveness()` kept on the interface for existing callers/specs, but sourced from
2280
+ // the same census so the two views cannot drift.
2281
+ // NOTE: each call runs the FULL census (allocating one record, and touching the child registry and
2282
+ // promotion lifecycle) where it used to be a bare `store.listAll()` check. Free today: as of this
2283
+ // writing nothing in `src/` calls either accessor — the registry ranks via `liveness()` directly and
2284
+ // the only callers are specs. If a hot path (a per-frame dispatch gate, say) ever starts calling one
2285
+ // per message, give it a direct single-field read instead of routing it through the census.
2286
+ hasState: (): boolean => liveness().records,
2287
+ hasForwarders: (): boolean => liveness().forwarders,
2049
2288
  holds: (topicId: Uint8Array, participantId: Uint8Array): boolean =>
2050
2289
  store.getByParticipant(topicId, participantId) !== undefined,
2051
2290
  records: (topicId: Uint8Array): readonly RegistrationRecord[] => store.listByTopic(topicId),
@@ -2088,7 +2327,10 @@ function createCoordEngine(ctx: CoordEngineContext, servedCoord: RingCoord, tree
2088
2327
  pumpMembership: (now: number): Promise<MembershipCertV1 | undefined> => publishMembership(now, true),
2089
2328
  gossipRound,
2090
2329
  demotionTick,
2091
- close: (): void => bus.close(),
2330
+ close: (): void => {
2331
+ closed = true;
2332
+ bus.close();
2333
+ },
2092
2334
  };
2093
2335
  }
2094
2336
 
@@ -2398,14 +2640,25 @@ export type NoticeOutcome = "applied" | "untrusted" | "dropped";
2398
2640
  *
2399
2641
  * - `"undecodable"` — the frame is neither a promotion nor a demotion notice.
2400
2642
  * - `"rate-limited"` — the dialing `(peer, topic)` is over its `register_rate_per_peer` ceiling.
2401
- * - `"stale"` — the notice's `effectiveAt` is at or below the last *applied* notice for its served
2402
- * cohort coord (a replay / out-of-order frame); dropped before `verifyMessage`.
2643
+ * - `"stale"` — the notice's `effectiveAt` is at or below the last *adopted* transition recorded for
2644
+ * its `(cohortCoord, tier, topicId)` (a replay / out-of-order frame); dropped before `verifyMessage`.
2403
2645
  * - `"unlinked"` — a demotion notice that did not sibling-adopt on this node (no local child-coord engine,
2404
2646
  * or that path was stale) but **did** verify + unrecord the demoting child at its parent cohort here (the
2405
2647
  * parent-unlink path). Distinct from `"applied"` (a sibling-adopt) so a test can assert the parent-only case.
2406
2648
  */
2407
2649
  export type InboundNoticeResult = NoticeOutcome | "undecodable" | "rate-limited" | "stale" | "unlinked";
2408
2650
 
2651
+ /**
2652
+ * The last promotion/demotion transition this node adopted for one `(coord, tier, topic)`: its
2653
+ * `effectiveAt` (the replay ordering) and its direction (`promoted` — what the transition left the cohort
2654
+ * as). Stored in {@link PromoteGate.transitions}; direction matters because a recreated engine must
2655
+ * restore *which way* the last transition went, not just how recent it was.
2656
+ */
2657
+ export interface AdoptedTransition {
2658
+ readonly effectiveAt: number;
2659
+ readonly promoted: boolean;
2660
+ }
2661
+
2409
2662
  /**
2410
2663
  * Node-level anti-abuse state for the `promote` handler (`cohort-topic-promote-handler-verify-amplification`).
2411
2664
  * The handler is node-level (one per node, not per coord), so unlike the per-{@link CoordEngine} register-path
@@ -2418,35 +2671,85 @@ export interface PromoteGate {
2418
2671
  */
2419
2672
  readonly rateLimiter: RegisterRateLimiter;
2420
2673
  /**
2421
- * Per-served-coord high-water (key: `` `${cohortCoord}|${tier}` ``) of the last *applied* notice's
2422
- * `effectiveAt`. A notice at or below the water is a replay / out-of-order frame and is dropped before
2423
- * verification. Keyed by the served coord — not `(topic, tier)` so two sibling cohorts a node serves for
2424
- * one `(topic, tier)` do not share an entry (an applied notice for one must not stale-drop a legitimate
2425
- * notice for the other). Updated **only** on an `"applied"` outcome (never on an unverified frame), so a
2426
- * forged notice carrying `effectiveAt = Infinity` cannot poison the water and lock out legitimate notices.
2674
+ * The node's durable record of the last adopted promotion/demotion transition per
2675
+ * `(cohortCoord, tier, topicId)` (key: {@link transitionKey}). Written by **every** adopt path
2676
+ * a verified inbound apply ({@link handleInboundNotice}) *and* a locally-originated broadcast (the
2677
+ * host's `broadcastNotice`, which `broadcastOver` never echoes back to self) via
2678
+ * {@link recordAdoptedTransition}, and read by two consumers: the inbound stale gate (a notice at or
2679
+ * below the recorded `effectiveAt` is dropped before verification) and engine seeding (a freshly
2680
+ * created {@link CoordEngine}'s {@link PromotionLifecycle} initializes its per-topic ordering and
2681
+ * direction from here, via `CoordEngineContext.adoptedTransition` → `PromotionDeps.seedTransition`).
2427
2682
  *
2428
- * **Bounded.** An {@link LruMap} capped at {@link PROMOTE_HIGHWATER_MAX_KEYS} so the retain-forever shape
2429
- * cannot leak on a long-lived node. Unlike the limiter this is *not* attacker-growable (it is written only
2430
- * on an `"applied"` outcome, which needs a verified `≥ minSigs` cohort signature reads of forged
2431
- * `topicId`s via `.get` create nothing), so it never evicts under legitimate load; the cap is the
2432
- * belt-and-suspenders bound. Evicting an entry is safe: the engine's {@link PromotionLifecycle} is
2433
- * independently idempotent and `effectiveAt`-ordered (`PromotionState.lastEffectiveAt`), so an
2434
- * evicted-then-replayed older notice re-verifies (one bounded, rate-capped `verifyMessage`) and then
2435
- * **no-ops at the engine** rather than (re-)applying. Water absence only *opens* the gate, never closes it.
2683
+ * This map not the engine is the node's replay-ordering authority: the engine's
2684
+ * `PromotionState.lastEffectiveAt` is a same-process second layer that is discarded whenever the
2685
+ * registry evicts the engine under memory pressure, while this map outlives every engine. Keying
2686
+ * includes the coord so two sibling cohorts a node serves for one `(topic, tier)` never share an
2687
+ * entry, and the topic so two topics at one coord order independently.
2688
+ *
2689
+ * **Bounded.** An {@link LruMap} capped at {@link PROMOTE_TRANSITIONS_MAX_KEYS}, so the retain-forever
2690
+ * shape cannot leak on a long-lived node. Not attacker-growable: it is written only on adopted
2691
+ * transitions — a verified `≥ minSigs` cohort signature, or a locally threshold-signed notice — and
2692
+ * reads of forged keys via `.get` create nothing. Both consumers read through `.get`, which refreshes
2693
+ * recency, so an actively-transitioning `(coord, tier, topic)` stays resident. Record absence only
2694
+ * *opens* the stale gate, never closes it: losing an entry while its engine is still resident costs
2695
+ * one re-verify (the engine's `lastEffectiveAt` still no-ops the replay), and only losing BOTH the
2696
+ * entry and the engine reopens the replay window.
2436
2697
  */
2437
- readonly highWater: LruMap<string, number>;
2698
+ readonly transitions: LruMap<string, AdoptedTransition>;
2438
2699
  }
2439
2700
 
2440
2701
  /**
2441
- * Hard cap on tracked per-served-coord high-water entries; the least-recently-touched are evicted beyond
2442
- * this. A modest bound is plenty — only verified applies grow the map, so it never evicts under legitimate
2443
- * load — but it caps the otherwise retain-forever shape on a long-lived node.
2702
+ * Hard cap on tracked adopted-transition entries; the least-recently-touched are evicted beyond this. A
2703
+ * modest bound is plenty — only adopted transitions grow the map but it caps the otherwise
2704
+ * retain-forever shape on a long-lived node.
2705
+ *
2706
+ * NOTE: this counts `(coord, tier, topic)` triples, NOT engines, so it is not a per-engine guarantee: a
2707
+ * node at the {@link DEFAULT_COORD_ENGINES_MAX} (2048) registry cap that has transitioned more than ~4
2708
+ * topics per coord can overflow it, and the least-recently-touched entries go first. That is safe while
2709
+ * the owning engine is resident (its `lastEffectiveAt` still orders replays); if a node ever runs that
2710
+ * wide AND evicts engines, raise this cap rather than relying on the engine layer.
2444
2711
  */
2445
- export const PROMOTE_HIGHWATER_MAX_KEYS = 8192;
2712
+ export const PROMOTE_TRANSITIONS_MAX_KEYS = 8192;
2446
2713
 
2447
2714
  /** Build the default {@link PromoteGate} from the (optional) anti-DoS rate-limiter config. */
2448
2715
  export function createPromoteGate(rateLimiterConfig?: RegisterRateLimiterConfig): PromoteGate {
2449
- return { rateLimiter: createRegisterRateLimiter(rateLimiterConfig), highWater: new LruMap<string, number>(PROMOTE_HIGHWATER_MAX_KEYS) };
2716
+ return { rateLimiter: createRegisterRateLimiter(rateLimiterConfig), transitions: new LruMap<string, AdoptedTransition>(PROMOTE_TRANSITIONS_MAX_KEYS) };
2717
+ }
2718
+
2719
+ /** The {@link PromoteGate.transitions} key for one `(cohortCoord, tier, topicId)` (all b64url-encoded parts). */
2720
+ export function transitionKey(cohortCoordB64: string, tier: number, topicIdB64: string): string {
2721
+ return `${cohortCoordB64}|${tier}|${topicIdB64}`;
2722
+ }
2723
+
2724
+ /**
2725
+ * The {@link transitionKey} a notice's own fields address: the served `cohortCoord` it was decided at, the
2726
+ * tier the deciding cohort serves the topic at (`fromTier` on a promotion, `tier` on a demotion — both the
2727
+ * engine's own tree tier for a notice at its served coord), and the topic. A demotion also fans to the
2728
+ * parent coord, but that parent-unlink path is deliberately outside this ordering — see
2729
+ * {@link applyDemotionUnlinkAtParent}.
2730
+ */
2731
+ export function noticeTransitionKey(notice: PromotionNoticeV1 | DemotionNoticeV1): string {
2732
+ return transitionKey(notice.cohortCoord, "parentCohortCoord" in notice ? notice.tier : notice.fromTier, notice.topicId);
2733
+ }
2734
+
2735
+ /**
2736
+ * Record `notice` as the last adopted transition for its `(coord, tier, topic)` — monotonic per key (an
2737
+ * older or equal `effectiveAt` never overwrites), with the direction taken from the notice's shape (a
2738
+ * promotion sets `promoted = true`, a demotion `false`). Called from BOTH adopt paths: the verified
2739
+ * inbound apply in {@link handleInboundNotice} and the host's `broadcastNotice` origination seam.
2740
+ */
2741
+ export function recordAdoptedTransition(gate: PromoteGate, notice: PromotionNoticeV1 | DemotionNoticeV1): void {
2742
+ // NOTE: this write is monotonic but `promote()` / `demote()` origination is not — the lifecycle stamps
2743
+ // `effectiveAt = now` without consulting its own `lastEffectiveAt`. A backwards local clock can therefore
2744
+ // originate a transition the record refuses, leaving the record's direction stale relative to the engine
2745
+ // until the next forward-clock transition. Harmless today (the wall clock only skews backwards on an NTP
2746
+ // step, and the same skew already perturbs every `effectiveAt`-ordered path); if origination ever gains a
2747
+ // monotonic clock, drop the divergence by ordering `promote()` / `demote()` against `lastEffectiveAt` too.
2748
+ const key = noticeTransitionKey(notice);
2749
+ const held = gate.transitions.get(key);
2750
+ if (held === undefined || notice.effectiveAt > held.effectiveAt) {
2751
+ gate.transitions.set(key, { effectiveAt: notice.effectiveAt, promoted: !("parentCohortCoord" in notice) });
2752
+ }
2450
2753
  }
2451
2754
 
2452
2755
  /**
@@ -2565,11 +2868,12 @@ export type ParentUnlinkOutcome = "unlinked" | "no-parent" | "untrusted";
2565
2868
  * - `"unlinked"` — verified; the child was unrecorded (or was already released — the child registry's own
2566
2869
  * per-`(topic, childCoord)` freshness makes a replay an idempotent no-op).
2567
2870
  *
2568
- * **Freshness is the child registry's, not the promote-gate high-water.** The sibling-adopt high-water is keyed
2569
- * by the child coord and advanced only on a sibling-adopt `"applied"`; the unlink is ordered independently by
2570
- * the child registry's per-child `lastEffectiveAt`, so a demotion that is a stale no-op for the sibling-adopt
2571
- * target still applies the unlink at the parent, and vice-versa. The verify carries the same
2572
- * {@link PROMOTE_REFETCH_MIN_INTERVAL_MS} bound as the sibling-adopt, so it cannot amplify into dials.
2871
+ * **Freshness is the child registry's, not the promote-gate transition record.** The sibling-adopt record is
2872
+ * keyed by the child coord and, on the inbound path, advanced only on a sibling-adopt `"applied"`; the unlink
2873
+ * is ordered independently by the child registry's per-child `lastEffectiveAt`, so a demotion that is a stale
2874
+ * no-op for the sibling-adopt target still applies the unlink at the parent, and vice-versa. The verify
2875
+ * carries the same {@link PROMOTE_REFETCH_MIN_INTERVAL_MS} bound as the sibling-adopt, so it cannot amplify
2876
+ * into dials.
2573
2877
  */
2574
2878
  export async function applyDemotionUnlinkAtParent(
2575
2879
  notice: DemotionNoticeV1,
@@ -2608,16 +2912,17 @@ export async function applyDemotionUnlinkAtParent(
2608
2912
  * work:
2609
2913
  *
2610
2914
  * ```
2611
- * decode → per-(peer,topic) rate limit → resolve engine by carried cohortCoord → effectiveAt high-water → verify+apply
2612
- * ↘ (demotion) parent-unlink at parentCohortCoord
2915
+ * decode → per-(peer,topic) rate limit → resolve engine by carried cohortCoord → adopted-transition stale gate → verify+apply
2916
+ * ↘ (demotion) parent-unlink at parentCohortCoord
2613
2917
  * ```
2614
2918
  *
2615
2919
  * A **demotion** carries a second, independent apply semantics: beyond the sibling-adopt above, it also
2616
2920
  * releases the demoting child at its parent cohort ({@link applyDemotionUnlinkAtParent}). Both paths may fire
2617
2921
  * on one node (one that serves both the child coord and the parent coord). The parent-unlink runs OUTSIDE the
2618
- * sibling-adopt high-water (which is keyed by the child coord and would otherwise stale-drop the parent-coord
2619
- * frame after the child-coord frame advanced it); its freshness is the child registry's own per-child key.
2620
- * `"unlinked"` is returned when the unlink fired but the sibling-adopt did not (a parent-only node).
2922
+ * sibling-adopt transition record (which is keyed by the child coord and would otherwise stale-drop the
2923
+ * parent-coord frame after the child-coord frame advanced it); its freshness is the child registry's own
2924
+ * per-child key. `"unlinked"` is returned when the unlink fired but the sibling-adopt did not (a parent-only
2925
+ * node).
2621
2926
  *
2622
2927
  * - **Rate limit** (`gate.rateLimiter`) keys on `(from, topicId)`; an over-rate peer is dropped before the
2623
2928
  * coord lookup and the verify, so a peer cannot amplify junk into verify/network work.
@@ -2625,10 +2930,11 @@ export async function applyDemotionUnlinkAtParent(
2625
2930
  * notice was decided for, covered by its signature. A node serving several sibling cohorts for one
2626
2931
  * `(topic, tier)` applies the notice to the cohort that produced it, never a first-match `(topic, tier)`
2627
2932
  * scan; a coord this node does not serve is dropped.
2628
- * - **High-water** (`gate.highWater`, keyed per served `cohortCoord`) drops a notice whose `effectiveAt` is
2629
- * at or below the last *applied* one — a replay / out-of-order frame — before `verifyMessage`. It is
2630
- * advanced **only** on an `"applied"` outcome, so a forged frame (which never verifies) cannot poison it.
2631
- * Keying by coord (not `(topic, tier)`) keeps two sibling cohorts on one node from sharing a water.
2933
+ * - **Stale gate** (`gate.transitions`, keyed per `(cohortCoord, tier, topicId)`) drops a notice whose
2934
+ * `effectiveAt` is at or below the last *adopted* transition — a replay / out-of-order frame — before
2935
+ * `verifyMessage`. On this inbound path it advances **only** on an `"applied"` outcome, so a forged frame
2936
+ * (which never verifies) cannot poison it. Keying by coord keeps two sibling cohorts on one node from
2937
+ * sharing an entry; keying by topic keeps two topics at one coord ordering independently.
2632
2938
  * - The receiver-side `cohortEpoch` is intentionally **not** gated on: the epoch rotates on every
2633
2939
  * membership change, so a legitimately in-flight notice can briefly carry the prior epoch right after a
2634
2940
  * rotation — making an epoch check a brittle, false-positive-prone filter. The rate limiter + high-water
@@ -2673,20 +2979,19 @@ export async function handleInboundNotice(
2673
2979
  if (target === undefined) {
2674
2980
  siblingOutcome = "dropped";
2675
2981
  } else {
2676
- // Freshness / replay gate: drop an at-or-below-high-water notice before the expensive verify. Keyed by the
2677
- // served coord (which uniquely identifies the cohort) so two sibling cohorts on one node do not share a
2678
- // high-water an applied notice for cohort A must not stale-drop a legitimate cohort-B notice. `tier` is
2679
- // kept in the key only for readability.
2680
- const waterKey = `${inbound.notice.cohortCoord}|${tier}`;
2681
- const water = gate.highWater.get(waterKey);
2682
- if (water !== undefined && inbound.notice.effectiveAt <= water) {
2683
- log("promote: stale %s notice for topic %s tier %d (effectiveAt %d <= high-water %d)", inbound.kind, inbound.notice.topicId, tier, inbound.notice.effectiveAt, water);
2982
+ // Freshness / replay gate: drop a notice at or below the last adopted transition for its
2983
+ // `(coord, tier, topic)` before the expensive verify. Keyed per coord (so two sibling cohorts on one
2984
+ // node never share an entry) AND per topic (so an applied notice for topic A never stale-drops a
2985
+ // legitimate topic-B notice at the same coord).
2986
+ const held = gate.transitions.get(noticeTransitionKey(inbound.notice));
2987
+ if (held !== undefined && inbound.notice.effectiveAt <= held.effectiveAt) {
2988
+ log("promote: stale %s notice for topic %s tier %d (effectiveAt %d <= last adopted %d)", inbound.kind, inbound.notice.topicId, tier, inbound.notice.effectiveAt, held.effectiveAt);
2684
2989
  siblingOutcome = "stale";
2685
2990
  } else {
2686
2991
  siblingOutcome = await verifyAndApplyNotice(inbound, target, verifier, now);
2687
2992
  if (siblingOutcome === "applied") {
2688
- // Advance the high-water only on a *verified-and-applied* notice, so a forged frame cannot poison it.
2689
- gate.highWater.set(waterKey, inbound.notice.effectiveAt);
2993
+ // Record only a *verified-and-applied* notice, so a forged frame cannot poison the ordering.
2994
+ recordAdoptedTransition(gate, inbound.notice);
2690
2995
  } else {
2691
2996
  log("promote: %s %s notice for topic %s tier %d", siblingOutcome, inbound.kind, inbound.notice.topicId, tier);
2692
2997
  }
@@ -2699,12 +3004,12 @@ export async function handleInboundNotice(
2699
3004
 
2700
3005
  // --- Parent-unlink path (demotion only): additionally release the demoting child at its parent cohort. ---
2701
3006
  // A demotion is fanned to BOTH the child coord (sibling-adopt, above) and the parent coord (this unlink),
2702
- // arriving as two independent frames. This path is deliberately OUTSIDE the sibling-adopt high-water: that
2703
- // water is keyed by the child coord and advanced only on a sibling-adopt apply, so — on a node serving both
2704
- // coords — the child-coord frame would advance the water and stale-drop the parent-coord frame before it
2705
- // could unrecord. The child registry's own per-`(topic, childCoord)` freshness orders the unlink instead, so
2706
- // a replay is an idempotent no-op. The verify is against the child cohort cert (same as the sibling-adopt),
2707
- // so a forged demotion cannot unrecord.
3007
+ // arriving as two independent frames. This path is deliberately OUTSIDE the sibling-adopt transition
3008
+ // record: that record is keyed by the child coord and advanced on a sibling-adopt apply, so — on a node
3009
+ // serving both coords — the child-coord frame would advance it and stale-drop the parent-coord frame before
3010
+ // it could unrecord. The child registry's own per-`(topic, childCoord)` freshness orders the unlink instead,
3011
+ // so a replay is an idempotent no-op. The verify is against the child cohort cert (same as the
3012
+ // sibling-adopt), so a forged demotion cannot unrecord.
2708
3013
  const unlink = await applyDemotionUnlinkAtParent(inbound.notice, registry, verifier, now);
2709
3014
  if (siblingOutcome === "applied") {
2710
3015
  return "applied"; // a node serving both coords: the sibling-adopt is the primary reported outcome