@optimystic/db-p2p 0.21.0 → 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/dist/src/cluster/cluster-policy.d.ts +13 -2
  2. package/dist/src/cluster/cluster-policy.d.ts.map +1 -1
  3. package/dist/src/cluster/cluster-policy.js +51 -4
  4. package/dist/src/cluster/cluster-policy.js.map +1 -1
  5. package/dist/src/cluster/cluster-repo.d.ts +3 -3
  6. package/dist/src/cluster/cluster-repo.js +3 -3
  7. package/dist/src/cluster/cluster-size-coupling.d.ts +28 -0
  8. package/dist/src/cluster/cluster-size-coupling.d.ts.map +1 -0
  9. package/dist/src/cluster/cluster-size-coupling.js +35 -0
  10. package/dist/src/cluster/cluster-size-coupling.js.map +1 -0
  11. package/dist/src/cluster/quorum-restore.d.ts +6 -0
  12. package/dist/src/cluster/quorum-restore.d.ts.map +1 -1
  13. package/dist/src/cluster/quorum-restore.js +1 -1
  14. package/dist/src/cluster/quorum-restore.js.map +1 -1
  15. package/dist/src/cluster/reconcile-block.d.ts.map +1 -1
  16. package/dist/src/cluster/reconcile-block.js +15 -3
  17. package/dist/src/cluster/reconcile-block.js.map +1 -1
  18. package/dist/src/index.d.ts +2 -0
  19. package/dist/src/index.d.ts.map +1 -1
  20. package/dist/src/index.js +2 -0
  21. package/dist/src/index.js.map +1 -1
  22. package/dist/src/libp2p-key-network.d.ts +46 -5
  23. package/dist/src/libp2p-key-network.d.ts.map +1 -1
  24. package/dist/src/libp2p-key-network.js +40 -9
  25. package/dist/src/libp2p-key-network.js.map +1 -1
  26. package/dist/src/libp2p-node-base.d.ts +3 -2
  27. package/dist/src/libp2p-node-base.d.ts.map +1 -1
  28. package/dist/src/libp2p-node-base.js +834 -777
  29. package/dist/src/libp2p-node-base.js.map +1 -1
  30. package/dist/src/libp2p-node-rn.d.ts +2 -2
  31. package/dist/src/libp2p-node-rn.d.ts.map +1 -1
  32. package/dist/src/libp2p-node-rn.js.map +1 -1
  33. package/dist/src/libp2p-node.d.ts +2 -2
  34. package/dist/src/libp2p-node.d.ts.map +1 -1
  35. package/dist/src/libp2p-node.js.map +1 -1
  36. package/dist/src/network/network-manager-service.d.ts +2 -0
  37. package/dist/src/network/network-manager-service.d.ts.map +1 -1
  38. package/dist/src/network/network-manager-service.js +4 -0
  39. package/dist/src/network/network-manager-service.js.map +1 -1
  40. package/dist/src/optimystic-node.d.ts +35 -0
  41. package/dist/src/optimystic-node.d.ts.map +1 -0
  42. package/dist/src/optimystic-node.js +2 -0
  43. package/dist/src/optimystic-node.js.map +1 -0
  44. package/dist/src/repo/coordinator-repo.d.ts +16 -5
  45. package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
  46. package/dist/src/repo/coordinator-repo.js +24 -8
  47. package/dist/src/repo/coordinator-repo.js.map +1 -1
  48. package/dist/src/rn.d.ts +2 -0
  49. package/dist/src/rn.d.ts.map +1 -1
  50. package/dist/src/rn.js +2 -0
  51. package/dist/src/rn.js.map +1 -1
  52. package/dist/src/storage/block-storage.d.ts.map +1 -1
  53. package/dist/src/storage/block-storage.js +57 -5
  54. package/dist/src/storage/block-storage.js.map +1 -1
  55. package/dist/src/storage/i-block-storage.d.ts +20 -1
  56. package/dist/src/storage/i-block-storage.d.ts.map +1 -1
  57. package/dist/src/storage/storage-repo.d.ts +56 -3
  58. package/dist/src/storage/storage-repo.d.ts.map +1 -1
  59. package/dist/src/storage/storage-repo.js +124 -18
  60. package/dist/src/storage/storage-repo.js.map +1 -1
  61. package/dist/src/testing/raw-storage-conformance.d.ts.map +1 -1
  62. package/dist/src/testing/raw-storage-conformance.js +17 -0
  63. package/dist/src/testing/raw-storage-conformance.js.map +1 -1
  64. package/package.json +2 -2
  65. package/readme.md +41 -26
  66. package/src/cluster/cluster-policy.ts +55 -4
  67. package/src/cluster/cluster-repo.ts +3 -3
  68. package/src/cluster/cluster-size-coupling.ts +45 -0
  69. package/src/cluster/quorum-restore.ts +1 -1
  70. package/src/cluster/reconcile-block.ts +15 -3
  71. package/src/index.ts +2 -0
  72. package/src/libp2p-key-network.ts +41 -9
  73. package/src/libp2p-node-base.ts +907 -847
  74. package/src/libp2p-node-rn.ts +2 -2
  75. package/src/libp2p-node.ts +2 -2
  76. package/src/network/network-manager-service.ts +5 -0
  77. package/src/optimystic-node.ts +36 -0
  78. package/src/repo/coordinator-repo.ts +24 -8
  79. package/src/rn.ts +2 -0
  80. package/src/storage/block-storage.ts +59 -6
  81. package/src/storage/i-block-storage.ts +20 -1
  82. package/src/storage/storage-repo.ts +129 -18
  83. package/src/testing/raw-storage-conformance.ts +20 -0
@@ -22,9 +22,11 @@ import { seedOwnedBlocksFromStorage } from './owned-block-seed.js';
22
22
  import { clusterMember, type ReconcileBlockCallback, type CommitCertificateSink, type DeriveExpectedClusterCallback } from './cluster/cluster-repo.js';
23
23
  import { createReconcileBlock } from './cluster/reconcile-block.js';
24
24
  import { resolveClusterPolicy, type ClusterPolicyOptions } from './cluster/cluster-policy.js';
25
+ import { assertClusterSizeCoupling } from './cluster/cluster-size-coupling.js';
25
26
  import { createCommitCertStore, makeClusterCommitCertExtractor, type CommitCertStore } from './cluster/commit-cert.js';
26
27
  import { coordinatorRepo } from './repo/coordinator-repo.js';
27
28
  import { Libp2pKeyPeerNetwork, type NetworkMode, type NetworkStatePersistence } from './libp2p-key-network.js';
29
+ import type { OptimysticNode, OptimysticNodeAttachments } from './optimystic-node.js';
28
30
  import { ClusterClient } from './cluster/client.js';
29
31
  import type { IRepo, ICluster, ITransactionValidator, BlockId, IBlockChangeNotifier } from '@optimystic/db-core';
30
32
  import type { ITransactionStateStore } from './cluster/i-transaction-state-store.js';
@@ -370,7 +372,7 @@ export async function createLibp2pNodeBase(
370
372
  listenAddrs: string[];
371
373
  transports: Libp2pTransports;
372
374
  }
373
- ): Promise<Libp2p> {
375
+ ): Promise<OptimysticNode> {
374
376
  const rawStorage = resolveStorage(options.storage);
375
377
 
376
378
  // Create placeholder restore callback (will be replaced after node starts)
@@ -458,6 +460,16 @@ export async function createLibp2pNodeBase(
458
460
  ? (actionId, cert): void => { certStore.put(actionId, cert); options.onCommitCertificate?.(actionId, cert); }
459
461
  : options.onCommitCertificate;
460
462
 
463
+ // Every cluster-policy default lives in `cluster/cluster-policy.ts` — including WHY the admission
464
+ // gate and the repair corroboration floor resolve the one operator field
465
+ // (`clusterPolicy.assumedClusterSize`) to different values when it is absent. Resolved ONCE, here,
466
+ // before anything that reads a cluster size is constructed: `networkManagerService` below,
467
+ // `Libp2pKeyPeerNetwork`, and the spread-on-churn monitor init must all read `consensusConfig.clusterSize`
468
+ // rather than `options.clusterSize` directly, or they can each apply their own fallback default and
469
+ // silently disagree (ticket bug-cluster-size-resolution-single-source). `assertClusterSizeCoupling`
470
+ // below is the fail-fast backstop if a future edit reintroduces that split.
471
+ const consensusConfig = resolveClusterPolicy(options);
472
+
461
473
  const libp2pOptions: Libp2pInit = {
462
474
  start: false,
463
475
  privateKey: nodePrivateKey,
@@ -613,7 +625,7 @@ export async function createLibp2pNodeBase(
613
625
 
614
626
  networkManager: (components: any) => {
615
627
  const svcFactory = networkManagerService({
616
- clusterSize: options.clusterSize ?? 10,
628
+ clusterSize: consensusConfig.clusterSize,
617
629
  expectedRemotes: (options.bootstrapNodes?.length ?? 0) > 0,
618
630
  allowClusterDownsize: options.clusterPolicy?.allowDownsize ?? true,
619
631
  clusterSizeTolerance: options.clusterPolicy?.sizeTolerance ?? 0.5
@@ -670,646 +682,659 @@ export async function createLibp2pNodeBase(
670
682
 
671
683
  await node.start();
672
684
 
673
- // Initialize peer reputation service
674
- const reputation = new PeerReputationService();
675
-
676
- // Initialize cluster coordination components
677
- const networkMode: NetworkMode = (options.bootstrapNodes?.length ?? 0) > 0 ? 'joining' : 'forming';
678
- // Network-namespaced protocol prefix, threaded into the key network so coordinator/
679
- // cohort selection is scoped to peers that serve THIS network's cluster/repo protocol.
680
- // A peer that only belongs to another network sharing the same physical nodes/
681
- // bootstraps registers a different (network-namespaced) identify protocol, so it is
682
- // never selected and can't drag this network's super-majority below quorum.
683
- const protocolPrefix = `/optimystic/${options.networkName}`;
684
- const keyNetwork = new Libp2pKeyPeerNetwork(node, options.clusterSize, undefined, networkMode, options.persistence, reputation, protocolPrefix);
685
- await keyNetwork.initFromPersistedState();
686
- const createClusterClient = (peerId: any) => ClusterClient.create(peerId, keyNetwork, protocolPrefix);
687
-
688
- // Inject reputation into NetworkManagerService. Load-bearing and non-optional: the service is
689
- // unconditionally present, so a throw is a real wiring bug. Unlike the pre-start injections above
690
- // the node has already started here, so stop it before rethrowing rather than leaking a started
691
- // node + open transports (mirrors the cohortTopic hard-fail blocks below).
685
+ // Everything from here to the `return` runs against an ALREADY STARTED node (open transports,
686
+ // listening addresses, running services). A rejection out of that span used to hand the caller an
687
+ // error and no handle, leaving the node running with its listener port still bound — unrecoverable
688
+ // for the caller and enough to block the port for the next start attempt. So the whole post-start
689
+ // body rolls back: see the `catch` at the bottom of this function.
692
690
  try {
693
- wired.networkManager.setReputation(reputation);
694
- } catch (err) {
695
- await node.stop();
696
- throw err;
697
- }
698
691
 
699
- // Create partition detector and get FRET service
700
- const partitionDetector = new PartitionDetector();
701
- const fretSvc = (node as any).services?.fret as FretService | undefined;
692
+ // Initialize peer reputation service
693
+ const reputation = new PeerReputationService();
694
+
695
+ // Initialize cluster coordination components
696
+ const networkMode: NetworkMode = (options.bootstrapNodes?.length ?? 0) > 0 ? 'joining' : 'forming';
697
+ // Network-namespaced protocol prefix, threaded into the key network so coordinator/
698
+ // cohort selection is scoped to peers that serve THIS network's cluster/repo protocol.
699
+ // A peer that only belongs to another network sharing the same physical nodes/
700
+ // bootstraps registers a different (network-namespaced) identify protocol, so it is
701
+ // never selected and can't drag this network's super-majority below quorum.
702
+ const protocolPrefix = `/optimystic/${options.networkName}`;
703
+ const keyNetwork = new Libp2pKeyPeerNetwork(node, consensusConfig.clusterSize, undefined, networkMode, options.persistence, reputation, protocolPrefix);
704
+ await keyNetwork.initFromPersistedState();
705
+ const createClusterClient = (peerId: any) => ClusterClient.create(peerId, keyNetwork, protocolPrefix);
706
+
707
+ // Inject reputation into NetworkManagerService. Load-bearing and non-optional: the service is
708
+ // unconditionally present, so a throw is a real wiring bug. The node has already started here, but
709
+ // no ad-hoc stop is needed: the post-start rollback `catch` at the bottom of this function stops it.
710
+ wired.networkManager.setReputation(reputation);
702
711
 
703
- // Every cluster-policy default lives in `cluster/cluster-policy.ts` — including WHY the admission
704
- // gate and the repair corroboration floor resolve the one operator field
705
- // (`clusterPolicy.assumedClusterSize`) to different values when it is absent. Extracted so those
706
- // defaults can be asserted without booting a node.
707
- const consensusConfig = resolveClusterPolicy(options);
712
+ // Create partition detector and get FRET service
713
+ const partitionDetector = new PartitionDetector();
714
+ const fretSvc = (node as any).services?.fret as FretService | undefined;
708
715
 
709
- // Fetch a block archive from one cohort peer over the sync protocol, bounded by a
710
- // per-peer timeout so an unreachable peer can't stall reconciliation. Mirrors the
711
- // SyncClient query in `clusterLatestCallback`, but returns the full archive (which
712
- // carries the materialized block) rather than only the latest ActionRev.
713
- const fetchArchiveFromPeer = async (peerIdStr: string, blockId: BlockId): Promise<BlockArchive | undefined> => {
714
- let peerId: ReturnType<typeof peerIdFromString>;
715
- try {
716
- peerId = peerIdFromString(peerIdStr);
717
- } catch {
718
- return undefined;
719
- }
720
- if (peerId.equals(node.peerId)) return undefined;
721
- const syncClient = new SyncClient(peerId, keyNetwork, protocolPrefix);
722
- try {
723
- const response = await Promise.race<SyncResponse>([
724
- syncClient.requestBlock({ blockId, rev: undefined }),
725
- new Promise<SyncResponse>(resolve => { setTimeout(() => resolve({ success: false }), 1000).unref(); })
726
- ]);
727
- return response.success ? response.archive : undefined;
728
- } catch {
729
- // Peer unreachable / no data — caller falls back to the next cohort peer.
730
- return undefined;
731
- }
732
- };
733
-
734
- // Active reconciliation for a block this member committed without a materializable base
735
- // (cohort drift, or a refused `missing-base-revision` commit). See `reconcile-block.ts` for
736
- // the corroboration rules — in particular why both quorums are capped by how many peers
737
- // could answer at all, which is what lets a genuinely two-node cohort heal.
738
- // NOTE: this and the CoordinatorRepo below must cap against the SAME
739
- // repairCorroborationClusterSize, or the two restoration paths disagree about how much trust a
740
- // lone peer gets. Safe today because both read the one `resolveClusterPolicy` result above; if
741
- // either ever resolves its own value, add a fail-fast coupling check like
742
- // `assertSuperMajorityCoupling` rather than relying on proximity.
743
- const reconcileBlock: ReconcileBlockCallback = createReconcileBlock({
744
- selfPeerId: node.peerId.toString(),
745
- fetchArchive: fetchArchiveFromPeer,
746
- saveReplicatedBlock: (blockId, block, source) => storageRepo.saveReplicatedBlock(blockId, block, source),
747
- simpleMajorityThreshold: consensusConfig.simpleMajorityThreshold,
748
- repairCorroborationClusterSize: consensusConfig.repairCorroborationClusterSize,
749
- reputation
750
- });
751
-
752
- // Member-side membership derivation for the admission gate: independently re-derive this block's
753
- // responsible cluster from the SAME source the coordinator uses (IKeyNetwork.findCluster), plus FRET's
754
- // network-size confidence. A member gates a coordinator-declared peer set against this view before
755
- // voting, so a self-shrunk minority-partition set cannot be voted into super-majority (see cluster-repo
756
- // admitMembership). No FRET ⇒ confidence 0 ⇒ the gate fails closed for any downsize.
757
- const deriveExpectedCluster: DeriveExpectedClusterCallback = async (blockId) => {
758
- const peers = await keyNetwork.findCluster(new TextEncoder().encode(blockId));
759
- let confidence = 0;
760
- if (fretSvc) {
716
+ // Fetch a block archive from one cohort peer over the sync protocol, bounded by a
717
+ // per-peer timeout so an unreachable peer can't stall reconciliation. Mirrors the
718
+ // SyncClient query in `clusterLatestCallback`, but returns the full archive (which
719
+ // carries the materialized block) rather than only the latest ActionRev.
720
+ const fetchArchiveFromPeer = async (peerIdStr: string, blockId: BlockId): Promise<BlockArchive | undefined> => {
721
+ let peerId: ReturnType<typeof peerIdFromString>;
761
722
  try {
762
- confidence = fretSvc.getNetworkSizeEstimate().confidence;
723
+ peerId = peerIdFromString(peerIdStr);
763
724
  } catch {
764
- // Leave confidence 0 → fail closed for downsizing.
725
+ return undefined;
765
726
  }
766
- }
767
- return { peers: peers ?? {}, confidence };
768
- };
769
-
770
- clusterImpl = clusterMember({
771
- storageRepo,
772
- peerNetwork: keyNetwork,
773
- peerId: node.peerId,
774
- privateKey: nodePrivateKey,
775
- protocolPrefix,
776
- partitionDetector,
777
- fretService: fretSvc,
778
- validator: options.validator,
779
- reputation,
780
- consensusConfig,
781
- stateStore: options.transactionStateStore,
782
- reconcileBlock,
783
- onCommitCertificate,
784
- deriveExpectedCluster
785
- // `recomputeArbitratorSet` (invalidation layer-2) is intentionally NOT wired here yet: a live FRET
786
- // recompute needs a churn-tolerance window so it does not false-reject legitimate certificates from
787
- // late-joiners (a liveness regression). Until that is tuned against live topology — and the
788
- // cohort-topic membership-cert trust anchor (layer 3) lands — invalidation verification runs on the
789
- // challenger-bound set + membership + dedup (layer 1) and LOGS the residual anchoring gap. See
790
- // `verifyInvalidationCertificate` and `tickets/plan/cohort-topic-membership-cert-trust-anchoring.md`.
791
- });
792
-
793
- const coordinatorRepoFactory = coordinatorRepo(
794
- keyNetwork,
795
- createClusterClient,
796
- {
797
- // clusterSize is now part of consensusConfig (member + coordinator share one reference).
798
- ...consensusConfig
799
- },
800
- fretSvc,
801
- reputation,
802
- options.transactionStateStore
803
- );
804
-
805
- // Create callback for querying cluster peers for their latest block revision. Three-way
806
- // contract (see ClusterLatestCallback): an ActionRev is the peer's claim, a resolved
807
- // `undefined` is the peer answering "I hold nothing", and a REJECTION is silence — the
808
- // coordinator counts it as "did not answer" and refuses to report an authoritative absent
809
- // over it. Transport errors must therefore propagate, not collapse into `undefined` (that
810
- // collapse let a slow two-node cohort report a missing block as authoritatively absent —
811
- // ticket cluster-read-consult-cannot-report-unreachable).
812
- const clusterLatestCallback: ClusterLatestCallback = async (peerId, blockId, context?) => {
813
- // Self-read short-circuit: dialling self via SyncClient is a round trip
814
- // with no remote on the other end, and on nodes without listen addresses
815
- // (solo WebSocket-only, bare-RN, etc.) the self-dial can hang the dial
816
- // queue. Read directly from the local storage repo instead. The catch stays:
817
- // a local storage error is not a cohort peer being unreachable, and the
818
- // coordinator ignores a self rejection anyway.
819
- if (peerId.equals(node.peerId)) {
727
+ if (peerId.equals(node.peerId)) return undefined;
728
+ const syncClient = new SyncClient(peerId, keyNetwork, protocolPrefix);
820
729
  try {
821
- const result = await storageRepo.get({ blockIds: [blockId], context });
822
- return result[blockId]?.state?.latest;
730
+ const response = await Promise.race<SyncResponse>([
731
+ syncClient.requestBlock({ blockId, rev: undefined }),
732
+ new Promise<SyncResponse>(resolve => { setTimeout(() => resolve({ success: false }), 1000).unref(); })
733
+ ]);
734
+ return response.success ? response.archive : undefined;
823
735
  } catch {
736
+ // Peer unreachable / no data — caller falls back to the next cohort peer.
824
737
  return undefined;
825
738
  }
826
- }
827
- const syncClient = new SyncClient(peerId, keyNetwork, protocolPrefix);
828
- // No try/catch: a dial or protocol failure rejects through to the coordinator, whose
829
- // per-peer deadline also bounds a hung request slowness needs no race here.
830
- const response = await syncClient.requestBlock({ blockId, rev: undefined });
831
- if (response.success && response.archive) {
832
- const revisions = Object.keys(response.archive.revisions).map(Number);
833
- if (revisions.length > 0) {
834
- const maxRev = Math.max(...revisions);
835
- const revisionData = response.archive.revisions[maxRev];
836
- if (revisionData?.action) {
837
- return { actionId: revisionData.action.actionId, rev: maxRev };
739
+ };
740
+
741
+ // Active reconciliation for a block this member committed without a materializable base
742
+ // (cohort drift, or a refused `missing-base-revision` commit). See `reconcile-block.ts` for
743
+ // the corroboration rules in particular why both quorums are capped by how many peers
744
+ // could answer at all, which is what lets a genuinely two-node cohort heal.
745
+ // NOTE: this and the CoordinatorRepo below must cap against the SAME
746
+ // repairCorroborationClusterSize, or the two restoration paths disagree about how much trust a
747
+ // lone peer gets. Safe today because both read the one `resolveClusterPolicy` result above; if
748
+ // either ever resolves its own value, add a fail-fast coupling check like
749
+ // `assertSuperMajorityCoupling` rather than relying on proximity.
750
+ const reconcileBlock: ReconcileBlockCallback = createReconcileBlock({
751
+ selfPeerId: node.peerId.toString(),
752
+ fetchArchive: fetchArchiveFromPeer,
753
+ saveReplicatedBlock: (blockId, block, source) => storageRepo.saveReplicatedBlock(blockId, block, source),
754
+ simpleMajorityThreshold: consensusConfig.simpleMajorityThreshold,
755
+ repairCorroborationClusterSize: consensusConfig.repairCorroborationClusterSize,
756
+ reputation
757
+ });
758
+
759
+ // Member-side membership derivation for the admission gate: independently re-derive this block's
760
+ // responsible cluster from the SAME source the coordinator uses (IKeyNetwork.findCluster), plus FRET's
761
+ // network-size confidence. A member gates a coordinator-declared peer set against this view before
762
+ // voting, so a self-shrunk minority-partition set cannot be voted into super-majority (see cluster-repo
763
+ // admitMembership). No FRET ⇒ confidence 0 ⇒ the gate fails closed for any downsize.
764
+ const deriveExpectedCluster: DeriveExpectedClusterCallback = async (blockId) => {
765
+ const peers = await keyNetwork.findCluster(new TextEncoder().encode(blockId));
766
+ let confidence = 0;
767
+ if (fretSvc) {
768
+ try {
769
+ confidence = fretSvc.getNetworkSizeEstimate().confidence;
770
+ } catch {
771
+ // Leave confidence 0 → fail closed for downsizing.
838
772
  }
839
773
  }
840
- }
841
- // The peer DID answer, without data: `success:false` is the sync service's "Block not
842
- // found in local storage", and an archive with no usable revisions holds nothing either
843
- // way. Both are absent claims, not silence.
844
- return undefined;
845
- };
774
+ return { peers: peers ?? {}, confidence };
775
+ };
846
776
 
847
- coordinatedRepo = coordinatorRepoFactory({
848
- storageRepo,
849
- localCluster: clusterImpl,
850
- localPeerId: node.peerId,
851
- clusterLatestCallback,
852
- // Read-driven acquisition shares the commit path's reconcile callback verbatim: same bounded
853
- // archive fetch, same (rev, actionId) and content quorums, same monotonic saveReplicatedBlock
854
- // funnel. `clusterLatestCallback` alone can only tell the reader WHICH revision the cohort
855
- // holds; this is what moves the bytes. Only reached once a corroborated revision exists, so a
856
- // genuinely absent block still costs no archive fetch.
857
- acquireBlockFromCohort: reconcileBlock
858
- });
859
-
860
- // Fail-fast coupling: the cluster member (what accepts a super-majority as sufficient) and the
861
- // coordinator (what declares a transaction committed on that super-majority) MUST run the same
862
- // threshold, or the node would come up able to disagree with itself mid-consensus. Both are fed from
863
- // the single `consensusConfig` above; this asserts on their RESOLVED values so any future drift throws
864
- // HERE at construction. See `assertSuperMajorityCoupling`.
865
- assertSuperMajorityCoupling(
866
- clusterImpl as import('./cluster/cluster-repo.js').ClusterMember,
867
- coordinatedRepo as import('./repo/coordinator-repo.js').CoordinatorRepo
868
- );
777
+ clusterImpl = clusterMember({
778
+ storageRepo,
779
+ peerNetwork: keyNetwork,
780
+ peerId: node.peerId,
781
+ privateKey: nodePrivateKey,
782
+ protocolPrefix,
783
+ partitionDetector,
784
+ fretService: fretSvc,
785
+ validator: options.validator,
786
+ reputation,
787
+ consensusConfig,
788
+ stateStore: options.transactionStateStore,
789
+ reconcileBlock,
790
+ onCommitCertificate,
791
+ deriveExpectedCluster
792
+ // `recomputeArbitratorSet` (invalidation layer-2) is intentionally NOT wired here yet: a live FRET
793
+ // recompute needs a churn-tolerance window so it does not false-reject legitimate certificates from
794
+ // late-joiners (a liveness regression). Until that is tuned against live topology — and the
795
+ // cohort-topic membership-cert trust anchor (layer 3) lands — invalidation verification runs on the
796
+ // challenger-bound set + membership + dedup (layer 1) and LOGS the residual anchoring gap. See
797
+ // `verifyInvalidationCertificate` and `tickets/plan/cohort-topic-membership-cert-trust-anchoring.md`.
798
+ });
869
799
 
870
- // Recover persisted transaction state before accepting new requests
871
- if (options.transactionStateStore) {
872
- await (clusterImpl as import('./cluster/cluster-repo.js').ClusterMember).recoverTransactions();
873
- await (coordinatedRepo as import('./repo/coordinator-repo.js').CoordinatorRepo).recoverTransactions();
874
- }
800
+ // Cleanup cluster member intervals on node stop. Installed HERE, immediately after clusterImpl
801
+ // exists, rather than further down: the post-start rollback only unwinds resources whose stop
802
+ // wrapper is already installed at the moment of the throw, so a wrapper trailing its resource by
803
+ // hundreds of lines leaves those intervals running on a failed startup. Same reasoning as the
804
+ // owned-block-feed wrapper below.
805
+ {
806
+ const previousStop = node.stop.bind(node);
807
+ node.stop = async () => {
808
+ try {
809
+ (clusterImpl as import('./cluster/cluster-repo.js').ClusterMember).dispose();
810
+ } finally {
811
+ // Never let a dispose failure strand the transports — same try/finally shape every
812
+ // other wrapper in this chain uses.
813
+ await previousStop();
814
+ }
815
+ };
816
+ }
875
817
 
876
- // --- Shared owned-block set for the resilience monitors ---
877
- // SpreadOnChurnMonitor (sender) and RebalanceMonitor (responsibility tracker) both act on "the
878
- // blocks this node physically holds". They share ONE Set so the two can never drift: a single
879
- // owned-block feed populates it, and the rebalance responsibility-loss signal evicts from it
880
- // (in the rebalance block below). Both monitors take this exact instance via deps.trackedBlocks.
881
- const networkManager = (node as any).services?.networkManager as NetworkManagerService | undefined;
882
- const ownedBlocks = new Set<string>();
883
- // Single owned-block feed: every block this node commits OR receives as a replica fires
884
- // storageRepo.onAnyCollectionChange. Subscribe to storageRepo DIRECTLY (not
885
- // node.blockChangeNotifier): the cohort-topic activation block below may replace
886
- // blockChangeNotifier with a decorating bridge, but storageRepo keeps emitting on its own
887
- // surface regardless of that opt-in. NOTE: this feed does NOT re-emit blocks already durable
888
- // from a previous run; those are seeded once at startup by the storage-enumeration scan wired
889
- // below (seedOwnedBlocksFromStorage), so a restarted node protects on-disk data without waiting
890
- // for each block to be touched again. Registered lazily the first time a
891
- // monitor that reads ownedBlocks is wired, so when BOTH monitors are disabled no subscription
892
- // leaks; torn down exactly once in the stop wrapper below.
893
- let offOwnedBlockFeed: (() => void) | undefined;
894
- const ensureOwnedBlockFeed = (): void => {
895
- if (offOwnedBlockFeed) return;
896
- offOwnedBlockFeed = storageRepo.onAnyCollectionChange((e) => {
897
- for (const blockId of e.blockIds) ownedBlocks.add(blockId);
898
- });
899
- };
900
- // Single owned-block-feed teardown. Registered up front (before either monitor's own stop
901
- // wrapper) so it runs regardless of WHICH monitor subscribed the feed - including the
902
- // spread-disabled / rebalance-only case. Idempotent: offOwnedBlockFeed is undefined-guarded.
903
- {
904
- const previousStop = node.stop.bind(node);
905
- node.stop = async () => {
906
- try {
907
- offOwnedBlockFeed?.();
908
- } finally {
909
- await previousStop();
818
+ const coordinatorRepoFactory = coordinatorRepo(
819
+ keyNetwork,
820
+ createClusterClient,
821
+ {
822
+ // clusterSize is now part of consensusConfig (member + coordinator share one reference).
823
+ ...consensusConfig
824
+ },
825
+ fretSvc,
826
+ reputation,
827
+ options.transactionStateStore
828
+ );
829
+
830
+ // Create callback for querying cluster peers for their latest block revision. Three-way
831
+ // contract (see ClusterLatestCallback): an ActionRev is the peer's claim, a resolved
832
+ // `undefined` is the peer answering "I hold nothing", and a REJECTION is silence — the
833
+ // coordinator counts it as "did not answer" and refuses to report an authoritative absent
834
+ // over it. Transport errors must therefore propagate, not collapse into `undefined` (that
835
+ // collapse let a slow two-node cohort report a missing block as authoritatively absent —
836
+ // ticket cluster-read-consult-cannot-report-unreachable).
837
+ const clusterLatestCallback: ClusterLatestCallback = async (peerId, blockId, context?) => {
838
+ // Self-read short-circuit: dialling self via SyncClient is a round trip
839
+ // with no remote on the other end, and on nodes without listen addresses
840
+ // (solo WebSocket-only, bare-RN, etc.) the self-dial can hang the dial
841
+ // queue. Read directly from the local storage repo instead. The catch stays:
842
+ // a local storage error is not a cohort peer being unreachable, and the
843
+ // coordinator ignores a self rejection anyway.
844
+ if (peerId.equals(node.peerId)) {
845
+ try {
846
+ const result = await storageRepo.get({ blockIds: [blockId], context });
847
+ return result[blockId]?.state?.latest;
848
+ } catch {
849
+ return undefined;
850
+ }
910
851
  }
852
+ const syncClient = new SyncClient(peerId, keyNetwork, protocolPrefix);
853
+ // No try/catch: a dial or protocol failure rejects through to the coordinator, whose
854
+ // per-peer deadline also bounds a hung request — slowness needs no race here.
855
+ const response = await syncClient.requestBlock({ blockId, rev: undefined });
856
+ if (response.success && response.archive) {
857
+ const revisions = Object.keys(response.archive.revisions).map(Number);
858
+ if (revisions.length > 0) {
859
+ const maxRev = Math.max(...revisions);
860
+ const revisionData = response.archive.revisions[maxRev];
861
+ if (revisionData?.action) {
862
+ return { actionId: revisionData.action.actionId, rev: maxRev };
863
+ }
864
+ }
865
+ }
866
+ // The peer DID answer, without data: `success:false` is the sync service's "Block not
867
+ // found in local storage", and an archive with no usable revisions holds nothing either
868
+ // way. Both are absent claims, not silence.
869
+ return undefined;
911
870
  };
912
- }
913
871
 
914
- // --- Churn-resilient spread: drive SpreadOnChurnMonitor on a live node ---
915
- // Nothing previously activated the SENDING side of the churn-resilient spread protocol on a
916
- // real node. Here we init + start the monitor (sharing ownedBlocks) and ensure the single
917
- // owned-block feed is live, so a debounced connection:close re-pushes the node's blocks to
918
- // expansion-cohort peers (the receiver durably persists each push via saveReplicatedBlock).
919
- let spreadMonitor: SpreadOnChurnMonitor | undefined;
920
- if (networkManager && (options.spreadOnChurn?.enabled ?? true) !== false) {
921
- try {
922
- spreadMonitor = networkManager.initSpreadOnChurnMonitor(
923
- partitionDetector,
924
- storageRepo,
925
- keyNetwork,
926
- options.clusterSize ?? 10,
927
- protocolPrefix,
928
- ownedBlocks,
929
- options.spreadOnChurn,
930
- );
931
- await spreadMonitor.start();
932
- ensureOwnedBlockFeed();
933
- } catch (err) {
934
- // Spread is a resilience optimization, not a correctness requirement - a wiring
935
- // failure (e.g. FRET briefly unavailable) must NOT hard-fail node startup, unlike the
936
- // operator-opted-in cohortTopic block. Log and continue with spread inert.
937
- ((node as any).logger?.forComponent?.('db-p2p:spread-on-churn'))?.('init failed: %o', err);
872
+ coordinatedRepo = coordinatorRepoFactory({
873
+ storageRepo,
874
+ localCluster: clusterImpl,
875
+ localPeerId: node.peerId,
876
+ clusterLatestCallback,
877
+ // Read-driven acquisition shares the commit path's reconcile callback verbatim: same bounded
878
+ // archive fetch, same (rev, actionId) and content quorums, same monotonic saveReplicatedBlock
879
+ // funnel. `clusterLatestCallback` alone can only tell the reader WHICH revision the cohort
880
+ // holds; this is what moves the bytes. Only reached once a corroborated revision exists, so a
881
+ // genuinely absent block still costs no archive fetch.
882
+ acquireBlockFromCohort: reconcileBlock
883
+ });
884
+
885
+ // Fail-fast coupling: the cluster member (what accepts a super-majority as sufficient) and the
886
+ // coordinator (what declares a transaction committed on that super-majority) MUST run the same
887
+ // threshold, or the node would come up able to disagree with itself mid-consensus. Both are fed from
888
+ // the single `consensusConfig` above; this asserts on their RESOLVED values so any future drift throws
889
+ // HERE at construction. See `assertSuperMajorityCoupling`.
890
+ assertSuperMajorityCoupling(
891
+ clusterImpl as import('./cluster/cluster-repo.js').ClusterMember,
892
+ coordinatedRepo as import('./repo/coordinator-repo.js').CoordinatorRepo
893
+ );
894
+
895
+ // Recover persisted transaction state before accepting new requests
896
+ if (options.transactionStateStore) {
897
+ await (clusterImpl as import('./cluster/cluster-repo.js').ClusterMember).recoverTransactions();
898
+ await (coordinatedRepo as import('./repo/coordinator-repo.js').CoordinatorRepo).recoverTransactions();
938
899
  }
939
- }
940
900
 
941
- // Expose for tests/diagnostics (mirrors node.keyNetwork / node.reputation).
942
- (node as any).spreadOnChurnMonitor = spreadMonitor;
943
-
944
- // Disposal: stop the spread monitor deterministically before the transports close. Composes
945
- // with the arachnode / clusterMember / cohort-topic stop wrappers (each calls its captured
946
- // previousStop last). Idempotent (SpreadOnChurnMonitor.stop early-returns when not running), so
947
- // a double node.stop() does not throw. The owned-block feed teardown is the separate up-front
948
- // wrapper above (shared across both monitors).
949
- {
950
- const previousStop = node.stop.bind(node);
951
- node.stop = async () => {
901
+ // --- Shared owned-block set for the resilience monitors ---
902
+ // SpreadOnChurnMonitor (sender) and RebalanceMonitor (responsibility tracker) both act on "the
903
+ // blocks this node physically holds". They share ONE Set so the two can never drift: a single
904
+ // owned-block feed populates it, and the rebalance responsibility-loss signal evicts from it
905
+ // (in the rebalance block below). Both monitors take this exact instance via deps.trackedBlocks.
906
+ const networkManager = (node as any).services?.networkManager as NetworkManagerService | undefined;
907
+
908
+ // See the comment above `consensusConfig` for why every cluster-size consumer must read the SAME
909
+ // resolved value. This throws at construction (rather than letting a node come up mismatched) if a
910
+ // future edit gives `keyNetwork` or `networkManager` their own fallback again.
911
+ assertClusterSizeCoupling(consensusConfig.clusterSize, { keyNetwork, networkManager });
912
+
913
+ const ownedBlocks = new Set<string>();
914
+ // Single owned-block feed: every block this node commits OR receives as a replica fires
915
+ // storageRepo.onAnyCollectionChange. Subscribe to storageRepo DIRECTLY (not
916
+ // node.blockChangeNotifier): the cohort-topic activation block below may replace
917
+ // blockChangeNotifier with a decorating bridge, but storageRepo keeps emitting on its own
918
+ // surface regardless of that opt-in. NOTE: this feed does NOT re-emit blocks already durable
919
+ // from a previous run; those are seeded once at startup by the storage-enumeration scan wired
920
+ // below (seedOwnedBlocksFromStorage), so a restarted node protects on-disk data without waiting
921
+ // for each block to be touched again. Registered lazily the first time a
922
+ // monitor that reads ownedBlocks is wired, so when BOTH monitors are disabled no subscription
923
+ // leaks; torn down exactly once in the stop wrapper below.
924
+ let offOwnedBlockFeed: (() => void) | undefined;
925
+ const ensureOwnedBlockFeed = (): void => {
926
+ if (offOwnedBlockFeed) return;
927
+ offOwnedBlockFeed = storageRepo.onAnyCollectionChange((e) => {
928
+ for (const blockId of e.blockIds) ownedBlocks.add(blockId);
929
+ });
930
+ };
931
+ // Single owned-block-feed teardown. Registered up front (before either monitor's own stop
932
+ // wrapper) so it runs regardless of WHICH monitor subscribed the feed - including the
933
+ // spread-disabled / rebalance-only case. Idempotent: offOwnedBlockFeed is undefined-guarded.
934
+ {
935
+ const previousStop = node.stop.bind(node);
936
+ node.stop = async () => {
937
+ try {
938
+ offOwnedBlockFeed?.();
939
+ } finally {
940
+ await previousStop();
941
+ }
942
+ };
943
+ }
944
+
945
+ // --- Churn-resilient spread: drive SpreadOnChurnMonitor on a live node ---
946
+ // Nothing previously activated the SENDING side of the churn-resilient spread protocol on a
947
+ // real node. Here we init + start the monitor (sharing ownedBlocks) and ensure the single
948
+ // owned-block feed is live, so a debounced connection:close re-pushes the node's blocks to
949
+ // expansion-cohort peers (the receiver durably persists each push via saveReplicatedBlock).
950
+ let spreadMonitor: SpreadOnChurnMonitor | undefined;
951
+ if (networkManager && (options.spreadOnChurn?.enabled ?? true) !== false) {
952
952
  try {
953
- if (spreadMonitor) await spreadMonitor.stop();
954
- } finally {
955
- await previousStop();
953
+ spreadMonitor = networkManager.initSpreadOnChurnMonitor(
954
+ partitionDetector,
955
+ storageRepo,
956
+ keyNetwork,
957
+ consensusConfig.clusterSize,
958
+ protocolPrefix,
959
+ ownedBlocks,
960
+ options.spreadOnChurn,
961
+ );
962
+ await spreadMonitor.start();
963
+ ensureOwnedBlockFeed();
964
+ } catch (err) {
965
+ // Spread is a resilience optimization, not a correctness requirement - a wiring
966
+ // failure (e.g. FRET briefly unavailable) must NOT hard-fail node startup, unlike the
967
+ // operator-opted-in cohortTopic block. Log and continue with spread inert.
968
+ ((node as any).logger?.forComponent?.('db-p2p:spread-on-churn'))?.('init failed: %o', err);
956
969
  }
957
- };
958
- }
970
+ }
959
971
 
960
- // Initialize Arachnode ring membership and restoration
961
- const enableArachnode = options.arachnode?.enableRingZulu ?? true;
962
- if (enableArachnode) {
963
- const log = (node as any).logger?.forComponent?.('db-p2p:arachnode');
964
- const fret = (node as any).services?.fret as any;
965
-
966
- if (fret) {
967
- const fretAdapter = new ArachnodeFretAdapter(fret, node.peerId.toString());
968
-
969
- // Blocks whose shed range has been RELEASED (Phase C of a ring shift, or a confirmed
970
- // rebalance release). This is the GC-eligibility signal the future storage sweep
971
- // (`st-storage-sweep-archival-and-capacity-estimate`) must consult: a block's local bytes may
972
- // be reclaimed ONLY once it appears here, so an unconfirmed / still-served range is never
973
- // swept. Populated strictly after replication is confirmed. See
974
- // docs/arachnode-ring-handoff.md § Part 2 (Local bytes vs. tracking).
975
- // NOTE: no sweep consumes this set yet; it is the coordinated eligibility handoff the sweep
976
- // ticket will read. Until then it grows unbounded — bound it when the sweep lands.
977
- const gcEligible = new Set<string>();
978
- (node as any).gcEligibleBlocks = gcEligible;
979
-
980
- // The ring-shift state machine (advertise→confirm→release). Wired inside the rebalance block
981
- // below (it needs the BlockTransferCoordinator confirmer + the cohort-size floor); left
982
- // undefined when the rebalance reaction is not wired, in which case ring shifts stay inert —
983
- // a move-out is unsafe without the confirm/release path.
984
- let ringShift: RingShiftCoordinator | undefined;
985
-
986
- const storageMonitor = new StorageMonitor(rawStorage, options.arachnode?.storage ?? {});
987
- const ringSelector = new RingSelector(fretAdapter, storageMonitor, {
988
- minCapacity: 100 * 1024 * 1024,
989
- thresholds: {
990
- moveOut: 0.85,
991
- moveIn: 0.40
992
- },
993
- // Damping so the ring decision cannot thrash near a boundary
994
- // (docs/arachnode-ring-handoff.md § Part 1).
995
- smoothingAlpha: 0.2,
996
- deadband: 0.5,
997
- minDwellMs: 10 * 60 * 1000
998
- });
972
+ // Expose for tests/diagnostics (mirrors node.keyNetwork / node.reputation).
973
+ (node as any).spreadOnChurnMonitor = spreadMonitor;
974
+
975
+ // Disposal: stop the spread monitor deterministically before the transports close. Composes
976
+ // with the arachnode / clusterMember / cohort-topic stop wrappers (each calls its captured
977
+ // previousStop last). Idempotent (SpreadOnChurnMonitor.stop early-returns when not running), so
978
+ // a double node.stop() does not throw. The owned-block feed teardown is the separate up-front
979
+ // wrapper above (shared across both monitors).
980
+ {
981
+ const previousStop = node.stop.bind(node);
982
+ node.stop = async () => {
983
+ try {
984
+ if (spreadMonitor) await spreadMonitor.stop();
985
+ } finally {
986
+ await previousStop();
987
+ }
988
+ };
989
+ }
999
990
 
1000
- // Determine and announce ring membership
1001
- const peerId = node.peerId.toString();
1002
- const arachnodeInfo = await ringSelector.createArachnodeInfo(peerId);
1003
- fretAdapter.setArachnodeInfo(arachnodeInfo);
991
+ // Initialize Arachnode ring membership and restoration
992
+ const enableArachnode = options.arachnode?.enableRingZulu ?? true;
993
+ if (enableArachnode) {
994
+ const log = (node as any).logger?.forComponent?.('db-p2p:arachnode');
995
+ const fret = (node as any).services?.fret as any;
996
+
997
+ if (fret) {
998
+ const fretAdapter = new ArachnodeFretAdapter(fret, node.peerId.toString());
999
+
1000
+ // Blocks whose shed range has been RELEASED (Phase C of a ring shift, or a confirmed
1001
+ // rebalance release). This is the GC-eligibility signal the future storage sweep
1002
+ // (`st-storage-sweep-archival-and-capacity-estimate`) must consult: a block's local bytes may
1003
+ // be reclaimed ONLY once it appears here, so an unconfirmed / still-served range is never
1004
+ // swept. Populated strictly after replication is confirmed. See
1005
+ // docs/arachnode-ring-handoff.md § Part 2 (Local bytes vs. tracking).
1006
+ // NOTE: no sweep consumes this set yet; it is the coordinated eligibility handoff the sweep
1007
+ // ticket will read. Until then it grows unbounded — bound it when the sweep lands.
1008
+ const gcEligible = new Set<string>();
1009
+ (node as any).gcEligibleBlocks = gcEligible;
1010
+
1011
+ // The ring-shift state machine (advertise→confirm→release). Wired inside the rebalance block
1012
+ // below (it needs the BlockTransferCoordinator confirmer + the cohort-size floor); left
1013
+ // undefined when the rebalance reaction is not wired, in which case ring shifts stay inert —
1014
+ // a move-out is unsafe without the confirm/release path.
1015
+ let ringShift: RingShiftCoordinator | undefined;
1016
+
1017
+ const storageMonitor = new StorageMonitor(rawStorage, options.arachnode?.storage ?? {});
1018
+ const ringSelector = new RingSelector(fretAdapter, storageMonitor, {
1019
+ minCapacity: 100 * 1024 * 1024,
1020
+ thresholds: {
1021
+ moveOut: 0.85,
1022
+ moveIn: 0.40
1023
+ },
1024
+ // Damping so the ring decision cannot thrash near a boundary
1025
+ // (docs/arachnode-ring-handoff.md § Part 1).
1026
+ smoothingAlpha: 0.2,
1027
+ deadband: 0.5,
1028
+ minDwellMs: 10 * 60 * 1000
1029
+ });
1004
1030
 
1005
- log?.('Announced Arachnode membership: Ring %d', arachnodeInfo.ringDepth);
1031
+ // Determine and announce ring membership
1032
+ const peerId = node.peerId.toString();
1033
+ const arachnodeInfo = await ringSelector.createArachnodeInfo(peerId);
1034
+ fretAdapter.setArachnodeInfo(arachnodeInfo);
1006
1035
 
1007
- // Setup restoration coordinator with FRET adapter
1008
- const restorationCoordinatorV2 = new RestorationCoordinator(
1009
- fretAdapter,
1010
- { connect: (pid, protocol) => node.dialProtocol(pid as Parameters<typeof node.dialProtocol>[0], [protocol]) },
1011
- `/optimystic/${options.networkName}`,
1012
- node.peerId.toString()
1013
- );
1036
+ log?.('Announced Arachnode membership: Ring %d', arachnodeInfo.ringDepth);
1014
1037
 
1015
- // Update restore callback to use new coordinator
1016
- const newRestoreCallback: RestoreCallback = async (blockId, rev?) => {
1017
- return await restorationCoordinatorV2.restore(blockId, rev);
1018
- };
1038
+ // Setup restoration coordinator with FRET adapter
1039
+ const restorationCoordinatorV2 = new RestorationCoordinator(
1040
+ fretAdapter,
1041
+ { connect: (pid, protocol) => node.dialProtocol(pid as Parameters<typeof node.dialProtocol>[0], [protocol]) },
1042
+ `/optimystic/${options.networkName}`,
1043
+ node.peerId.toString()
1044
+ );
1019
1045
 
1020
- // Replace the restore callback (this is a bit hacky, but works for now)
1021
- (storageRepo as any).createBlockStorage = (blockId: string) =>
1022
- new BlockStorage(blockId, rawStorage, newRestoreCallback);
1023
-
1024
- // --- Rebalance reaction: drive RebalanceMonitor + react via BlockTransferCoordinator ---
1025
- // Nothing previously activated the rebalance path on a real node: initRebalanceMonitor was
1026
- // never called, the monitor was never start()ed, and BlockTransferCoordinator (the
1027
- // pull-gained / push-lost reaction primitive) was never constructed in src. This block lives
1028
- // inside the arachnode `if (fret)` gate because both dependencies only exist here — the
1029
- // fretAdapter and the RestorationCoordinator. When arachnode is disabled or FRET is absent the
1030
- // rebalance path stays inert (acceptable: rebalance is a resilience optimization). A wiring
1031
- // failure here is non-fatal (log + continue), unlike the operator-opted-in cohortTopic block.
1032
- if (networkManager && (options.rebalance?.enabled ?? true) !== false) {
1033
- try {
1034
- // repo → the LOCAL storageRepo (not repoProxy/coordinatedRepo): a pulled/pushed replica
1035
- // must land in / be read from this node's own storage, same reasoning as the
1036
- // blockTransfer service handler registration. protocolPrefix (/optimystic/<networkName>)
1037
- // MUST match the prefix the node registers its block-transfer handler under, or every
1038
- // lost-block push dials the wrong protocol and fails to connect.
1039
- const coordinator = new BlockTransferCoordinator(
1040
- storageRepo,
1041
- keyNetwork,
1042
- restorationCoordinatorV2,
1043
- partitionDetector,
1044
- protocolPrefix,
1045
- );
1046
+ // Update restore callback to use new coordinator
1047
+ const newRestoreCallback: RestoreCallback = async (blockId, rev?) => {
1048
+ return await restorationCoordinatorV2.restore(blockId, rev);
1049
+ };
1046
1050
 
1047
- const rebalanceMonitor = networkManager.initRebalanceMonitor(
1048
- partitionDetector,
1049
- fretAdapter,
1050
- ownedBlocks,
1051
- options.rebalance,
1052
- );
1053
- await rebalanceMonitor.start();
1054
-
1055
- // onRebalance fires synchronously from the monitor's debounced check; the coordinator's
1056
- // reaction (pull gained / push lost, each partition-guarded) is async, so hop it off the
1057
- // handler rather than blocking the monitor's emit loop. handleRebalanceEvent can REJECT
1058
- // (e.g. RestorationCoordinator.restore() throws while pulling a gained block) and a bare
1059
- // `void` would surface that as an unhandled rejection (process-fatal on Node >=15); the
1060
- // reaction is a resilience optimization, so swallow + log instead.
1061
- //
1062
- // ALONGSIDE dispatching to the coordinator, drive the shared owned-block set off this
1063
- // authoritative responsibility signal. A GAINED block is added immediately so it is
1064
- // tracked even before its next commit/replica touches the feed.
1065
- //
1066
- // A LOST block is NO LONGER released synchronously: doing so stopped spreading a block
1067
- // whose push to the new owners might fail, drop it below the replication floor, and let a
1068
- // later sweep reclaim it (docs/arachnode-ring-handoff.md § Why the current code violates
1069
- // it #2). Instead the release is GATED on confirmation — the coordinator returns the lost
1070
- // blocks it confirmed replicated to ≥ floor new owners, and ONLY those are untracked
1071
- // (authoritative eviction from the shared set — complements spread's lazy self-prune) and
1072
- // marked GC-eligible. A lost block whose push failed / was partition-skipped stays
1073
- // tracked and served, and is retried on the next rebalance.
1074
- //
1075
- // Best-effort iteration safety: this eviction can mutate ownedBlocks while
1076
- // SpreadOnChurnMonitor (or this monitor) is mid for...of over the same Set inside an
1077
- // async loop. Adding/deleting a Set entry during iteration does not throw in JS — entries
1078
- // are visited best-effort — which is acceptable for a resilience mechanism, so we
1079
- // document it here rather than add locking.
1080
- rebalanceMonitor.onRebalance((event) => {
1081
- for (const blockId of event.gained) ownedBlocks.add(blockId);
1082
- coordinator.handleRebalanceEvent(event).then((result) => {
1083
- for (const blockId of result.released) {
1084
- rebalanceMonitor.untrackBlock(blockId); // also evicts from the shared ownedBlocks set
1085
- gcEligible.add(blockId); // confirmed replicated safe to sweep
1051
+ // Replace the restore callback (this is a bit hacky, but works for now)
1052
+ (storageRepo as any).createBlockStorage = (blockId: string) =>
1053
+ new BlockStorage(blockId, rawStorage, newRestoreCallback);
1054
+
1055
+ // --- Rebalance reaction: drive RebalanceMonitor + react via BlockTransferCoordinator ---
1056
+ // Nothing previously activated the rebalance path on a real node: initRebalanceMonitor was
1057
+ // never called, the monitor was never start()ed, and BlockTransferCoordinator (the
1058
+ // pull-gained / push-lost reaction primitive) was never constructed in src. This block lives
1059
+ // inside the arachnode `if (fret)` gate because both dependencies only exist here — the
1060
+ // fretAdapter and the RestorationCoordinator. When arachnode is disabled or FRET is absent the
1061
+ // rebalance path stays inert (acceptable: rebalance is a resilience optimization). A wiring
1062
+ // failure here is non-fatal (log + continue), unlike the operator-opted-in cohortTopic block.
1063
+ if (networkManager && (options.rebalance?.enabled ?? true) !== false) {
1064
+ try {
1065
+ // repo → the LOCAL storageRepo (not repoProxy/coordinatedRepo): a pulled/pushed replica
1066
+ // must land in / be read from this node's own storage, same reasoning as the
1067
+ // blockTransfer service handler registration. protocolPrefix (/optimystic/<networkName>)
1068
+ // MUST match the prefix the node registers its block-transfer handler under, or every
1069
+ // lost-block push dials the wrong protocol and fails to connect.
1070
+ const coordinator = new BlockTransferCoordinator(
1071
+ storageRepo,
1072
+ keyNetwork,
1073
+ restorationCoordinatorV2,
1074
+ partitionDetector,
1075
+ protocolPrefix,
1076
+ );
1077
+
1078
+ const rebalanceMonitor = networkManager.initRebalanceMonitor(
1079
+ partitionDetector,
1080
+ fretAdapter,
1081
+ ownedBlocks,
1082
+ options.rebalance,
1083
+ );
1084
+ await rebalanceMonitor.start();
1085
+
1086
+ // onRebalance fires synchronously from the monitor's debounced check; the coordinator's
1087
+ // reaction (pull gained / push lost, each partition-guarded) is async, so hop it off the
1088
+ // handler rather than blocking the monitor's emit loop. handleRebalanceEvent can REJECT
1089
+ // (e.g. RestorationCoordinator.restore() throws while pulling a gained block) and a bare
1090
+ // `void` would surface that as an unhandled rejection (process-fatal on Node >=15); the
1091
+ // reaction is a resilience optimization, so swallow + log instead.
1092
+ //
1093
+ // ALONGSIDE dispatching to the coordinator, drive the shared owned-block set off this
1094
+ // authoritative responsibility signal. A GAINED block is added immediately so it is
1095
+ // tracked even before its next commit/replica touches the feed.
1096
+ //
1097
+ // A LOST block is NO LONGER released synchronously: doing so stopped spreading a block
1098
+ // whose push to the new owners might fail, drop it below the replication floor, and let a
1099
+ // later sweep reclaim it (docs/arachnode-ring-handoff.md § Why the current code violates
1100
+ // it #2). Instead the release is GATED on confirmation — the coordinator returns the lost
1101
+ // blocks it confirmed replicated to ≥ floor new owners, and ONLY those are untracked
1102
+ // (authoritative eviction from the shared set — complements spread's lazy self-prune) and
1103
+ // marked GC-eligible. A lost block whose push failed / was partition-skipped stays
1104
+ // tracked and served, and is retried on the next rebalance.
1105
+ //
1106
+ // Best-effort iteration safety: this eviction can mutate ownedBlocks while
1107
+ // SpreadOnChurnMonitor (or this monitor) is mid for...of over the same Set inside an
1108
+ // async loop. Adding/deleting a Set entry during iteration does not throw in JS — entries
1109
+ // are visited best-effort — which is acceptable for a resilience mechanism, so we
1110
+ // document it here rather than add locking.
1111
+ rebalanceMonitor.onRebalance((event) => {
1112
+ for (const blockId of event.gained) ownedBlocks.add(blockId);
1113
+ coordinator.handleRebalanceEvent(event).then((result) => {
1114
+ for (const blockId of result.released) {
1115
+ rebalanceMonitor.untrackBlock(blockId); // also evicts from the shared ownedBlocks set
1116
+ gcEligible.add(blockId); // confirmed replicated → safe to sweep
1117
+ }
1118
+ }).catch((err) => {
1119
+ log?.('rebalance reaction failed: %o', err);
1120
+ });
1121
+ });
1122
+
1123
+ // Ring-shift handoff (advertise→confirm→release). It needs the confirmer (this
1124
+ // coordinator) and the cohort-size floor (this monitor), so it is wired here. The
1125
+ // `onRelease` callback runs Phase C's local effect: stop serving/spreading the shed
1126
+ // range and mark it GC-eligible — the same authoritative eviction the confirmed-rebalance
1127
+ // release performs.
1128
+ ringShift = new RingShiftCoordinator({
1129
+ fretAdapter,
1130
+ ringSelector,
1131
+ fret,
1132
+ partitionDetector,
1133
+ confirmer: coordinator,
1134
+ ownedBlocks,
1135
+ selfPeerId: peerId,
1136
+ getFloor: () => rebalanceMonitor.getCohortSize(),
1137
+ onRelease: (blockIds) => {
1138
+ for (const blockId of blockIds) {
1139
+ rebalanceMonitor.untrackBlock(blockId);
1140
+ gcEligible.add(blockId);
1141
+ }
1086
1142
  }
1087
- }).catch((err) => {
1088
- log?.('rebalance reaction failed: %o', err);
1089
1143
  });
1090
- });
1091
-
1092
- // Ring-shift handoff (advertise→confirm→release). It needs the confirmer (this
1093
- // coordinator) and the cohort-size floor (this monitor), so it is wired here. The
1094
- // `onRelease` callback runs Phase C's local effect: stop serving/spreading the shed
1095
- // range and mark it GC-eligible the same authoritative eviction the confirmed-rebalance
1096
- // release performs.
1097
- ringShift = new RingShiftCoordinator({
1098
- fretAdapter,
1099
- ringSelector,
1100
- fret,
1101
- partitionDetector,
1102
- confirmer: coordinator,
1103
- ownedBlocks,
1104
- selfPeerId: peerId,
1105
- getFloor: () => rebalanceMonitor.getCohortSize(),
1106
- onRelease: (blockIds) => {
1107
- for (const blockId of blockIds) {
1108
- rebalanceMonitor.untrackBlock(blockId);
1109
- gcEligible.add(blockId);
1144
+ // Reconcile any stale `moving` advertisement left by a crash mid-handoff (no-op unless
1145
+ // arachnode metadata survived a restart still marked `moving`).
1146
+ ringShift.reconcileOnStart();
1147
+
1148
+ // Feed owned blocks via the SINGLE shared feed (idempotent — already live if the spread
1149
+ // block above wired it). Both monitors read the same ownedBlocks set this populates.
1150
+ ensureOwnedBlockFeed();
1151
+
1152
+ // Expose for tests/diagnostics (mirrors node.spreadOnChurnMonitor).
1153
+ (node as any).rebalanceMonitor = rebalanceMonitor;
1154
+ (node as any).blockTransferCoordinator = coordinator;
1155
+ (node as any).ringShiftCoordinator = ringShift;
1156
+
1157
+ // Disposal: stop the monitor before transports close. Composes with the other stop
1158
+ // wrappers (each calls its captured previousStop last). Idempotent — RebalanceMonitor.stop()
1159
+ // early-returns when not running (NetworkManagerService.stop() also stops it). The shared
1160
+ // owned-block feed teardown is the separate up-front wrapper (not duplicated here).
1161
+ const previousStop = node.stop.bind(node);
1162
+ node.stop = async () => {
1163
+ try {
1164
+ await rebalanceMonitor.stop();
1165
+ } finally {
1166
+ await previousStop();
1110
1167
  }
1111
- }
1112
- });
1113
- // Reconcile any stale `moving` advertisement left by a crash mid-handoff (no-op unless
1114
- // arachnode metadata survived a restart still marked `moving`).
1115
- ringShift.reconcileOnStart();
1116
-
1117
- // Feed owned blocks via the SINGLE shared feed (idempotent — already live if the spread
1118
- // block above wired it). Both monitors read the same ownedBlocks set this populates.
1119
- ensureOwnedBlockFeed();
1120
-
1121
- // Expose for tests/diagnostics (mirrors node.spreadOnChurnMonitor).
1122
- (node as any).rebalanceMonitor = rebalanceMonitor;
1123
- (node as any).blockTransferCoordinator = coordinator;
1124
- (node as any).ringShiftCoordinator = ringShift;
1125
-
1126
- // Disposal: stop the monitor before transports close. Composes with the other stop
1127
- // wrappers (each calls its captured previousStop last). Idempotent RebalanceMonitor.stop()
1128
- // early-returns when not running (NetworkManagerService.stop() also stops it). The shared
1129
- // owned-block feed teardown is the separate up-front wrapper (not duplicated here).
1130
- const previousStop = node.stop.bind(node);
1131
- node.stop = async () => {
1168
+ };
1169
+ } catch (err) {
1170
+ // Rebalance is a resilience optimization, not a correctness requirement - a wiring
1171
+ // failure (e.g. FRET briefly unavailable) must NOT hard-fail node startup.
1172
+ log?.('rebalance wiring init failed: %o', err);
1173
+ }
1174
+ }
1175
+
1176
+ // Monitor capacity and adjust ring periodically. The damped `shouldTransition()` decides
1177
+ // WHETHER/where to move (docs/arachnode-ring-handoff.md § Part 1); the RingShiftCoordinator
1178
+ // carries the move out through the advertise→confirm→release handoff (§ Part 2) so a shift
1179
+ // never drops a key below its replication floor. The old unilateral `setArachnodeInfo` flip —
1180
+ // which changed advertised responsibility instantly with no data handoff — is gone.
1181
+ //
1182
+ // Ring shifts run ONLY when `ringShift` is wired (i.e. the rebalance reaction is enabled): a
1183
+ // move-out is unsafe without the confirm/release path, so a node with the rebalance reaction
1184
+ // disabled stays at its bootstrap ring rather than flipping unsafely.
1185
+ const monitorInterval = setInterval(async () => {
1186
+ if (!ringShift) return;
1187
+ const transition = await ringSelector.shouldTransition();
1188
+ if (transition.shouldMove && transition.direction && transition.newRingDepth !== undefined) {
1189
+ log?.('Ring transition needed: moving %s to Ring %d', transition.direction, transition.newRingDepth);
1132
1190
  try {
1133
- await rebalanceMonitor.stop();
1191
+ const outcome = await ringShift.executeShift({
1192
+ direction: transition.direction,
1193
+ newRingDepth: transition.newRingDepth
1194
+ });
1195
+ log?.('Ring shift outcome: %o', outcome);
1196
+ } catch (err) {
1197
+ log?.('Ring shift failed: %o', err);
1134
1198
  } finally {
1135
- await previousStop();
1199
+ // Measure the minimum dwell from the SETTLED shift (completed or rolled back), not
1200
+ // just the trigger stamped inside shouldTransition (docs/arachnode-ring-handoff.md §1.3).
1201
+ ringSelector.recordShiftSettled();
1136
1202
  }
1137
- };
1138
- } catch (err) {
1139
- // Rebalance is a resilience optimization, not a correctness requirement - a wiring
1140
- // failure (e.g. FRET briefly unavailable) must NOT hard-fail node startup.
1141
- log?.('rebalance wiring init failed: %o', err);
1142
- }
1143
- }
1144
-
1145
- // Monitor capacity and adjust ring periodically. The damped `shouldTransition()` decides
1146
- // WHETHER/where to move (docs/arachnode-ring-handoff.md § Part 1); the RingShiftCoordinator
1147
- // carries the move out through the advertise→confirm→release handoff (§ Part 2) so a shift
1148
- // never drops a key below its replication floor. The old unilateral `setArachnodeInfo` flip —
1149
- // which changed advertised responsibility instantly with no data handoff — is gone.
1150
- //
1151
- // Ring shifts run ONLY when `ringShift` is wired (i.e. the rebalance reaction is enabled): a
1152
- // move-out is unsafe without the confirm/release path, so a node with the rebalance reaction
1153
- // disabled stays at its bootstrap ring rather than flipping unsafely.
1154
- const monitorInterval = setInterval(async () => {
1155
- if (!ringShift) return;
1156
- const transition = await ringSelector.shouldTransition();
1157
- if (transition.shouldMove && transition.direction && transition.newRingDepth !== undefined) {
1158
- log?.('Ring transition needed: moving %s to Ring %d', transition.direction, transition.newRingDepth);
1159
- try {
1160
- const outcome = await ringShift.executeShift({
1161
- direction: transition.direction,
1162
- newRingDepth: transition.newRingDepth
1163
- });
1164
- log?.('Ring shift outcome: %o', outcome);
1165
- } catch (err) {
1166
- log?.('Ring shift failed: %o', err);
1167
- } finally {
1168
- // Measure the minimum dwell from the SETTLED shift (completed or rolled back), not
1169
- // just the trigger stamped inside shouldTransition (docs/arachnode-ring-handoff.md §1.3).
1170
- ringSelector.recordShiftSettled();
1171
1203
  }
1172
- }
1173
- }, 60_000);
1204
+ }, 60_000);
1174
1205
 
1175
- // Cleanup on node stop
1176
- const originalStop = node.stop.bind(node);
1206
+ // Cleanup on node stop
1207
+ const originalStop = node.stop.bind(node);
1208
+ node.stop = async () => {
1209
+ clearInterval(monitorInterval);
1210
+ await originalStop();
1211
+ };
1212
+ } else {
1213
+ log?.('FRET service not available, Arachnode disabled');
1214
+ }
1215
+ }
1216
+
1217
+ // --- Seed the shared owned-block set from already-durable storage ---
1218
+ // Blocks durable from a previous run are otherwise untracked until next touched (see the
1219
+ // onAnyCollectionChange comment above where ownedBlocks is declared). Placed here, AFTER both
1220
+ // monitor-wiring blocks (spread ~line 862, rebalance ~line 974) have had their chance to call
1221
+ // ensureOwnedBlockFeed():
1222
+ // - Gate on offOwnedBlockFeed: only seed when a monitor actually consumes ownedBlocks; if both
1223
+ // are disabled the set is unused and the scan (plus the background task) is wasted work.
1224
+ // - Feed-before-scan ordering is load-bearing: because the feed is already live, a block
1225
+ // committed/replicated DURING the scan is caught by the feed; Set.add is idempotent so the
1226
+ // overlap is harmless. Scanning before subscribing would drop a block committed in the gap.
1227
+ // - Fire-and-forget so a large store never blocks startup; the .catch keeps a scan rejection
1228
+ // from becoming an unhandled rejection.
1229
+ // - Cancellable: a stop wrapper flips seedStopping so the scan loop breaks against a
1230
+ // stopping/closing backend rather than running the enumeration to completion.
1231
+ // NOTE: a concurrent rebalance release can untrackBlock (delete from ownedBlocks) a confirmed-
1232
+ // released block while this scan is still running, and the scan could then re-add that id. Benign
1233
+ // transient: the block is still in the metadata store (no sweep reclaims metadata yet), so a
1234
+ // re-added released block is simply re-evaluated and re-released on the next rebalance tick. Right
1235
+ // after a restart, responsibility-loss detection lags this fast metadata scan, so the window is
1236
+ // small. Accepted rather than synchronized.
1237
+ if (offOwnedBlockFeed && typeof rawStorage.listBlockIds === 'function') {
1238
+ let seedStopping = false;
1239
+ const previousStop = node.stop.bind(node);
1177
1240
  node.stop = async () => {
1178
- clearInterval(monitorInterval);
1179
- await originalStop();
1241
+ seedStopping = true;
1242
+ await previousStop();
1180
1243
  };
1181
- } else {
1182
- log?.('FRET service not available, Arachnode disabled');
1244
+ void seedOwnedBlocksFromStorage(rawStorage, ownedBlocks, () => seedStopping)
1245
+ .catch((err) => ((node as any).logger?.forComponent?.('db-p2p:owned-block-seed'))?.('seed failed: %o', err));
1183
1246
  }
1184
- }
1185
1247
 
1186
- // --- Seed the shared owned-block set from already-durable storage ---
1187
- // Blocks durable from a previous run are otherwise untracked until next touched (see the
1188
- // onAnyCollectionChange comment above where ownedBlocks is declared). Placed here, AFTER both
1189
- // monitor-wiring blocks (spread ~line 862, rebalance ~line 974) have had their chance to call
1190
- // ensureOwnedBlockFeed():
1191
- // - Gate on offOwnedBlockFeed: only seed when a monitor actually consumes ownedBlocks; if both
1192
- // are disabled the set is unused and the scan (plus the background task) is wasted work.
1193
- // - Feed-before-scan ordering is load-bearing: because the feed is already live, a block
1194
- // committed/replicated DURING the scan is caught by the feed; Set.add is idempotent so the
1195
- // overlap is harmless. Scanning before subscribing would drop a block committed in the gap.
1196
- // - Fire-and-forget so a large store never blocks startup; the .catch keeps a scan rejection
1197
- // from becoming an unhandled rejection.
1198
- // - Cancellable: a stop wrapper flips seedStopping so the scan loop breaks against a
1199
- // stopping/closing backend rather than running the enumeration to completion.
1200
- // NOTE: a concurrent rebalance release can untrackBlock (delete from ownedBlocks) a confirmed-
1201
- // released block while this scan is still running, and the scan could then re-add that id. Benign
1202
- // transient: the block is still in the metadata store (no sweep reclaims metadata yet), so a
1203
- // re-added released block is simply re-evaluated and re-released on the next rebalance tick. Right
1204
- // after a restart, responsibility-loss detection lags this fast metadata scan, so the window is
1205
- // small. Accepted rather than synchronized.
1206
- if (offOwnedBlockFeed && typeof rawStorage.listBlockIds === 'function') {
1207
- let seedStopping = false;
1208
- const previousStop = node.stop.bind(node);
1209
- node.stop = async () => {
1210
- seedStopping = true;
1211
- await previousStop();
1212
- };
1213
- void seedOwnedBlocksFromStorage(rawStorage, ownedBlocks, () => seedStopping)
1214
- .catch((err) => ((node as any).logger?.forComponent?.('db-p2p:owned-block-seed'))?.('seed failed: %o', err));
1215
- }
1248
+ // [dispute-subsystem-dormant] The DisputeService object is constructed below so tests and
1249
+ // getDisputeStatus() work, but it is unreachable from the live network path:
1250
+ // - No inbound handler: disputeProtocolService is NOT in the services map above.
1251
+ // - onInvalidation is deliberately unset: maybeInvalidate() is a no-op on live nodes.
1252
+ // - revalidate is deliberately unset: handleChallenge always votes inconclusive on live nodes.
1253
+ // Full activation requires arbitrator-set anchoring before a forged synthetic cohort can pass resolution.
1254
+ // Gate: tickets/backlog/hardening/invalidation-live-wiring-requires-arbitrator-set-anchoring
1255
+ // Wiring plan: tickets/backlog/feat-dispute-subsystem-live-activation
1256
+ // Initialize dispute service if enabled
1257
+ let disputeServiceInstance: DisputeService | undefined;
1258
+ if (options.dispute?.disputeEnabled) {
1259
+ const createDisputeClient = (peerId: any) => DisputeClient.create(peerId, keyNetwork, protocolPrefix);
1260
+ disputeServiceInstance = new DisputeService({
1261
+ peerId: node.peerId,
1262
+ privateKey: nodePrivateKey,
1263
+ peerNetwork: keyNetwork,
1264
+ createDisputeClient,
1265
+ reputation,
1266
+ validator: options.validator,
1267
+ config: options.dispute,
1268
+ selectArbitrators: async (blockId: string, excludePeers: string[], count: number, round: number, epoch: Uint8Array) => {
1269
+ const { hashKey: fretHashKey } = await import('p2p-fret');
1270
+ const fret = (node as any).services?.fret as FretService | undefined;
1271
+ if (!fret) return [];
1272
+ // Dispersed sampling: draw `count` peers from coordinates spread across the whole keyspace
1273
+ // (hash(blockId ‖ round ‖ epoch ‖ i)) rather than the block's XOR neighborhood, so an attacker
1274
+ // who owns the block's locale does not thereby own the arbitrators. `assembleCohort` already
1275
+ // filters to known members; excluding the original cluster + self keeps arbitrators independent.
1276
+ const excludeSet = new Set(excludePeers);
1277
+ // NOTE: adding the local node's own id to `exclude` makes the draw node-relative. Cross-node
1278
+ // determinism (the verifiable-recompute property) holds today only because the dissent
1279
+ // coordinator running this is itself a member of the original cluster, so `self` is already in
1280
+ // `excludePeers` — the add is a no-op and every honest node excludes the identical set. When a
1281
+ // verify-path recompute lands, it MUST reconstruct `exclude` from the challenger's identity
1282
+ // (`proof.challengerPeerId`) + original cluster, never the verifier's own id, or re-derivation diverges.
1283
+ excludeSet.add(node.peerId.toString());
1284
+ const picks = await sampleArbitrators(
1285
+ { blockId: new TextEncoder().encode(blockId), round, epoch, count, exclude: excludeSet },
1286
+ (coord, wants) => fret.assembleCohort(coord, wants) as string[],
1287
+ fretHashKey,
1288
+ );
1289
+ return picks.map(pid => peerIdFromString(pid));
1290
+ },
1291
+ });
1292
+ }
1216
1293
 
1217
- // [dispute-subsystem-dormant] The DisputeService object is constructed below so tests and
1218
- // getDisputeStatus() work, but it is unreachable from the live network path:
1219
- // - No inbound handler: disputeProtocolService is NOT in the services map above.
1220
- // - onInvalidation is deliberately unset: maybeInvalidate() is a no-op on live nodes.
1221
- // - revalidate is deliberately unset: handleChallenge always votes inconclusive on live nodes.
1222
- // Full activation requires arbitrator-set anchoring before a forged synthetic cohort can pass resolution.
1223
- // Gate: tickets/backlog/hardening/invalidation-live-wiring-requires-arbitrator-set-anchoring
1224
- // Wiring plan: tickets/backlog/feat-dispute-subsystem-live-activation
1225
- // Initialize dispute service if enabled
1226
- let disputeServiceInstance: DisputeService | undefined;
1227
- if (options.dispute?.disputeEnabled) {
1228
- const createDisputeClient = (peerId: any) => DisputeClient.create(peerId, keyNetwork, protocolPrefix);
1229
- disputeServiceInstance = new DisputeService({
1230
- peerId: node.peerId,
1231
- privateKey: nodePrivateKey,
1232
- peerNetwork: keyNetwork,
1233
- createDisputeClient,
1294
+ // The host-facing attachment surface, declared once in `optimystic-node.ts` and written here
1295
+ // through ONE object literal so every field is type-checked AND a field added to
1296
+ // `OptimysticNodeAttachments` but never assigned here is a compile error rather than an
1297
+ // `undefined` a host reads as present. Keeping it typed is load-bearing: when
1298
+ // `node.keyNetwork` was reachable only through a cast, three hosts found it easier to build a
1299
+ // SECOND Libp2pKeyPeerNetwork from constructor defaults a different cohort width and no
1300
+ // network-membership filter than this node's own consensus path uses for the same key
1301
+ // (ticket bug-second-key-network-built-with-defaults).
1302
+ const attachments: OptimysticNodeAttachments = {
1303
+ coordinatedRepo,
1304
+ storageRepo,
1305
+ // The StorageRepo is the single commit funnel for both the coordinated and
1306
+ // direct paths, so it is the node's per-collection change-notifier origin. This is the
1307
+ // default; the cohort-topic activation block below REPLACES it with the origination-decorating
1308
+ // bridge notifier when the substrate is enabled.
1309
+ blockChangeNotifier: storageRepo,
1310
+ keyNetwork,
1234
1311
  reputation,
1235
- validator: options.validator,
1236
- config: options.dispute,
1237
- selectArbitrators: async (blockId: string, excludePeers: string[], count: number, round: number, epoch: Uint8Array) => {
1238
- const { hashKey: fretHashKey } = await import('p2p-fret');
1239
- const fret = (node as any).services?.fret as FretService | undefined;
1240
- if (!fret) return [];
1241
- // Dispersed sampling: draw `count` peers from coordinates spread across the whole keyspace
1242
- // (hash(blockId ‖ round ‖ epoch ‖ i)) rather than the block's XOR neighborhood, so an attacker
1243
- // who owns the block's locale does not thereby own the arbitrators. `assembleCohort` already
1244
- // filters to known members; excluding the original cluster + self keeps arbitrators independent.
1245
- const excludeSet = new Set(excludePeers);
1246
- // NOTE: adding the local node's own id to `exclude` makes the draw node-relative. Cross-node
1247
- // determinism (the verifiable-recompute property) holds today only because the dissent
1248
- // coordinator running this is itself a member of the original cluster, so `self` is already in
1249
- // `excludePeers` — the add is a no-op and every honest node excludes the identical set. When a
1250
- // verify-path recompute lands, it MUST reconstruct `exclude` from the challenger's identity
1251
- // (`proof.challengerPeerId`) + original cluster, never the verifier's own id, or re-derivation diverges.
1252
- excludeSet.add(node.peerId.toString());
1253
- const picks = await sampleArbitrators(
1254
- { blockId: new TextEncoder().encode(blockId), round, epoch, count, exclude: excludeSet },
1255
- (coord, wants) => fret.assembleCohort(coord, wants) as string[],
1256
- fretHashKey,
1257
- );
1258
- return picks.map(pid => peerIdFromString(pid));
1259
- },
1260
- });
1261
- }
1262
-
1263
- // Cleanup cluster member intervals on node stop
1264
- {
1265
- const previousStop = node.stop.bind(node);
1266
- node.stop = async () => {
1267
- (clusterImpl as import('./cluster/cluster-repo.js').ClusterMember).dispose();
1268
- await previousStop();
1312
+ disputeService: disputeServiceInstance,
1313
+ // The node's libp2p Ed25519 identity key. Exposed on the same attachment surface as
1314
+ // coordinatedRepo/keyNetwork so a host can bind a client-transaction signer to it (the Quereus
1315
+ // collection-factory's getSigner reuses this via signPeer). libp2p does not surface the private
1316
+ // key on its public `Libp2p` interface, so this attachment is the sanctioned in-process handle.
1317
+ // Ed25519 by construction (options.privateKey defaults to generateKeyPair('Ed25519')).
1318
+ peerPrivateKey: nodePrivateKey,
1269
1319
  };
1270
- }
1271
-
1272
- // Expose coordinated repo and storage for external use
1273
- (node as any).coordinatedRepo = coordinatedRepo;
1274
- (node as any).storageRepo = storageRepo;
1275
- // The StorageRepo is the single commit funnel for both the coordinated and
1276
- // direct paths, so it is the node's per-collection change-notifier origin. This is the
1277
- // default; the cohort-topic activation block below REPLACES it with the origination-decorating
1278
- // bridge notifier when the substrate is enabled.
1279
- (node as any).blockChangeNotifier = storageRepo;
1280
- (node as any).keyNetwork = keyNetwork;
1281
- (node as any).reputation = reputation;
1282
- (node as any).disputeService = disputeServiceInstance;
1283
- // The node's libp2p Ed25519 identity key. Exposed on the same `(node as any).*` surface as
1284
- // coordinatedRepo/keyNetwork so a host can bind a client-transaction signer to it (the Quereus
1285
- // collection-factory's getSigner reuses this via signPeer). libp2p does not surface the private
1286
- // key on its public `Libp2p` interface, so this attachment is the sanctioned in-process handle.
1287
- // Ed25519 by construction (options.privateKey defaults to generateKeyPair('Ed25519')).
1288
- (node as any).peerPrivateKey = nodePrivateKey;
1289
-
1290
- // --- Cohort-topic origination activation (post-node: consumes the fully-assembled node + FRET) ---
1291
- // This is the only place that is after the node + FRET are assembled (node.start() done, fretSvc
1292
- // available) yet before any caller can capture `blockChangeNotifier` — the Quereus collection-factory
1293
- // captures it once, immediately after createLibp2pNode returns, and reuses that reference as
1294
- // `localChangeNotifier` for every NetworkTransactor it builds. Installing the bridge here makes the
1295
- // origination path live for ALL collections created on the node.
1296
- if (cohortEnabled) {
1297
- // The host needs the full FRET engine surface; node.services.fret is the wrapper (see resolveFretEngine).
1298
- const fret = resolveFretEngine(fretSvc);
1299
- if (!fret) {
1300
- // Operator opted in; degrading silently to the bare notifier would hide misconfiguration.
1301
- // The node has already started (transports open, FRET running), so tear it down before the
1302
- // hard-fail rather than leaking a started node + open transports on the rejection.
1303
- await node.stop();
1304
- throw new Error('cohortTopic enabled but the FRET service is unavailable on the node');
1305
- }
1320
+ Object.assign(node, attachments);
1321
+
1322
+ // --- Cohort-topic origination activation (post-node: consumes the fully-assembled node + FRET) ---
1323
+ // This is the only place that is after the node + FRET are assembled (node.start() done, fretSvc
1324
+ // available) yet before any caller can capture `blockChangeNotifier` — the Quereus collection-factory
1325
+ // captures it once, immediately after createLibp2pNode returns, and reuses that reference as
1326
+ // `localChangeNotifier` for every NetworkTransactor it builds. Installing the bridge here makes the
1327
+ // origination path live for ALL collections created on the node.
1328
+ if (cohortEnabled) {
1329
+ // The host needs the full FRET engine surface; node.services.fret is the wrapper (see resolveFretEngine).
1330
+ const fret = resolveFretEngine(fretSvc);
1331
+ if (!fret) {
1332
+ // Operator opted in; degrading silently to the bare notifier would hide misconfiguration.
1333
+ // (The started node is torn down by the post-start rollback `catch` at the bottom of this function.)
1334
+ throw new Error('cohortTopic enabled but the FRET service is unavailable on the node');
1335
+ }
1306
1336
 
1307
- // A host-construction failure also hard-fails (operator opted in); stop the started node first so
1308
- // the rejection does not leak open transports / a running FRET service. node.stop() runs the
1309
- // already-installed arachnode + clusterMember teardown wrappers and closes the node's connections.
1310
- let host: Awaited<ReturnType<typeof createCohortTopicHost>>;
1311
- try {
1312
- host = await createCohortTopicHost(node, fret, {
1337
+ const host = await createCohortTopicHost(node, fret, {
1313
1338
  ...(options.cohortTopic!.host ?? {}),
1314
1339
  // Wire the node's reputation service in as the production backing for the bootstrap-evidence
1315
1340
  // referee verifier (the `{ isBanned, getScore }` view `PeerReputationService` satisfies), so a
@@ -1328,264 +1353,299 @@ export async function createLibp2pNodeBase(
1328
1353
  privateKey: nodePrivateKey, // real k − x threshold signing
1329
1354
  wantK: cohortWantK,
1330
1355
  });
1331
- } catch (err) {
1332
- await node.stop();
1333
- throw err;
1334
- }
1335
1356
 
1336
- // selfIsCohortMember: this node owns the collection's reactivity-topic fan-out iff it is in the
1337
- // FRET cohort around coord_0(H(currentTailId "reactivity")). Uses db-core's default hashes
1338
- // (createReactivityTopicAnchor / createTierAddressing / createRingHash), byte-identical to the
1339
- // host's internal `new RingHash()` and the subscriber-side anchor, and the SAME cohortWantK as
1340
- // the host so the coord + cohort line up across origination and subscription.
1341
- const selfIsCohortMember = createReactivitySelfMembershipGate({
1342
- fret,
1343
- selfPeerId: node.peerId.toString(),
1344
- wantK: cohortWantK,
1345
- });
1346
-
1347
- const { unsubscribe } = attachCohortChangeBridge(
1348
- node as unknown as { blockChangeNotifier?: IBlockChangeNotifier },
1357
+ // --- Cohort-topic + reactivity + matchmaking teardown ---
1358
+ // Installed HERE, immediately after `host` exists and BEFORE the ~230 lines of reactivity /
1359
+ // matchmaking wiring below, because the post-start rollback only unwinds resources whose stop
1360
+ // wrapper is already installed at the moment of the throw. With the wrapper at the END of the
1361
+ // block (where it used to live) a throw mid-wiring left the host's gossip timer and cohort-topic
1362
+ // protocol handlers running. The bindings it releases are therefore declared up front and
1363
+ // undefined-guarded — same idiom as `offOwnedBlockFeed` above — so this tears down exactly what
1364
+ // has been created so far, whether that is the host alone or the whole wiring.
1365
+ //
1366
+ // Ordering (load-bearing): release reactivity timers + protocol handlers BEFORE host.stop()
1367
+ // (which clears the cohort gossip timer + unhandles the cohort-topic protocols) BEFORE the node's
1368
+ // transports close (previousStop). Composes with the existing arachnode + clusterMember stop
1369
+ // wrappers (each calls its captured previousStop last). `node.unhandle` on a protocol that was
1370
+ // never registered does not throw — libp2p's registrar deletes each id from its handler map
1371
+ // (a miss is silently ignored) and then re-patches the peer store's advertised protocol list —
1372
+ // so the handler releases need no separate registration flags.
1373
+ const reactivityProtocols = DEFAULT_REACTIVITY_PROTOCOLS;
1374
+ const matchmakingProtocols = DEFAULT_MATCHMAKING_PROTOCOLS;
1375
+ let unsubscribeCohortBridge: (() => void) | undefined;
1376
+ let offInboundNotify: (() => void) | undefined;
1377
+ let pushStateGossip: ReactivityPushStateGossipDriver | undefined;
1378
+ let reactivityRotation: RotationReRegistrationScheduler | undefined;
1349
1379
  {
1350
- source: storageRepo,
1351
- service: host.service,
1352
- selfIsCohortMember,
1353
- extractCommitCert: makeClusterCommitCertExtractor(certStore!),
1354
- },
1355
- );
1356
-
1357
- // Expose the host so the reactivity origination wiring (and the activation test) can install
1358
- // `CohortTopicService.onLocalCommit`.
1359
- (node as any).cohortTopicHost = host;
1360
-
1361
- // --- Reactivity notification transport (origination → fan-out → inbound delivery → push-state gossip) ---
1362
- // Compose notify + forwarder-host + push-state-gossip onto the cohort-topic host so a committed change
1363
- // on a tail-cohort member actually reaches subscribers on OTHER nodes over real sockets. The change
1364
- // bridge above fires `onLocalCommit`; this is what the emitted notifications travel over.
1365
- // (docs/reactivity.md §Notification origination / §Propagation.) Reactivity reuses the canonical,
1366
- // network-agnostic protocol IDs, matching the cohort-topic family's production default.
1367
- const selfPeerId = node.peerId.toString();
1368
- const reactivityProtocols = DEFAULT_REACTIVITY_PROTOCOLS;
1369
- const reactivityProfile = host.profile; // Edge ⇒ subscriber-only via the policy gate; Core forwards.
1370
- const reactivityPolicy = reactivityNodePolicy(reactivityProfile);
1371
- // db-core default anchor + tier addressing, byte-identical to the host's `new RingHash()`, the
1372
- // origination gate, and the subscriber-side anchor — so coord_0 derivation lines up everywhere.
1373
- const reactivityAddressing = createTierAddressing(createRingHash());
1374
- // Reactivity's forwarder cohort sits at coord_0 — TREE tier 0 (peer-independent), distinct from the
1375
- // CAPACITY tier T3 the verifier/willingness use. `registry.findServing` keys on the engine's tree
1376
- // depth, so the served reactivity engine is found at tree tier 0, never at 3.
1377
- const REACTIVITY_FORWARDER_TREE_TIER = 0;
1378
-
1379
- // Node-level subscriber registry: a constructed ReactivitySubscriptionManager registers here so a
1380
- // socket-delivered NotificationV1 reaches it. (The Quereus Database.watch → manager bridge that
1381
- // CONSTRUCTS managers stays the backlog item optimystic-network-reactive-watch-integration-test.)
1382
- const reactivitySubscribers = new ReactivitySubscriberRegistry();
1383
- (node as any).reactivitySubscribers = reactivitySubscribers;
1384
-
1385
- // 1. Notify transport — unicast NotificationV1 send + inbound subscribe. selfPeerId guards self-dials.
1386
- const notify = new Libp2pReactivityNotifyTransport(node, { selfPeerId });
1387
-
1388
- // 2. Forwarder host — turns the forward decision into live fan-out over the notify transport.
1389
- const forwarderHost = new ReactivityForwarderHost({
1390
- transport: notify,
1391
- selfPeerId,
1392
- profile: reactivityProfile,
1393
- pushStateInit: (topicId: Uint8Array, n: NotificationV1): PushStateInit => ({
1394
- collectionId: n.collectionId,
1395
- topicId: bytesToB64url(topicId),
1396
- tailIdAtJoin: n.tailId,
1397
- deltaMaxBytes: reactivityPolicy.deltaMaxBytes,
1398
- }),
1399
- verifierFor: (): NotificationVerifier => createNotificationVerifier({ verifier: host.service.verifier(), tier: Tier.T3 }),
1400
- directSubscribers: (topicId: Uint8Array): string[] => {
1401
- // Find the served reactivity engine at TREE tier 0 (see REACTIVITY_FORWARDER_TREE_TIER) and read
1402
- // its direct-subscriber records. The adapter filters to reactivity appState and maps participantId
1403
- // bytes → dialable peer-id strings (the transport's `peerIdFromString` space) — NOT base64url,
1404
- // which would silently fail to dial. `undefined` (no subscriber has registered here yet) ⇒ [].
1405
- const engine = host.registry.findServing(topicId, REACTIVITY_FORWARDER_TREE_TIER);
1406
- return engine === undefined ? [] : reactivityDirectSubscribers(engine, topicId);
1407
- },
1408
- // No childCohorts until cohort-topic-parent-child-link populates PushState.childCohorts (single
1409
- // tier-0 reach today); wire the resolver anyway. A child cohort's primary is the FRET-nearest member
1410
- // of its coord, returned as a peer-id string (the dial space).
1411
- resolveChildPrimary: (ref: CohortRef): string | undefined => {
1412
- const peers = fret.assembleCohort(b64urlToBytes(ref.coord), cohortWantK);
1413
- return peers.length > 0 ? peers[0] : undefined;
1414
- },
1415
- deliverLocal: (topicId: Uint8Array, n: NotificationV1): void => reactivitySubscribers.deliver(topicId, n),
1416
- });
1380
+ const previousStop = node.stop.bind(node);
1381
+ node.stop = async (): Promise<void> => {
1382
+ try {
1383
+ reactivityRotation?.stop();
1384
+ pushStateGossip?.stop();
1385
+ offInboundNotify?.();
1386
+ await node.unhandle(reactivityProtocolList(reactivityProtocols));
1387
+ await node.unhandle(matchmakingProtocolList(matchmakingProtocols));
1388
+ unsubscribeCohortBridge?.();
1389
+ await host.stop();
1390
+ } finally {
1391
+ await previousStop();
1392
+ }
1393
+ };
1394
+ }
1417
1395
 
1418
- // Inbound notify frames forwarder host (subscriber role delivers in-process; forwarder role fans out).
1419
- registerNotifyHandler(node, reactivityProtocols.notify, notify);
1420
- const offInboundNotify = notify.onNotification((from, n): void => { void forwarderHost.onInbound(from, n); });
1396
+ // selfIsCohortMember: this node owns the collection's reactivity-topic fan-out iff it is in the
1397
+ // FRET cohort around coord_0(H(currentTailId ‖ "reactivity")). Uses db-core's default hashes
1398
+ // (createReactivityTopicAnchor / createTierAddressing / createRingHash), byte-identical to the
1399
+ // host's internal `new RingHash()` and the subscriber-side anchor, and the SAME cohortWantK as
1400
+ // the host — so the coord + cohort line up across origination and subscription.
1401
+ const selfIsCohortMember = createReactivitySelfMembershipGate({
1402
+ fret,
1403
+ selfPeerId: node.peerId.toString(),
1404
+ wantK: cohortWantK,
1405
+ });
1421
1406
 
1422
- // 3. Origination emit — install onLocalCommit: a member commit builds a NotificationV1 and ingests it.
1423
- const origination = new ReactivityOriginationManager({
1424
- service: host.service,
1425
- resolveContext: (event) => {
1426
- if (event.tailId === undefined) {
1427
- return undefined; // tail-less (read-driven promotion) never originates (the gate also returns first)
1428
- }
1429
- return {
1430
- // MUST reuse the gate's `reactivityTailBytes` (utf8), NOT db-core's double-hashing
1431
- // blockIdToBytes — else origination derives a different coord than subscribers resolve.
1432
- tailId: reactivityTailBytes(event.tailId),
1407
+ unsubscribeCohortBridge = attachCohortChangeBridge(
1408
+ node as unknown as { blockChangeNotifier?: IBlockChangeNotifier },
1409
+ {
1410
+ source: storageRepo,
1411
+ service: host.service,
1412
+ selfIsCohortMember,
1413
+ extractCommitCert: makeClusterCommitCertExtractor(certStore!),
1414
+ },
1415
+ ).unsubscribe;
1416
+
1417
+ // Expose the host so the reactivity origination wiring (and the activation test) can install
1418
+ // `CohortTopicService.onLocalCommit`.
1419
+ (node as any).cohortTopicHost = host;
1420
+
1421
+ // --- Reactivity notification transport (origination → fan-out → inbound delivery → push-state gossip) ---
1422
+ // Compose notify + forwarder-host + push-state-gossip onto the cohort-topic host so a committed change
1423
+ // on a tail-cohort member actually reaches subscribers on OTHER nodes over real sockets. The change
1424
+ // bridge above fires `onLocalCommit`; this is what the emitted notifications travel over.
1425
+ // (docs/reactivity.md §Notification origination / §Propagation.) Reactivity reuses the canonical,
1426
+ // network-agnostic protocol IDs, matching the cohort-topic family's production default.
1427
+ const selfPeerId = node.peerId.toString();
1428
+ const reactivityProfile = host.profile; // Edge ⇒ subscriber-only via the policy gate; Core forwards.
1429
+ const reactivityPolicy = reactivityNodePolicy(reactivityProfile);
1430
+ // db-core default anchor + tier addressing, byte-identical to the host's `new RingHash()`, the
1431
+ // origination gate, and the subscriber-side anchor — so coord_0 derivation lines up everywhere.
1432
+ const reactivityAddressing = createTierAddressing(createRingHash());
1433
+ // Reactivity's forwarder cohort sits at coord_0 — TREE tier 0 (peer-independent), distinct from the
1434
+ // CAPACITY tier T3 the verifier/willingness use. `registry.findServing` keys on the engine's tree
1435
+ // depth, so the served reactivity engine is found at tree tier 0, never at 3.
1436
+ const REACTIVITY_FORWARDER_TREE_TIER = 0;
1437
+
1438
+ // Node-level subscriber registry: a constructed ReactivitySubscriptionManager registers here so a
1439
+ // socket-delivered NotificationV1 reaches it. (The Quereus Database.watch → manager bridge that
1440
+ // CONSTRUCTS managers stays the backlog item optimystic-network-reactive-watch-integration-test.)
1441
+ const reactivitySubscribers = new ReactivitySubscriberRegistry();
1442
+ (node as any).reactivitySubscribers = reactivitySubscribers;
1443
+
1444
+ // 1. Notify transport — unicast NotificationV1 send + inbound subscribe. selfPeerId guards self-dials.
1445
+ const notify = new Libp2pReactivityNotifyTransport(node, { selfPeerId });
1446
+
1447
+ // 2. Forwarder host — turns the forward decision into live fan-out over the notify transport.
1448
+ const forwarderHost = new ReactivityForwarderHost({
1449
+ transport: notify,
1450
+ selfPeerId,
1451
+ profile: reactivityProfile,
1452
+ pushStateInit: (topicId: Uint8Array, n: NotificationV1): PushStateInit => ({
1453
+ collectionId: n.collectionId,
1454
+ topicId: bytesToB64url(topicId),
1455
+ tailIdAtJoin: n.tailId,
1433
1456
  deltaMaxBytes: reactivityPolicy.deltaMaxBytes,
1434
- // rotationHint stays undefined on a live node: the successor tail id is not knowable at the
1435
- // filling commit (random block ids; gated on 6.5-block-id-derivation). The authoritative,
1436
- // observable rotation signal is `event.tailId` CHANGING, which the manager observes via the
1437
- // `markRotated` binding below. (The pre-announce remains exercised in the mock-tier harness +
1438
- // the design simulator, both of which can synthesize the successor id.)
1439
- };
1440
- },
1441
- // reactivityNotificationTopicId(n) = reactivityTopicId(b64urlToBytes(n.tailId)); since
1442
- // n.tailId = b64url(reactivityTailBytes(tail)), this is the SAME topicId the gate assembled coord_0
1443
- // around and the subscriber/forwarder verifier derives — closing the encoding loop.
1444
- emit: (n): void => { void forwarderHost.ingest(reactivityNotificationTopicId(n), n); },
1445
- // Observe-rotation: when a collection's tail id changes between commits the OLD tail's reactivity
1446
- // topic has rotated. Start its drain so the recover serve begins redirecting to the new tree (the
1447
- // `reactivity-rotation-recover-redirect-drain` markRotated seam). `oldTopicId` is byte-identical to
1448
- // the topic a subscriber subscribed under (both `reactivityTopicId(reactivityTailBytes(tail))`).
1449
- markRotated: (oldTopicId, redirect, now): void => forwarderHost.markRotated(oldTopicId, redirect, now),
1450
- });
1451
- origination.install();
1452
-
1453
- // 4. PushState gossip — periodic intra-cohort convergence so any member (not just the primary) can
1454
- // serve a replay/backfill. Rides the host's cohort gossip transport (no second transport).
1455
- const pushStateGossip = new ReactivityPushStateGossipDriver({
1456
- gossipTransport: host.gossipTransport,
1457
- liveCollections: (): ReactivityGossipCollection[] => forwarderHost.livePushStates().map((pushState) => ({
1458
- pushState,
1459
- cohortCoord: reactivityAddressing.coord0(b64urlToBytes(pushState.topicId)),
1460
- })),
1461
- pushStateForGossip: (g: PushStateGossipV1) => forwarderHost.pushStateFor(b64urlToBytes(g.topicId)),
1462
- // Authenticity gate: accept gossip only from a member of the cohort around the frame's reactivity
1463
- // coord (per-frame peer-sig envelope signing is deferred — reactivity-pushstate-gossip's hardening backlog).
1464
- isCohortMember: (fromPeerId: string, g: PushStateGossipV1): boolean =>
1465
- fret.assembleCohort(reactivityAddressing.coord0(b64urlToBytes(g.topicId)), cohortWantK).includes(fromPeerId),
1466
- });
1467
- registerPushStateGossipHandler(node, reactivityProtocols.pushStateGossip, pushStateGossip);
1468
- pushStateGossip.start();
1469
-
1470
- // 5. Recover RPC — the pull companion to notify (docs/reactivity.md §Backfill RPC / §Resume). A
1471
- // subscriber that detected a gap, or woke from sleep past the live tail, asks a serving cohort member
1472
- // "what did I miss?" and is brought current over a real request-reply socket. The SERVE side is live
1473
- // here: this node answers RecoverRequestV1 frames against its live forwarder PushStates. The OUTBOUND
1474
- // transport + signers are constructed and exposed for the subscribe factory that CONSTRUCTS managers
1475
- // (the Quereus Database.watch app-bridge — backlog optimystic-network-reactive-watch-integration-test);
1476
- // no node-internal manager calls them yet, exactly as the notify subscriber side is constructed against
1477
- // `reactivitySubscribers` rather than from a watch.
1478
- //
1479
- // Node-level sticky cohort-hint cache (keyed by collectionId), shared between the outbound transport's
1480
- // sticky-primary lookup and a future manager's rotation-invalidation so both see ONE cache. It starts
1481
- // empty ⇒ the transport falls through to the cohort-walk (any member holding the gossiped PushState
1482
- // answers); populating the sticky primary is a one-RT optimization, not a correctness need.
1483
- const reactivityCohortHintCache = createStickyCohortHintCache();
1484
- // topicId → dialable cohort member peer-id strings: the SAME FRET coord_0 assembly the push-state-gossip
1485
- // authenticity gate uses (`reactivityAddressing.coord0` → `fret.assembleCohort`), so a recover walk
1486
- // reaches exactly the cohort that holds the topic's gossiped PushState. `assembleCohort` returns peer-id
1487
- // strings (the recover dialer's `peerIdFromString` space), matching the notify dial-target space.
1488
- const resolveReactivityCohort = (topicId: Uint8Array): string[] =>
1489
- fret.assembleCohort(reactivityAddressing.coord0(topicId), cohortWantK);
1490
-
1491
- // Outbound transport: exposes the db-core BackfillTransport / ResumeTransport seams against this node.
1492
- // maxBytes is omitted so the dialer + handler default to DEFAULT_STREAM_MAX_BYTES, matching the notify
1493
- // transport's default (constructed above without an override) — one frame ceiling across the family.
1494
- const recover = new Libp2pReactivityRecoverTransport({
1495
- dialer: createLibp2pRecoverDialer(node, reactivityProtocols.recover),
1496
- selfPeerId,
1497
- cohortHintCache: reactivityCohortHintCache,
1498
- resolveCohort: resolveReactivityCohort,
1499
- });
1457
+ }),
1458
+ verifierFor: (): NotificationVerifier => createNotificationVerifier({ verifier: host.service.verifier(), tier: Tier.T3 }),
1459
+ directSubscribers: (topicId: Uint8Array): string[] => {
1460
+ // Find the served reactivity engine at TREE tier 0 (see REACTIVITY_FORWARDER_TREE_TIER) and read
1461
+ // its direct-subscriber records. The adapter filters to reactivity appState and maps participantId
1462
+ // bytes → dialable peer-id strings (the transport's `peerIdFromString` space) — NOT base64url,
1463
+ // which would silently fail to dial. `undefined` (no subscriber has registered here yet) ⇒ [].
1464
+ const engine = host.registry.findServing(topicId, REACTIVITY_FORWARDER_TREE_TIER);
1465
+ return engine === undefined ? [] : reactivityDirectSubscribers(engine, topicId);
1466
+ },
1467
+ // No childCohorts until cohort-topic-parent-child-link populates PushState.childCohorts (single
1468
+ // tier-0 reach today); wire the resolver anyway. A child cohort's primary is the FRET-nearest member
1469
+ // of its coord, returned as a peer-id string (the dial space).
1470
+ resolveChildPrimary: (ref: CohortRef): string | undefined => {
1471
+ const peers = fret.assembleCohort(b64urlToBytes(ref.coord), cohortWantK);
1472
+ return peers.length > 0 ? peers[0] : undefined;
1473
+ },
1474
+ deliverLocal: (topicId: Uint8Array, n: NotificationV1): void => reactivitySubscribers.deliver(topicId, n),
1475
+ });
1500
1476
 
1501
- // Inbound serve handler: decode (bounded) verify the dialing peer's signature freshness/replay gate →
1502
- // resolve the live PushState off the forwarder host serveBackfill/serveResume reply (no reply on any
1503
- // failure; the stream aborts and the subscriber walks/chain-reads). One node-level replay guard is shared
1504
- // across all recover requests — a plain pruned-on-access map, so no new timer to tear down.
1505
- registerRecoverHandler(node, reactivityProtocols.recover, {
1506
- pushStateFor: forwarderHost.pushStateFor.bind(forwarderHost),
1507
- pushStateForCollection: forwarderHost.pushStateForCollection.bind(forwarderHost),
1508
- replayGuard: createCorrelationReplayGuard(),
1509
- rotationFor: (req, now) => {
1510
- // Drain-window redirect: a recover reaching an OLD (rotated, still-draining) tail is bounced to
1511
- // the new tree (reactivity-rotation-recover-redirect-drain). A resume carries the stale topic
1512
- // (topicId = reactivityTopicId(latestKnownTailId)); a backfill carries no topic, so resolve the
1513
- // collection's current served topic. rotationRedirectFor returns the gate's redirect while
1514
- // draining and undefined once drained (then evicting the gate + the old tail's served PushState).
1515
- const oldTopicId = req.topicId ?? resolveCurrentServedTopic(forwarderHost, req.collectionId);
1516
- return oldTopicId === undefined ? undefined : forwarderHost.rotationRedirectFor(oldTopicId, now);
1517
- },
1518
- });
1477
+ // Inbound notify frames forwarder host (subscriber role delivers in-process; forwarder role fans out).
1478
+ // NOTE: the four `register*Handler` helpers below (notify / pushStateGossip / recover /
1479
+ // matchmaking query) all call `node.handle(...)` fire-and-forget (`void`), so a rejected
1480
+ // registration escapes the post-start rollback `catch` as an UNHANDLED rejection instead of
1481
+ // failing node creation. Harmless today — every protocol id here is a fixed constant registered
1482
+ // exactly once, so the only realistic rejection is a duplicate, and that needs a caller to pass
1483
+ // overlapping custom `cohortTopic.host.protocols`. If any of these ids ever becomes
1484
+ // caller-configurable, or a helper grows a registration that can genuinely fail, make them await
1485
+ // their `node.handle` so the failure reaches the rollback.
1486
+ registerNotifyHandler(node, reactivityProtocols.notify, notify);
1487
+ offInboundNotify = notify.onNotification((from, n): void => { void forwarderHost.onInbound(from, n); });
1488
+
1489
+ // 3. Origination emit install onLocalCommit: a member commit builds a NotificationV1 and ingests it.
1490
+ const origination = new ReactivityOriginationManager({
1491
+ service: host.service,
1492
+ resolveContext: (event) => {
1493
+ if (event.tailId === undefined) {
1494
+ return undefined; // tail-less (read-driven promotion) never originates (the gate also returns first)
1495
+ }
1496
+ return {
1497
+ // MUST reuse the gate's `reactivityTailBytes` (utf8), NOT db-core's double-hashing
1498
+ // blockIdToBytes — else origination derives a different coord than subscribers resolve.
1499
+ tailId: reactivityTailBytes(event.tailId),
1500
+ deltaMaxBytes: reactivityPolicy.deltaMaxBytes,
1501
+ // rotationHint stays undefined on a live node: the successor tail id is not knowable at the
1502
+ // filling commit (random block ids; gated on 6.5-block-id-derivation). The authoritative,
1503
+ // observable rotation signal is `event.tailId` CHANGING, which the manager observes via the
1504
+ // `markRotated` binding below. (The pre-announce remains exercised in the mock-tier harness +
1505
+ // the design simulator, both of which can synthesize the successor id.)
1506
+ };
1507
+ },
1508
+ // reactivityNotificationTopicId(n) = reactivityTopicId(b64urlToBytes(n.tailId)); since
1509
+ // n.tailId = b64url(reactivityTailBytes(tail)), this is the SAME topicId the gate assembled coord_0
1510
+ // around and the subscriber/forwarder verifier derives — closing the encoding loop.
1511
+ emit: (n): void => { void forwarderHost.ingest(reactivityNotificationTopicId(n), n); },
1512
+ // Observe-rotation: when a collection's tail id changes between commits the OLD tail's reactivity
1513
+ // topic has rotated. Start its drain so the recover serve begins redirecting to the new tree (the
1514
+ // `reactivity-rotation-recover-redirect-drain` markRotated seam). `oldTopicId` is byte-identical to
1515
+ // the topic a subscriber subscribed under (both `reactivityTopicId(reactivityTailBytes(tail))`).
1516
+ markRotated: (oldTopicId, redirect, now): void => forwarderHost.markRotated(oldTopicId, redirect, now),
1517
+ });
1518
+ origination.install();
1519
+
1520
+ // 4. PushState gossip — periodic intra-cohort convergence so any member (not just the primary) can
1521
+ // serve a replay/backfill. Rides the host's cohort gossip transport (no second transport).
1522
+ pushStateGossip = new ReactivityPushStateGossipDriver({
1523
+ gossipTransport: host.gossipTransport,
1524
+ liveCollections: (): ReactivityGossipCollection[] => forwarderHost.livePushStates().map((pushState) => ({
1525
+ pushState,
1526
+ cohortCoord: reactivityAddressing.coord0(b64urlToBytes(pushState.topicId)),
1527
+ })),
1528
+ pushStateForGossip: (g: PushStateGossipV1) => forwarderHost.pushStateFor(b64urlToBytes(g.topicId)),
1529
+ // Authenticity gate: accept gossip only from a member of the cohort around the frame's reactivity
1530
+ // coord (per-frame peer-sig envelope signing is deferred — reactivity-pushstate-gossip's hardening backlog).
1531
+ isCohortMember: (fromPeerId: string, g: PushStateGossipV1): boolean =>
1532
+ fret.assembleCohort(reactivityAddressing.coord0(b64urlToBytes(g.topicId)), cohortWantK).includes(fromPeerId),
1533
+ });
1534
+ registerPushStateGossipHandler(node, reactivityProtocols.pushStateGossip, pushStateGossip);
1535
+ pushStateGossip.start();
1536
+
1537
+ // 5. Recover RPC — the pull companion to notify (docs/reactivity.md §Backfill RPC / §Resume). A
1538
+ // subscriber that detected a gap, or woke from sleep past the live tail, asks a serving cohort member
1539
+ // "what did I miss?" and is brought current over a real request-reply socket. The SERVE side is live
1540
+ // here: this node answers RecoverRequestV1 frames against its live forwarder PushStates. The OUTBOUND
1541
+ // transport + signers are constructed and exposed for the subscribe factory that CONSTRUCTS managers
1542
+ // (the Quereus Database.watch app-bridge — backlog optimystic-network-reactive-watch-integration-test);
1543
+ // no node-internal manager calls them yet, exactly as the notify subscriber side is constructed against
1544
+ // `reactivitySubscribers` rather than from a watch.
1545
+ //
1546
+ // Node-level sticky cohort-hint cache (keyed by collectionId), shared between the outbound transport's
1547
+ // sticky-primary lookup and a future manager's rotation-invalidation so both see ONE cache. It starts
1548
+ // empty ⇒ the transport falls through to the cohort-walk (any member holding the gossiped PushState
1549
+ // answers); populating the sticky primary is a one-RT optimization, not a correctness need.
1550
+ const reactivityCohortHintCache = createStickyCohortHintCache();
1551
+ // topicId → dialable cohort member peer-id strings: the SAME FRET coord_0 assembly the push-state-gossip
1552
+ // authenticity gate uses (`reactivityAddressing.coord0` → `fret.assembleCohort`), so a recover walk
1553
+ // reaches exactly the cohort that holds the topic's gossiped PushState. `assembleCohort` returns peer-id
1554
+ // strings (the recover dialer's `peerIdFromString` space), matching the notify dial-target space.
1555
+ const resolveReactivityCohort = (topicId: Uint8Array): string[] =>
1556
+ fret.assembleCohort(reactivityAddressing.coord0(topicId), cohortWantK);
1557
+
1558
+ // Outbound transport: exposes the db-core BackfillTransport / ResumeTransport seams against this node.
1559
+ // maxBytes is omitted so the dialer + handler default to DEFAULT_STREAM_MAX_BYTES, matching the notify
1560
+ // transport's default (constructed above without an override) — one frame ceiling across the family.
1561
+ const recover = new Libp2pReactivityRecoverTransport({
1562
+ dialer: createLibp2pRecoverDialer(node, reactivityProtocols.recover),
1563
+ selfPeerId,
1564
+ cohortHintCache: reactivityCohortHintCache,
1565
+ resolveCohort: resolveReactivityCohort,
1566
+ });
1519
1567
 
1520
- // The subscriber's synchronous request signers over the node's Ed25519 key (resolves the recover wiring's
1521
- // lone design point see recover-transport.ts §createRecoverRequestSigners). Fed to a manager by the
1522
- // subscribe factory alongside recover.backfillTransport(topicId, collectionId) /
1523
- // recover.resumeTransport(topicId, collectionId).
1524
- const recoverSigners = createRecoverRequestSigners(nodePrivateKey);
1525
-
1526
- // Expose the recover seams so the subscribe factory wires backfill/resume RPC + signers + the shared
1527
- // sticky cache (mirrors `reactivitySubscribers` above).
1528
- (node as any).reactivityRecover = recover;
1529
- (node as any).reactivityRecoverSigners = recoverSigners;
1530
- (node as any).reactivityCohortHintCache = reactivityCohortHintCache;
1531
-
1532
- // 6. Rotation re-registration scheduler the host timer that moves a subscriber to the rotated tree
1533
- // when its manager surfaces a `RotationNotice` (`reactivity-rotation-rereg-scheduler`). Constructed with
1534
- // the default unref'd `setTimeout` timer so an idle re-registration never pins the process. The
1535
- // `reRegister(plan)` MOVE belongs to the subscribe factory that CONSTRUCTS managers (the deferred Quereus
1536
- // `Database.watch` bridge — backlog optimystic-network-reactive-watch-integration-test): on fire it builds
1537
- // a fresh `ReactivitySubscriptionManager` under `plan.newTopicId` carrying `plan.lastRevision`, registers
1538
- // it, and swaps the `ReactivitySubscriberRegistry` entry — registering the NEW-topic handler BEFORE
1539
- // unregistering the old, so a notification mid-swap is never dropped. Until that factory lands no
1540
- // node-internal manager drives `schedule()`, so this seam is a logged no-op — exactly as 12.33 exposed
1541
- // `reactivitySubscribers` / `reactivityRecover` without a live manager constructor.
1542
- const reactivityRotation = new RotationReRegistrationScheduler({
1543
- reRegister: (plan): Promise<void> => {
1544
- reactivityWiringLog("reactivity rotation re-registration fired for successor topic=%s (lastRevision=%d) but no subscribe factory is wired yet — deferred to optimystic-network-reactive-watch-integration-test", bytesToB64url(plan.newTopicId), plan.lastRevision);
1545
- return Promise.resolve();
1546
- },
1547
- });
1548
- (node as any).reactivityRotation = reactivityRotation;
1549
-
1550
- // --- Matchmaking QueryV1 RPC — cohort serve side (docs/matchmaking.md §Seeker query) ---
1551
- // The server half of the seeker query transport: a remote seeker dials `/optimystic/matchmaking/1.0.0/query`
1552
- // and this node answers with its cohort's locally-held provider/seeker registrations, signed by the node
1553
- // peer key. Matchmaking is layered ABOVE the cohort-topic substrate, so it owns its own protocol family
1554
- // and is wired here (the composition root) over the host's PUBLIC surface only — mirroring the reactivity
1555
- // registration above; nothing reaches into host.ts internals. The OUTBOUND seeker walk client is the
1556
- // prereq follow-on `matchmaking-query-rpc-seeker-walk`; only the serve side is live here.
1557
- const matchmakingProtocols = DEFAULT_MATCHMAKING_PROTOCOLS;
1558
- registerMatchmakingQueryHandler(node, matchmakingProtocols.query, {
1559
- registry: host.registry,
1560
- // Reuse the reactivity addressing: createTierAddressing(createRingHash()) is byte-identical to the
1561
- // host's internal addressing for the tier-0 coord (peer- and fanout-independent), and the handler
1562
- // only ever derives coord_0(topicId).
1563
- addressing: reactivityAddressing,
1564
- // Single-member reply signature over the node peer key (same pattern reactivity uses for its signers).
1565
- sign: async (payload: Uint8Array): Promise<string> => bytesToB64url(await signPeer(nodePrivateKey, payload)),
1566
- // Anti-DoS rate-limit seam (backlog matchmaking-query-rate-limit) intentionally left unwired here:
1567
- // default-allow. When that ticket lands it passes a `gate: (from, topicId) => boolean` that limits on
1568
- // the connection's verified `from` peer (NOT the self-asserted query.requesterId).
1569
- });
1568
+ // Inbound serve handler: decode (bounded) verify the dialing peer's signature freshness/replay gate
1569
+ // resolve the live PushState off the forwarder host serveBackfill/serveResume reply (no reply on any
1570
+ // failure; the stream aborts and the subscriber walks/chain-reads). One node-level replay guard is shared
1571
+ // across all recover requests — a plain pruned-on-access map, so no new timer to tear down.
1572
+ registerRecoverHandler(node, reactivityProtocols.recover, {
1573
+ pushStateFor: forwarderHost.pushStateFor.bind(forwarderHost),
1574
+ pushStateForCollection: forwarderHost.pushStateForCollection.bind(forwarderHost),
1575
+ replayGuard: createCorrelationReplayGuard(),
1576
+ rotationFor: (req, now) => {
1577
+ // Drain-window redirect: a recover reaching an OLD (rotated, still-draining) tail is bounced to
1578
+ // the new tree (reactivity-rotation-recover-redirect-drain). A resume carries the stale topic
1579
+ // (topicId = reactivityTopicId(latestKnownTailId)); a backfill carries no topic, so resolve the
1580
+ // collection's current served topic. rotationRedirectFor returns the gate's redirect while
1581
+ // draining and undefined once drained (then evicting the gate + the old tail's served PushState).
1582
+ const oldTopicId = req.topicId ?? resolveCurrentServedTopic(forwarderHost, req.collectionId);
1583
+ return oldTopicId === undefined ? undefined : forwarderHost.rotationRedirectFor(oldTopicId, now);
1584
+ },
1585
+ });
1570
1586
 
1571
- // Teardown: release reactivity timers + protocol handlers BEFORE host.stop() (which clears the cohort
1572
- // gossip timer + unhandles the cohort-topic protocols) BEFORE the node's transports close (previousStop).
1573
- // Composes with the existing arachnode + clusterMember stop wrappers (each calls its captured previousStop last).
1574
- const previousStop = node.stop.bind(node);
1575
- node.stop = async (): Promise<void> => {
1576
- try {
1577
- reactivityRotation.stop();
1578
- pushStateGossip.stop();
1579
- offInboundNotify();
1580
- await node.unhandle(reactivityProtocolList(reactivityProtocols));
1581
- await node.unhandle(matchmakingProtocolList(matchmakingProtocols));
1582
- unsubscribe();
1583
- await host.stop();
1584
- } finally {
1585
- await previousStop();
1586
- }
1587
- };
1588
- }
1587
+ // The subscriber's synchronous request signers over the node's Ed25519 key (resolves the recover wiring's
1588
+ // lone design point see recover-transport.ts §createRecoverRequestSigners). Fed to a manager by the
1589
+ // subscribe factory alongside recover.backfillTransport(topicId, collectionId) /
1590
+ // recover.resumeTransport(topicId, collectionId).
1591
+ const recoverSigners = createRecoverRequestSigners(nodePrivateKey);
1592
+
1593
+ // Expose the recover seams so the subscribe factory wires backfill/resume RPC + signers + the shared
1594
+ // sticky cache (mirrors `reactivitySubscribers` above).
1595
+ (node as any).reactivityRecover = recover;
1596
+ (node as any).reactivityRecoverSigners = recoverSigners;
1597
+ (node as any).reactivityCohortHintCache = reactivityCohortHintCache;
1598
+
1599
+ // 6. Rotation re-registration scheduler — the host timer that moves a subscriber to the rotated tree
1600
+ // when its manager surfaces a `RotationNotice` (`reactivity-rotation-rereg-scheduler`). Constructed with
1601
+ // the default unref'd `setTimeout` timer so an idle re-registration never pins the process. The
1602
+ // `reRegister(plan)` MOVE belongs to the subscribe factory that CONSTRUCTS managers (the deferred Quereus
1603
+ // `Database.watch` bridge — backlog optimystic-network-reactive-watch-integration-test): on fire it builds
1604
+ // a fresh `ReactivitySubscriptionManager` under `plan.newTopicId` carrying `plan.lastRevision`, registers
1605
+ // it, and swaps the `ReactivitySubscriberRegistry` entry — registering the NEW-topic handler BEFORE
1606
+ // unregistering the old, so a notification mid-swap is never dropped. Until that factory lands no
1607
+ // node-internal manager drives `schedule()`, so this seam is a logged no-op — exactly as 12.33 exposed
1608
+ // `reactivitySubscribers` / `reactivityRecover` without a live manager constructor.
1609
+ reactivityRotation = new RotationReRegistrationScheduler({
1610
+ reRegister: (plan): Promise<void> => {
1611
+ reactivityWiringLog("reactivity rotation re-registration fired for successor topic=%s (lastRevision=%d) but no subscribe factory is wired yet — deferred to optimystic-network-reactive-watch-integration-test", bytesToB64url(plan.newTopicId), plan.lastRevision);
1612
+ return Promise.resolve();
1613
+ },
1614
+ });
1615
+ (node as any).reactivityRotation = reactivityRotation;
1616
+
1617
+ // --- Matchmaking QueryV1 RPC — cohort serve side (docs/matchmaking.md §Seeker query) ---
1618
+ // The server half of the seeker query transport: a remote seeker dials `/optimystic/matchmaking/1.0.0/query`
1619
+ // and this node answers with its cohort's locally-held provider/seeker registrations, signed by the node
1620
+ // peer key. Matchmaking is layered ABOVE the cohort-topic substrate, so it owns its own protocol family
1621
+ // and is wired here (the composition root) over the host's PUBLIC surface only — mirroring the reactivity
1622
+ // registration above; nothing reaches into host.ts internals. The OUTBOUND seeker walk client is the
1623
+ // prereq follow-on `matchmaking-query-rpc-seeker-walk`; only the serve side is live here.
1624
+ registerMatchmakingQueryHandler(node, matchmakingProtocols.query, {
1625
+ registry: host.registry,
1626
+ // Reuse the reactivity addressing: createTierAddressing(createRingHash()) is byte-identical to the
1627
+ // host's internal addressing for the tier-0 coord (peer- and fanout-independent), and the handler
1628
+ // only ever derives coord_0(topicId).
1629
+ addressing: reactivityAddressing,
1630
+ // Single-member reply signature over the node peer key (same pattern reactivity uses for its signers).
1631
+ sign: async (payload: Uint8Array): Promise<string> => bytesToB64url(await signPeer(nodePrivateKey, payload)),
1632
+ // Anti-DoS rate-limit seam (backlog matchmaking-query-rate-limit) intentionally left unwired here:
1633
+ // default-allow. When that ticket lands it passes a `gate: (from, topicId) => boolean` that limits on
1634
+ // the connection's verified `from` peer (NOT the self-asserted query.requesterId).
1635
+ });
1636
+ }
1589
1637
 
1590
- return node;
1638
+ return node as unknown as OptimysticNode;
1639
+ } catch (err) {
1640
+ // Post-start rollback. node.stop() runs whatever teardown wrappers were installed BEFORE the throw
1641
+ // (each wrapper is registered next to the resource it releases, precisely so this unwinds as much as
1642
+ // exists) and closes the transports. A rollback failure must never mask the real startup error, so it
1643
+ // is logged and swallowed; `err` is what the caller sees.
1644
+ try {
1645
+ await node.stop();
1646
+ } catch (stopErr) {
1647
+ wiringLog('rollback stop failed after startup error: %o', stopErr);
1648
+ }
1649
+ throw err;
1650
+ }
1591
1651
  }