@optimystic/db-p2p 1.0.0-beta.3 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (234) hide show
  1. package/dist/src/cluster/block-transfer-service.d.ts +73 -1
  2. package/dist/src/cluster/block-transfer-service.d.ts.map +1 -1
  3. package/dist/src/cluster/block-transfer-service.js +70 -3
  4. package/dist/src/cluster/block-transfer-service.js.map +1 -1
  5. package/dist/src/cluster/block-transfer.d.ts +16 -1
  6. package/dist/src/cluster/block-transfer.d.ts.map +1 -1
  7. package/dist/src/cluster/block-transfer.js +73 -72
  8. package/dist/src/cluster/block-transfer.js.map +1 -1
  9. package/dist/src/cluster/client.d.ts +4 -6
  10. package/dist/src/cluster/client.d.ts.map +1 -1
  11. package/dist/src/cluster/client.js +8 -13
  12. package/dist/src/cluster/client.js.map +1 -1
  13. package/dist/src/cluster/cluster-policy.d.ts +2 -1
  14. package/dist/src/cluster/cluster-policy.d.ts.map +1 -1
  15. package/dist/src/cluster/cluster-policy.js.map +1 -1
  16. package/dist/src/cluster/cluster-repo.d.ts +126 -6
  17. package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
  18. package/dist/src/cluster/cluster-repo.js +378 -50
  19. package/dist/src/cluster/cluster-repo.js.map +1 -1
  20. package/dist/src/cluster/commit-proof.js +2 -2
  21. package/dist/src/cluster/race-resolution.d.ts +51 -7
  22. package/dist/src/cluster/race-resolution.d.ts.map +1 -1
  23. package/dist/src/cluster/race-resolution.js +71 -7
  24. package/dist/src/cluster/race-resolution.js.map +1 -1
  25. package/dist/src/cluster/rebalance-monitor.d.ts +67 -5
  26. package/dist/src/cluster/rebalance-monitor.d.ts.map +1 -1
  27. package/dist/src/cluster/rebalance-monitor.js +120 -26
  28. package/dist/src/cluster/rebalance-monitor.js.map +1 -1
  29. package/dist/src/cluster/spread-on-churn.d.ts.map +1 -1
  30. package/dist/src/cluster/spread-on-churn.js +34 -47
  31. package/dist/src/cluster/spread-on-churn.js.map +1 -1
  32. package/dist/src/cohort-topic/host.d.ts.map +1 -1
  33. package/dist/src/cohort-topic/host.js +8 -6
  34. package/dist/src/cohort-topic/host.js.map +1 -1
  35. package/dist/src/cohort-topic/membership-source.js +1 -1
  36. package/dist/src/cohort-topic/membership-source.js.map +1 -1
  37. package/dist/src/cohort-topic/peer-codec.d.ts.map +1 -1
  38. package/dist/src/cohort-topic/peer-codec.js +5 -2
  39. package/dist/src/cohort-topic/peer-codec.js.map +1 -1
  40. package/dist/src/cohort-topic/stream-util.d.ts +24 -3
  41. package/dist/src/cohort-topic/stream-util.d.ts.map +1 -1
  42. package/dist/src/cohort-topic/stream-util.js +36 -3
  43. package/dist/src/cohort-topic/stream-util.js.map +1 -1
  44. package/dist/src/cohort-topic/topic-router.d.ts +9 -0
  45. package/dist/src/cohort-topic/topic-router.d.ts.map +1 -1
  46. package/dist/src/cohort-topic/topic-router.js +12 -2
  47. package/dist/src/cohort-topic/topic-router.js.map +1 -1
  48. package/dist/src/dispute/client.d.ts.map +1 -1
  49. package/dist/src/dispute/client.js +15 -3
  50. package/dist/src/dispute/client.js.map +1 -1
  51. package/dist/src/index.d.ts +5 -0
  52. package/dist/src/index.d.ts.map +1 -1
  53. package/dist/src/index.js +5 -0
  54. package/dist/src/index.js.map +1 -1
  55. package/dist/src/libp2p-key-network.d.ts +78 -17
  56. package/dist/src/libp2p-key-network.d.ts.map +1 -1
  57. package/dist/src/libp2p-key-network.js +234 -147
  58. package/dist/src/libp2p-key-network.js.map +1 -1
  59. package/dist/src/libp2p-node-base.d.ts +35 -1
  60. package/dist/src/libp2p-node-base.d.ts.map +1 -1
  61. package/dist/src/libp2p-node-base.js +151 -26
  62. package/dist/src/libp2p-node-base.js.map +1 -1
  63. package/dist/src/libp2p-node-rn.d.ts +2 -2
  64. package/dist/src/libp2p-node-rn.d.ts.map +1 -1
  65. package/dist/src/libp2p-node-rn.js.map +1 -1
  66. package/dist/src/libp2p-node.d.ts +2 -2
  67. package/dist/src/libp2p-node.d.ts.map +1 -1
  68. package/dist/src/libp2p-node.js.map +1 -1
  69. package/dist/src/logger.d.ts.map +1 -1
  70. package/dist/src/logger.js +6 -1
  71. package/dist/src/logger.js.map +1 -1
  72. package/dist/src/matchmaking/query-transport.d.ts +3 -4
  73. package/dist/src/matchmaking/query-transport.d.ts.map +1 -1
  74. package/dist/src/matchmaking/query-transport.js +9 -9
  75. package/dist/src/matchmaking/query-transport.js.map +1 -1
  76. package/dist/src/network/network-manager-service.d.ts +10 -5
  77. package/dist/src/network/network-manager-service.d.ts.map +1 -1
  78. package/dist/src/network/network-manager-service.js +11 -2
  79. package/dist/src/network/network-manager-service.js.map +1 -1
  80. package/dist/src/network/relay-reservation.d.ts +170 -0
  81. package/dist/src/network/relay-reservation.d.ts.map +1 -0
  82. package/dist/src/network/relay-reservation.js +527 -0
  83. package/dist/src/network/relay-reservation.js.map +1 -0
  84. package/dist/src/pend-validation.d.ts +3 -2
  85. package/dist/src/pend-validation.d.ts.map +1 -1
  86. package/dist/src/pend-validation.js +3 -2
  87. package/dist/src/pend-validation.js.map +1 -1
  88. package/dist/src/protocol-client.d.ts +8 -1
  89. package/dist/src/protocol-client.d.ts.map +1 -1
  90. package/dist/src/protocol-client.js +11 -0
  91. package/dist/src/protocol-client.js.map +1 -1
  92. package/dist/src/reactivity/forwarder-host.d.ts +2 -1
  93. package/dist/src/reactivity/forwarder-host.d.ts.map +1 -1
  94. package/dist/src/reactivity/forwarder-host.js +2 -1
  95. package/dist/src/reactivity/forwarder-host.js.map +1 -1
  96. package/dist/src/reactivity/origination-manager.d.ts +3 -3
  97. package/dist/src/reactivity/origination-manager.js +3 -3
  98. package/dist/src/reactivity/recover-transport.d.ts +22 -14
  99. package/dist/src/reactivity/recover-transport.d.ts.map +1 -1
  100. package/dist/src/reactivity/recover-transport.js +32 -19
  101. package/dist/src/reactivity/recover-transport.js.map +1 -1
  102. package/dist/src/reactivity/subscription-manager.d.ts +2 -2
  103. package/dist/src/reactivity/topic-bytes.d.ts +6 -7
  104. package/dist/src/reactivity/topic-bytes.d.ts.map +1 -1
  105. package/dist/src/reactivity/topic-bytes.js +8 -9
  106. package/dist/src/reactivity/topic-bytes.js.map +1 -1
  107. package/dist/src/repo/client.d.ts +5 -2
  108. package/dist/src/repo/client.d.ts.map +1 -1
  109. package/dist/src/repo/client.js +49 -40
  110. package/dist/src/repo/client.js.map +1 -1
  111. package/dist/src/repo/cluster-coordinator.d.ts +77 -5
  112. package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
  113. package/dist/src/repo/cluster-coordinator.js +131 -20
  114. package/dist/src/repo/cluster-coordinator.js.map +1 -1
  115. package/dist/src/repo/coordinator-repo.d.ts +238 -122
  116. package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
  117. package/dist/src/repo/coordinator-repo.js +625 -472
  118. package/dist/src/repo/coordinator-repo.js.map +1 -1
  119. package/dist/src/repo/i-under-replication-ledger.d.ts +87 -0
  120. package/dist/src/repo/i-under-replication-ledger.d.ts.map +1 -0
  121. package/dist/src/repo/i-under-replication-ledger.js +2 -0
  122. package/dist/src/repo/i-under-replication-ledger.js.map +1 -0
  123. package/dist/src/repo/kv-under-replication-ledger.d.ts +91 -0
  124. package/dist/src/repo/kv-under-replication-ledger.d.ts.map +1 -0
  125. package/dist/src/repo/kv-under-replication-ledger.js +287 -0
  126. package/dist/src/repo/kv-under-replication-ledger.js.map +1 -0
  127. package/dist/src/repo/responsibility.d.ts +37 -0
  128. package/dist/src/repo/responsibility.d.ts.map +1 -0
  129. package/dist/src/repo/responsibility.js +37 -0
  130. package/dist/src/repo/responsibility.js.map +1 -0
  131. package/dist/src/repo/service.d.ts +44 -15
  132. package/dist/src/repo/service.d.ts.map +1 -1
  133. package/dist/src/repo/service.js +64 -27
  134. package/dist/src/repo/service.js.map +1 -1
  135. package/dist/src/repo/stuck-reservation.d.ts +127 -0
  136. package/dist/src/repo/stuck-reservation.d.ts.map +1 -0
  137. package/dist/src/repo/stuck-reservation.js +179 -0
  138. package/dist/src/repo/stuck-reservation.js.map +1 -0
  139. package/dist/src/repo/under-replication-drain.d.ts +208 -0
  140. package/dist/src/repo/under-replication-drain.d.ts.map +1 -0
  141. package/dist/src/repo/under-replication-drain.js +461 -0
  142. package/dist/src/repo/under-replication-drain.js.map +1 -0
  143. package/dist/src/rn.d.ts +5 -0
  144. package/dist/src/rn.d.ts.map +1 -1
  145. package/dist/src/rn.js +5 -0
  146. package/dist/src/rn.js.map +1 -1
  147. package/dist/src/storage/block-latch.d.ts +3 -2
  148. package/dist/src/storage/block-latch.d.ts.map +1 -1
  149. package/dist/src/storage/block-latch.js +7 -4
  150. package/dist/src/storage/block-latch.js.map +1 -1
  151. package/dist/src/storage/block-storage.d.ts +45 -3
  152. package/dist/src/storage/block-storage.d.ts.map +1 -1
  153. package/dist/src/storage/block-storage.js +156 -9
  154. package/dist/src/storage/block-storage.js.map +1 -1
  155. package/dist/src/storage/i-block-storage.d.ts +57 -5
  156. package/dist/src/storage/i-block-storage.d.ts.map +1 -1
  157. package/dist/src/storage/i-block-storage.js.map +1 -1
  158. package/dist/src/storage/pending-claim.d.ts +167 -0
  159. package/dist/src/storage/pending-claim.d.ts.map +1 -0
  160. package/dist/src/storage/pending-claim.js +156 -0
  161. package/dist/src/storage/pending-claim.js.map +1 -0
  162. package/dist/src/storage/raw-store-codec.d.ts +13 -0
  163. package/dist/src/storage/raw-store-codec.d.ts.map +1 -1
  164. package/dist/src/storage/raw-store-codec.js +24 -3
  165. package/dist/src/storage/raw-store-codec.js.map +1 -1
  166. package/dist/src/storage/restoration-coordinator.d.ts.map +1 -1
  167. package/dist/src/storage/restoration-coordinator.js +2 -1
  168. package/dist/src/storage/restoration-coordinator.js.map +1 -1
  169. package/dist/src/storage/ring-shift-coordinator.d.ts.map +1 -1
  170. package/dist/src/storage/ring-shift-coordinator.js +3 -3
  171. package/dist/src/storage/ring-shift-coordinator.js.map +1 -1
  172. package/dist/src/storage/storage-repo.d.ts +122 -5
  173. package/dist/src/storage/storage-repo.d.ts.map +1 -1
  174. package/dist/src/storage/storage-repo.js +245 -68
  175. package/dist/src/storage/storage-repo.js.map +1 -1
  176. package/dist/src/storage/struct.d.ts +58 -0
  177. package/dist/src/storage/struct.d.ts.map +1 -1
  178. package/dist/src/testing/mesh-harness.d.ts +77 -11
  179. package/dist/src/testing/mesh-harness.d.ts.map +1 -1
  180. package/dist/src/testing/mesh-harness.js +226 -100
  181. package/dist/src/testing/mesh-harness.js.map +1 -1
  182. package/package.json +2 -2
  183. package/readme.md +27 -3
  184. package/src/cluster/block-transfer-service.ts +132 -4
  185. package/src/cluster/block-transfer.ts +79 -84
  186. package/src/cluster/client.ts +115 -119
  187. package/src/cluster/cluster-policy.ts +2 -1
  188. package/src/cluster/cluster-repo.ts +422 -56
  189. package/src/cluster/commit-proof.ts +2 -2
  190. package/src/cluster/race-resolution.ts +76 -7
  191. package/src/cluster/rebalance-monitor.ts +677 -539
  192. package/src/cluster/spread-on-churn.ts +34 -54
  193. package/src/cohort-topic/host.ts +8 -6
  194. package/src/cohort-topic/membership-source.ts +1 -1
  195. package/src/cohort-topic/peer-codec.ts +34 -31
  196. package/src/cohort-topic/stream-util.ts +39 -4
  197. package/src/cohort-topic/topic-router.ts +12 -2
  198. package/src/dispute/client.ts +17 -7
  199. package/src/index.ts +64 -59
  200. package/src/libp2p-key-network.ts +1377 -1260
  201. package/src/libp2p-node-base.ts +186 -27
  202. package/src/libp2p-node-rn.ts +31 -30
  203. package/src/libp2p-node.ts +37 -36
  204. package/src/logger.ts +6 -1
  205. package/src/matchmaking/query-transport.ts +9 -9
  206. package/src/network/network-manager-service.ts +16 -5
  207. package/src/network/relay-reservation.ts +636 -0
  208. package/src/pend-validation.ts +3 -2
  209. package/src/protocol-client.ts +13 -1
  210. package/src/reactivity/forwarder-host.ts +2 -1
  211. package/src/reactivity/origination-manager.ts +3 -3
  212. package/src/reactivity/recover-transport.ts +39 -22
  213. package/src/reactivity/subscription-manager.ts +2 -2
  214. package/src/reactivity/topic-bytes.ts +35 -37
  215. package/src/repo/client.ts +172 -173
  216. package/src/repo/cluster-coordinator.ts +145 -19
  217. package/src/repo/coordinator-repo.ts +652 -505
  218. package/src/repo/i-under-replication-ledger.ts +89 -0
  219. package/src/repo/kv-under-replication-ledger.ts +302 -0
  220. package/src/repo/responsibility.ts +48 -0
  221. package/src/repo/service.ts +76 -39
  222. package/src/repo/stuck-reservation.ts +224 -0
  223. package/src/repo/under-replication-drain.ts +582 -0
  224. package/src/rn.ts +74 -69
  225. package/src/storage/block-latch.ts +7 -4
  226. package/src/storage/block-storage.ts +161 -10
  227. package/src/storage/i-block-storage.ts +60 -5
  228. package/src/storage/pending-claim.ts +200 -0
  229. package/src/storage/raw-store-codec.ts +26 -3
  230. package/src/storage/restoration-coordinator.ts +212 -211
  231. package/src/storage/ring-shift-coordinator.ts +271 -271
  232. package/src/storage/storage-repo.ts +280 -72
  233. package/src/storage/struct.ts +58 -0
  234. package/src/testing/mesh-harness.ts +293 -112
@@ -1,539 +1,677 @@
1
- import type { Startable, Libp2p } from '@libp2p/interface'
2
- import { hashKey } from 'p2p-fret'
3
- import type { FretService } from 'p2p-fret'
4
- import type { PartitionDetector } from './partition-detector.js'
5
- import type { ArachnodeFretAdapter, ArachnodeInfo } from '../storage/arachnode-fret-adapter.js'
6
- import { createLogger } from '../logger.js'
7
-
8
- const log = createLogger('rebalance-monitor')
9
- const textEncoder = new TextEncoder()
10
-
11
- export interface RebalanceEvent {
12
- /** Block IDs this node has gained responsibility for */
13
- gained: string[]
14
- /** Block IDs this node has lost responsibility for */
15
- lost: string[]
16
- /** Peers that are now closer for the lost blocks: blockId → peerId[] */
17
- newOwners: Map<string, string[]>
18
- /**
19
- * Blocks this node KEEPS responsibility for whose cohort now contains peers it has not seen
20
- * co-responsible before: blockId → the newly co-responsible peer ids (never self). This is the
21
- * cohort-GROWTH arm: the founder case (a block committed while the deployment was one node) never
22
- * appears in `lost` — the holder keeps the block — so without this arm nothing ever pushes the
23
- * second copy and the block stays readable only by its sole holder. The reaction pushes each
24
- * block to these peers (capped by the replication floor). A block can appear in both `gained`
25
- * and `grown` (first observation after a restart/regain the push then finds no local data and
26
- * is a benign no-op); it can never appear in both `lost` and `grown` (lost not responsible).
27
- */
28
- grown: Map<string, string[]>
29
- /**
30
- * Replication floor `N` for this event — the cohort size FRET assembled at check time
31
- * ({@link RebalanceMonitor.getCohortSize}). The reaction gates release of a `lost` block on
32
- * confirming it replicated to this many new owners, so a lost block is never released below the
33
- * floor. See `docs/arachnode-ring-handoff.md` § Part 2.
34
- */
35
- floor: number
36
- /** Timestamp of the topology change that triggered this */
37
- triggeredAt: number
38
- }
39
-
40
- /**
41
- * What the growth reaction learned about ONE block reported `grown`. Fed back to
42
- * {@link RebalanceMonitor.recordGrowthOutcome} so the seen set is confirmation-driven: a peer
43
- * enters a block's seen set only once a replica is confirmed on it, or once the block has
44
- * otherwise reached its floor. A block the reaction had NO information about (its confirm was
45
- * deduped against one already in flight) gets no outcome at all — the monitor's state stays
46
- * untouched and the next check re-detects.
47
- */
48
- export interface GrowthOutcome {
49
- /** Newly co-responsible peers that may now be recorded as seen for this block. */
50
- satisfiedPeers: string[]
51
- /** True when nothing about this block is still owed a push. */
52
- complete: boolean
53
- }
54
-
55
- export interface RebalanceMonitorConfig {
56
- /** Debounce window for topology changes (ms). Default: 5000 */
57
- debounceMs?: number
58
- /** Maximum frequency of full rebalance scans (ms). Default: 60000 */
59
- minRebalanceIntervalMs?: number
60
- /** Whether to suppress rebalancing during detected partitions. Default: true */
61
- suppressDuringPartition?: boolean
62
- /**
63
- * Maximum blocks reported `grown` per check. Bounds the work a single peer join can trigger on a
64
- * node with a large owned-block set — the primary bound is already the floor (a cohort is at most
65
- * floor-sized, so each grown block pushes to ≤ floor−1 peers, and the reaction stops per block
66
- * once the floor is met); this cap bounds the block COUNT per pass. A block dropped by the cap is
67
- * NOT recorded as seen, so the next check re-detects the same growth deferred, never lost.
68
- * Default: 64
69
- */
70
- growthBlockBudget?: number
71
- /**
72
- * How many incomplete growth outcomes ({@link GrowthOutcome} with `complete: false`) a block
73
- * absorbs before its still-unsatisfied peers are moved to a per-block abandoned set and no longer
74
- * pushed to. Without this bound a peer that permanently refuses would be re-pushed on every check
75
- * forever, and its block would re-consume `growthBlockBudget` slots and starve genuinely-new
76
- * growth. An abandoned peer that leaves the cohort and later rejoins is retried from scratch.
77
- * Default: 5
78
- */
79
- growthMaxAttempts?: number
80
- /**
81
- * Self-arming re-check timer for outstanding growth work (reported-but-unconfirmed peers, or
82
- * blocks deferred by `growthBlockBudget`). Checks otherwise fire only on libp2p connection
83
- * events, so a failed push on a then-quiet network would never be retried. Armed at the end of a
84
- * check only while work is outstanding; fires `maybeRebalance()` so the existing
85
- * `minRebalanceIntervalMs` throttle still bounds the push rate. `0` disables.
86
- * Default: `minRebalanceIntervalMs`
87
- */
88
- growthRecheckIntervalMs?: number
89
- }
90
-
91
- export interface RebalanceMonitorDeps {
92
- libp2p: Libp2p
93
- fret: FretService
94
- partitionDetector: PartitionDetector
95
- fretAdapter: ArachnodeFretAdapter
96
- /**
97
- * The owned-block tracked set. When provided (e.g. the shared `ownedBlocks` set wired in
98
- * `libp2p-node-base`), the monitor references this exact `Set` instead of constructing its own,
99
- * so it stays in lock-step with the `SpreadOnChurnMonitor` that shares it. Omit for standalone
100
- * construction (unit tests) a fresh private `Set` preserves all existing behavior. Note: only
101
- * `trackedBlocks` is shared; `responsibilitySnapshot` stays per-monitor (it is rebalance's own
102
- * was-responsible memory, not owned-block tracking).
103
- */
104
- trackedBlocks?: Set<string>
105
- }
106
-
107
- type RebalanceHandler = (event: RebalanceEvent) => void
108
-
109
- /** Per-block rebalance/growth state (the `responsibilitySnapshot` entry). */
110
- interface BlockGrowthState {
111
- responsible: boolean
112
- /**
113
- * The growth arm's seen set: peers CONFIRMED to hold a replica of this block (or satisfied
114
- * another way — floor met, or nothing local to push). Peers enter ONLY via
115
- * {@link RebalanceMonitor.recordGrowthOutcome}, never at report time.
116
- */
117
- cohortPeers: Set<string>
118
- /** Peers reported grown at the last emitting check whose confirmation is still outstanding. */
119
- pendingPeers: Set<string>
120
- /** Consecutive incomplete growth outcomes — the give-up counter against `growthMaxAttempts`. */
121
- growthAttempts: number
122
- /**
123
- * Peers given up on after `growthMaxAttempts` incomplete outcomes — excluded from growth reports
124
- * until they leave the cohort and rejoin (each check intersects this with the current cohort).
125
- */
126
- abandonedPeers: Set<string>
127
- }
128
-
129
- const emptyGrowthState = (responsible: boolean): BlockGrowthState => ({
130
- responsible,
131
- cohortPeers: new Set<string>(),
132
- pendingPeers: new Set<string>(),
133
- growthAttempts: 0,
134
- abandonedPeers: new Set<string>()
135
- })
136
-
137
- const intersect = (remembered: Set<string> | undefined, current: Set<string>): Set<string> =>
138
- new Set([...(remembered ?? [])].filter(id => current.has(id)))
139
-
140
- /**
141
- * Carry a still-responsible block's growth state into the next check, intersecting both remembered
142
- * peer sets against the CURRENT cohort: a peer that left drops out of `cohortPeers` (so its return
143
- * is re-detected the departure self-heal) and out of `abandonedPeers` (so a rejoin is retried from
144
- * scratch). `pendingPeers` is always rebuilt from this check's own report, never carried.
145
- */
146
- const carryGrowthState = (prior: BlockGrowthState | undefined, currentPeers: Set<string>): BlockGrowthState => ({
147
- responsible: true,
148
- cohortPeers: intersect(prior?.cohortPeers, currentPeers),
149
- pendingPeers: new Set<string>(),
150
- growthAttempts: prior?.growthAttempts ?? 0,
151
- abandonedPeers: intersect(prior?.abandonedPeers, currentPeers)
152
- })
153
-
154
- export class RebalanceMonitor implements Startable {
155
- private running = false
156
- private readonly trackedBlocks: Set<string>
157
- // Per-monitor was-responsible memory (NOT shared, unlike trackedBlocks). When the shared
158
- // trackedBlocks set is mutated externally — spread's no-local-data self-prune, or the node's
159
- // responsibility-loss eviction going through untrackBlock — a snapshot entry for a since-removed
160
- // block may linger here. That is acceptable: performRebalanceCheck only iterates trackedBlocks, so
161
- // a lingering entry is inert; if the block is later re-fed, its responsibility is simply re-derived.
162
- //
163
- // `cohortPeers` is the growth arm's CONFIRMED-co-responsible memory. A MISSING entry is
164
- // deliberately treated as "prior cohort = empty", so the first check after a topology event
165
- // reports the whole non-self cohort as grownthat is what heals the founder case (A alone
166
- // commits; B joins first check pushes to B) and the restarted-holder case (snapshot memory is
167
- // process-local, so a restarted holder re-pushes to everyone once). Peers enter the set only
168
- // through recordGrowthOutcome — a reported-but-unconfirmed peer stays out, so a failed push is
169
- // re-detected on the next check instead of being recorded as done.
170
- private readonly responsibilitySnapshot = new Map<string, BlockGrowthState>()
171
- private readonly handlers: RebalanceHandler[] = []
172
- private debounceTimer: ReturnType<typeof setTimeout> | null = null
173
- private recheckTimer: ReturnType<typeof setTimeout> | null = null
174
- private lastRebalanceAt = 0
175
- private pendingTopologyChange = false
176
- private topologyChangeTimestamp = 0
177
- private lastGrowthDeferred = 0
178
-
179
- private readonly debounceMs: number
180
- private readonly minRebalanceIntervalMs: number
181
- private readonly suppressDuringPartition: boolean
182
- private readonly growthBlockBudget: number
183
- private readonly growthMaxAttempts: number
184
- private readonly growthRecheckIntervalMs: number
185
-
186
- private readonly onConnectionOpen: () => void
187
- private readonly onConnectionClose: () => void
188
-
189
- constructor(
190
- private readonly deps: RebalanceMonitorDeps,
191
- config: RebalanceMonitorConfig = {}
192
- ) {
193
- // Share the injected owned-block set when present (so spread + rebalance never drift);
194
- // otherwise own a private set (standalone construction / unit tests). Only trackedBlocks is
195
- // shared responsibilitySnapshot stays per-monitor.
196
- this.trackedBlocks = deps.trackedBlocks ?? new Set<string>()
197
- this.debounceMs = config.debounceMs ?? 5000
198
- this.minRebalanceIntervalMs = config.minRebalanceIntervalMs ?? 60000
199
- this.suppressDuringPartition = config.suppressDuringPartition ?? true
200
- this.growthBlockBudget = config.growthBlockBudget ?? 64
201
- this.growthMaxAttempts = config.growthMaxAttempts ?? 5
202
- this.growthRecheckIntervalMs = config.growthRecheckIntervalMs ?? this.minRebalanceIntervalMs
203
-
204
- this.onConnectionOpen = () => this.handleTopologyChange()
205
- this.onConnectionClose = () => this.handleTopologyChange()
206
- }
207
-
208
- async start(): Promise<void> {
209
- if (this.running) return
210
- this.running = true
211
-
212
- this.deps.libp2p.addEventListener('connection:open', this.onConnectionOpen)
213
- this.deps.libp2p.addEventListener('connection:close', this.onConnectionClose)
214
-
215
- log('started, tracking %d blocks', this.trackedBlocks.size)
216
- }
217
-
218
- async stop(): Promise<void> {
219
- if (!this.running) return
220
- this.running = false
221
-
222
- this.deps.libp2p.removeEventListener('connection:open', this.onConnectionOpen)
223
- this.deps.libp2p.removeEventListener('connection:close', this.onConnectionClose)
224
-
225
- if (this.debounceTimer) {
226
- clearTimeout(this.debounceTimer)
227
- this.debounceTimer = null
228
- }
229
- if (this.recheckTimer) {
230
- clearTimeout(this.recheckTimer)
231
- this.recheckTimer = null
232
- }
233
-
234
- this.pendingTopologyChange = false
235
- log('stopped')
236
- }
237
-
238
- onRebalance(handler: RebalanceHandler): void {
239
- this.handlers.push(handler)
240
- }
241
-
242
- trackBlock(blockId: string): void {
243
- this.trackedBlocks.add(blockId)
244
- }
245
-
246
- untrackBlock(blockId: string): void {
247
- this.trackedBlocks.delete(blockId)
248
- this.responsibilitySnapshot.delete(blockId)
249
- }
250
-
251
- getTrackedBlockCount(): number {
252
- return this.trackedBlocks.size
253
- }
254
-
255
- async checkNow(): Promise<RebalanceEvent | null> {
256
- return this.performRebalanceCheck(Date.now())
257
- }
258
-
259
- /**
260
- * Feedback from the growth reaction for one block reported `grown`. `satisfiedPeers` enter the
261
- * block's seen set; an incomplete outcome counts an attempt against `growthMaxAttempts`, and on
262
- * reaching the bound the block's still-unsatisfied reported peers are abandoned (no longer
263
- * pushed to until they leave the cohort and rejoin). Never called for a block the reaction had
264
- * no information about (a confirm deduped against one already in flight) a missing outcome
265
- * leaves the block's state untouched so the next check retries.
266
- */
267
- recordGrowthOutcome(blockId: string, outcome: GrowthOutcome): void {
268
- const state = this.responsibilitySnapshot.get(blockId)
269
- // No state (untracked since) or responsibility lost since the report: the growth state was
270
- // cleared, and recording into it would survive the clear and suppress the regain re-push.
271
- if (!state || !state.responsible) return
272
-
273
- for (const peerId of outcome.satisfiedPeers) {
274
- state.cohortPeers.add(peerId)
275
- state.pendingPeers.delete(peerId)
276
- }
277
-
278
- if (outcome.complete) {
279
- state.growthAttempts = 0
280
- state.pendingPeers.clear()
281
- } else {
282
- // NOTE: growthMaxAttempts is a floor on the retry count, not an exact one. `pendingPeers` is
283
- // rebuilt from each check's own report, so two cases blunt the bound: a check that defers
284
- // this block on growthBlockBudget clears pendingPeers, and a give-up landing right then
285
- // abandons nobody while still resetting the counter; and two checks racing (the second
286
- // re-reporting the same peer after the first's confirm left `inFlight` but before its outcome
287
- // landed) double-count one attempt. Both are rare, both only change how many pushes a doomed
288
- // peer absorbs. If a deployment ever tracks far more blocks than growthBlockBudget, deferral
289
- // stops being rare — carry the report's peer list on GrowthOutcome and match it against
290
- // pendingPeers instead of trusting the latest report.
291
- state.growthAttempts++
292
- if (state.growthAttempts >= this.growthMaxAttempts) {
293
- for (const peerId of state.pendingPeers) {
294
- state.abandonedPeers.add(peerId)
295
- }
296
- log('growth give-up: block=%s abandoning %d unsatisfied peer(s) after %d attempts',
297
- blockId, state.pendingPeers.size, state.growthAttempts)
298
- state.pendingPeers.clear()
299
- state.growthAttempts = 0
300
- }
301
- }
302
-
303
- this.updateRecheckTimer()
304
- }
305
-
306
- /**
307
- * Growth-arm observability: how many tracked blocks still await confirmation on reported peers,
308
- * how many (block, peer) pairs have been given up on, and whether the re-check timer is armed.
309
- */
310
- getGrowthDiagnostics(): { blocksAwaitingConfirmation: number; abandonedPairs: number; recheckArmed: boolean } {
311
- let blocksAwaitingConfirmation = 0
312
- let abandonedPairs = 0
313
- for (const [blockId, state] of this.responsibilitySnapshot) {
314
- if (!state.responsible || !this.trackedBlocks.has(blockId)) continue
315
- if (state.pendingPeers.size > 0) blocksAwaitingConfirmation++
316
- abandonedPairs += state.abandonedPeers.size
317
- }
318
- return { blocksAwaitingConfirmation, abandonedPairs, recheckArmed: this.recheckTimer !== null }
319
- }
320
-
321
- /** Growth work is outstanding while any reported peer is unconfirmed or blocks were budget-deferred. */
322
- private hasOutstandingGrowthWork(): boolean {
323
- if (this.lastGrowthDeferred > 0) return true
324
- for (const [blockId, state] of this.responsibilitySnapshot) {
325
- if (state.responsible && state.pendingPeers.size > 0 && this.trackedBlocks.has(blockId)) {
326
- return true
327
- }
328
- }
329
- return false
330
- }
331
-
332
- /**
333
- * Arm the growth re-check timer while work is outstanding; disarm it when there is none. The
334
- * timer fires maybeRebalance(), so minRebalanceIntervalMs still bounds the push rate, and it
335
- * re-arms itself after firing for as long as work remains. unref'd so it never holds the
336
- * process open; stop() clears it.
337
- */
338
- private updateRecheckTimer(): void {
339
- if (this.growthRecheckIntervalMs <= 0) return
340
-
341
- if (!this.running || !this.hasOutstandingGrowthWork()) {
342
- if (this.recheckTimer) {
343
- clearTimeout(this.recheckTimer)
344
- this.recheckTimer = null
345
- }
346
- return
347
- }
348
-
349
- if (this.recheckTimer) return // already armed
350
-
351
- this.recheckTimer = setTimeout(() => {
352
- this.recheckTimer = null
353
- void this.maybeRebalance()
354
- .catch(err => { log('recheck error: %O', err) })
355
- .finally(() => this.updateRecheckTimer())
356
- }, this.growthRecheckIntervalMs)
357
- ;(this.recheckTimer as unknown as { unref?: () => void }).unref?.()
358
- }
359
-
360
- private handleTopologyChange(): void {
361
- if (!this.running) return
362
-
363
- if (!this.pendingTopologyChange) {
364
- this.topologyChangeTimestamp = Date.now()
365
- }
366
- this.pendingTopologyChange = true
367
-
368
- if (this.debounceTimer) {
369
- clearTimeout(this.debounceTimer)
370
- }
371
-
372
- this.debounceTimer = setTimeout(() => {
373
- this.debounceTimer = null
374
- this.pendingTopologyChange = false
375
- this.maybeRebalance()
376
- }, this.debounceMs)
377
- }
378
-
379
- private async maybeRebalance(): Promise<void> {
380
- if (!this.running) return
381
-
382
- const now = Date.now()
383
- const elapsed = now - this.lastRebalanceAt
384
- if (elapsed < this.minRebalanceIntervalMs) {
385
- log('throttled, %dms since last rebalance', elapsed)
386
- return
387
- }
388
-
389
- const event = await this.performRebalanceCheck(this.topologyChangeTimestamp || now)
390
- if (event) {
391
- this.emitEvent(event)
392
- }
393
- }
394
-
395
- private async performRebalanceCheck(triggeredAt: number): Promise<RebalanceEvent | null> {
396
- if (this.suppressDuringPartition && this.deps.partitionDetector.detectPartition()) {
397
- log('partition detected, suppressing rebalance')
398
- return null
399
- }
400
-
401
- if (this.trackedBlocks.size === 0) {
402
- this.lastRebalanceAt = Date.now()
403
- // Nothing left to grow, so any prior deferral is moot — drop it and let the re-check timer
404
- // disarm, rather than re-arming forever against blocks that were untracked out from under it.
405
- this.lastGrowthDeferred = 0
406
- this.updateRecheckTimer()
407
- return null
408
- }
409
-
410
- const selfId = this.deps.libp2p.peerId.toString()
411
- const gained: string[] = []
412
- const lost: string[] = []
413
- const newOwners = new Map<string, string[]>()
414
- const grown = new Map<string, string[]>()
415
- let growthDeferred = 0
416
- const growthCandidates: Array<{ blockId: string; newPeers: string[]; state: BlockGrowthState }> = []
417
-
418
- for (const blockId of this.trackedBlocks) {
419
- const key = textEncoder.encode(blockId)
420
- const coord = await hashKey(key)
421
-
422
- // Get the current cohort — assembleCohort returns peer IDs sorted by distance
423
- const cohort = this.deps.fret.assembleCohort(coord, this.getCohortSize())
424
- const isResponsible = cohort.includes(selfId)
425
- const prior = this.responsibilitySnapshot.get(blockId)
426
- const wasResponsible = prior?.responsible ?? false
427
-
428
- if (isResponsible && !wasResponsible) {
429
- gained.push(blockId)
430
- } else if (!isResponsible && wasResponsible) {
431
- lost.push(blockId)
432
- // The cohort members are the new owners
433
- newOwners.set(blockId, cohort.filter(id => id !== selfId))
434
- }
435
-
436
- // Growth arm: while this node STAYS responsible, any cohort peer not yet CONFIRMED to hold
437
- // the block (and not abandoned) gets it pushed (up to the floor). Runs on every responsible
438
- // check — NOT gated on wasResponsible — so a first observation (no snapshot entry) treats
439
- // the whole non-self cohort as new; see the responsibilitySnapshot comment for why that is
440
- // load-bearing. Not responsible ⇒ arm skipped, so `lost` ∩ `grown` is impossible.
441
- //
442
- // Reporting a peer does NOT record it as seen — only recordGrowthOutcome does — so a push
443
- // that fails (dial timeout, receiver refused to persist, partition mid-reaction, reaction
444
- // threw) leaves the peer un-seen and the next check re-detects it.
445
- let state: BlockGrowthState
446
- if (isResponsible) {
447
- const currentSet = new Set(cohort.filter(id => id !== selfId))
448
- state = carryGrowthState(prior, currentSet)
449
- const newPeers = [...currentSet].filter(id => !state.cohortPeers.has(id) && !state.abandonedPeers.has(id))
450
- if (newPeers.length > 0) {
451
- growthCandidates.push({ blockId, newPeers, state })
452
- } else {
453
- // Nothing owed for this block. growthAttempts counts CONSECUTIVE incomplete outcomes
454
- // against outstanding growth, so it must not carry across a quiet stretch — otherwise a
455
- // block that failed a few times, then had that peer leave, would spend the leftovers on
456
- // whichever peer joins next and abandon it early.
457
- state.growthAttempts = 0
458
- }
459
- } else {
460
- // Not responsible: clear ALL growth state (seen set, attempts, abandoned peers), so a
461
- // later regain re-pushes to the whole cohort (benign when the local data is gone — the
462
- // push finds nothing and no-ops).
463
- state = emptyGrowthState(false)
464
- }
465
-
466
- this.responsibilitySnapshot.set(blockId, state)
467
- }
468
-
469
- // Fill the growth budget in two passes: fresh growth (no failed attempts yet) first, retrying
470
- // blocks with what remains otherwise a stuck retry set at the front of the tracked-block
471
- // insertion order would starve peers that just joined.
472
- for (const candidate of [
473
- ...growthCandidates.filter(c => c.state.growthAttempts === 0),
474
- ...growthCandidates.filter(c => c.state.growthAttempts > 0)
475
- ]) {
476
- if (grown.size < this.growthBlockBudget) {
477
- grown.set(candidate.blockId, candidate.newPeers)
478
- candidate.state.pendingPeers = new Set(candidate.newPeers)
479
- } else {
480
- // Budget-dropped: the seen set was not touched, so the next check re-detects the same
481
- // growth a deferral, not a loss.
482
- growthDeferred++
483
- }
484
- }
485
- this.lastGrowthDeferred = growthDeferred
486
-
487
- this.lastRebalanceAt = Date.now()
488
-
489
- if (growthDeferred > 0) {
490
- // Deferred blocks drain one budget-full per check. Checks fire on libp2p connection events
491
- // AND — while growth work is outstanding — on the growthRecheckIntervalMs timer armed
492
- // below, so a backlog drains even on a quiet network.
493
- log('growth budget reached: %d blocks deferred to the next check (budget=%d)',
494
- growthDeferred, this.growthBlockBudget)
495
- }
496
-
497
- this.updateRecheckTimer()
498
-
499
- if (gained.length === 0 && lost.length === 0 && grown.size === 0) {
500
- return null
501
- }
502
-
503
- log('rebalance check: gained=%d lost=%d grown=%d', gained.length, lost.length, grown.size)
504
-
505
- return { gained, lost, newOwners, grown, floor: this.getCohortSize(), triggeredAt }
506
- }
507
-
508
- /**
509
- * The replication floor `N` — the cohort size FRET assembles for a block. Public so the ring-shift
510
- * handoff and the rebalance reaction can gate release on confirming replication to this many
511
- * holders (`docs/arachnode-ring-handoff.md` § Replication floor). Derives from FRET's network-size
512
- * estimate: `clamp(ceil(sqrt(n_est)), 1, 3)`, defaulting to 3 when no confident estimate exists.
513
- */
514
- getCohortSize(): number {
515
- const diag: any = (this.deps.fret as any).getDiagnostics?.()
516
- const estimate = diag?.estimate ?? diag?.n
517
- if (typeof estimate === 'number' && Number.isFinite(estimate) && estimate > 0) {
518
- return Math.max(1, Math.min(3, Math.ceil(Math.sqrt(estimate))))
519
- }
520
- return 3
521
- }
522
-
523
- private emitEvent(event: RebalanceEvent): void {
524
- for (const handler of this.handlers) {
525
- try {
526
- handler(event)
527
- } catch (err) {
528
- log('handler error: %O', err)
529
- }
530
- }
531
- }
532
-
533
- /**
534
- * Update ArachnodeInfo status through the fret adapter.
535
- */
536
- setStatus(status: ArachnodeInfo['status']): void {
537
- this.deps.fretAdapter.setStatus(status)
538
- }
539
- }
1
+ import type { Startable, Libp2p } from '@libp2p/interface'
2
+ import { hashKey } from 'p2p-fret'
3
+ import type { FretService } from 'p2p-fret'
4
+ import { routingKeyForBlock } from '@optimystic/db-core'
5
+ import type { IKeyNetwork } from '@optimystic/db-core'
6
+ import type { PartitionDetector } from './partition-detector.js'
7
+ import type { ArachnodeFretAdapter, ArachnodeInfo } from '../storage/arachnode-fret-adapter.js'
8
+ import { createLogger } from '../logger.js'
9
+
10
+ const log = createLogger('rebalance-monitor')
11
+
12
+ export interface RebalanceEvent {
13
+ /**
14
+ * Block IDs this node has gained responsibility for (the reaction pulls each). A block first seen
15
+ * through a commit this node holds ({@link RebalanceMonitor.recordCommittedHolders}) is not
16
+ * reported: there is nothing to pull.
17
+ */
18
+ gained: string[]
19
+ /** Block IDs this node has lost responsibility for */
20
+ lost: string[]
21
+ /** Peers that are now closer for the lost blocks: blockId peerId[] */
22
+ newOwners: Map<string, string[]>
23
+ /**
24
+ * Blocks this node KEEPS responsibility for whose cohort now contains peers it has not seen
25
+ * co-responsible before: blockId the newly co-responsible peer ids (never self). This is the
26
+ * cohort-GROWTH arm: the founder case (a block committed while the deployment was one node) never
27
+ * appears in `lost` — the holder keeps the block — so without this arm nothing ever pushes the
28
+ * second copy and the block stays readable only by its sole holder. The reaction pushes each
29
+ * block to these peers (capped by the replication floor). A block can appear in both `gained`
30
+ * and `grown` (first observation after a restart/regain — the push then finds no local data and
31
+ * is a benign no-op); it can never appear in both `lost` and `grown` (lost ⇒ not responsible).
32
+ */
33
+ grown: Map<string, string[]>
34
+ /**
35
+ * Replication floor `N` for this event — the cohort size FRET assembled at check time
36
+ * ({@link RebalanceMonitor.getCohortSize}). The reaction gates release of a `lost` block on
37
+ * confirming it replicated to this many new owners, so a lost block is never released below the
38
+ * floor. See `docs/arachnode-ring-handoff.md` § Part 2.
39
+ */
40
+ floor: number
41
+ /** Timestamp of the topology change that triggered this */
42
+ triggeredAt: number
43
+ }
44
+
45
+ /**
46
+ * What the growth reaction learned about ONE block reported `grown`. Fed back to
47
+ * {@link RebalanceMonitor.recordGrowthOutcome} so the seen set is confirmation-driven: a peer
48
+ * enters a block's seen set only once a replica is confirmed on it, or once the block has
49
+ * otherwise reached its floor. A block the reaction had NO information about (its confirm was
50
+ * deduped against one already in flight) gets no outcome at all — the monitor's state stays
51
+ * untouched and the next check re-detects.
52
+ */
53
+ export interface GrowthOutcome {
54
+ /** Newly co-responsible peers that may now be recorded as seen for this block. */
55
+ satisfiedPeers: string[]
56
+ /** True when nothing about this block is still owed a push. */
57
+ complete: boolean
58
+ }
59
+
60
+ /**
61
+ * What an acknowledged commit proved about who holds its blocks, reported by the node that holds them.
62
+ * Only ever reported for a commit THIS node's own storage durably holds, so a block named here needs
63
+ * no pull. See {@link RebalanceMonitor.recordCommittedHolders}.
64
+ */
65
+ export interface CommittedHolders {
66
+ blockIds: readonly string[]
67
+ /** Peers evidenced to hold the committed revision. Self may appear; it is ignored. */
68
+ holders: readonly string[]
69
+ /**
70
+ * Cohort members the commit's coordinator knows did NOT confirm holding the revision. Withdraws
71
+ * any earlier record of them as holders, so the growth arm pushes them the block.
72
+ */
73
+ unconfirmed?: readonly string[]
74
+ }
75
+
76
+ /** Evidence gathered between two checks for one block: `holders` and `unconfirmed` stay disjoint. */
77
+ interface CommitEvidence {
78
+ holders: Set<string>
79
+ unconfirmed: Set<string>
80
+ }
81
+
82
+ export interface RebalanceMonitorConfig {
83
+ /** Debounce window for topology changes (ms). Default: 5000 */
84
+ debounceMs?: number
85
+ /** Maximum frequency of full rebalance scans (ms). Default: 60000 */
86
+ minRebalanceIntervalMs?: number
87
+ /** Whether to suppress rebalancing during detected partitions. Default: true */
88
+ suppressDuringPartition?: boolean
89
+ /**
90
+ * Maximum blocks reported `grown` per check. Bounds the work a single peer join can trigger on a
91
+ * node with a large owned-block set — the primary bound is already the floor (a cohort is at most
92
+ * floor-sized, so each grown block pushes to ≤ floor−1 peers, and the reaction stops per block
93
+ * once the floor is met); this cap bounds the block COUNT per pass. A block dropped by the cap is
94
+ * NOT recorded as seen, so the next check re-detects the same growth — deferred, never lost.
95
+ * Default: 64
96
+ */
97
+ growthBlockBudget?: number
98
+ /**
99
+ * How many incomplete growth outcomes ({@link GrowthOutcome} with `complete: false`) a block
100
+ * absorbs before its still-unsatisfied peers are moved to a per-block abandoned set and no longer
101
+ * pushed to. Without this bound a peer that permanently refuses would be re-pushed on every check
102
+ * forever, and its block would re-consume `growthBlockBudget` slots and starve genuinely-new
103
+ * growth. An abandoned peer that leaves the cohort and later rejoins is retried from scratch.
104
+ * Default: 5
105
+ */
106
+ growthMaxAttempts?: number
107
+ /**
108
+ * Self-arming re-check timer for outstanding growth work (reported-but-unconfirmed peers, or
109
+ * blocks deferred by `growthBlockBudget`). Checks otherwise fire only on libp2p connection
110
+ * events, so a failed push on a then-quiet network would never be retried. Armed at the end of a
111
+ * check only while work is outstanding; fires `maybeRebalance()` so the existing
112
+ * `minRebalanceIntervalMs` throttle still bounds the push rate. `0` disables.
113
+ * Default: `minRebalanceIntervalMs`
114
+ */
115
+ growthRecheckIntervalMs?: number
116
+ }
117
+
118
+ export interface RebalanceMonitorDeps {
119
+ libp2p: Libp2p
120
+ fret: FretService
121
+ partitionDetector: PartitionDetector
122
+ fretAdapter: ArachnodeFretAdapter
123
+ /**
124
+ * The owned-block tracked set. When provided (e.g. the shared `ownedBlocks` set wired in
125
+ * `libp2p-node-base`), the monitor references this exact `Set` instead of constructing its own,
126
+ * so it stays in lock-step with the `SpreadOnChurnMonitor` that shares it. Omit for standalone
127
+ * construction (unit tests) — a fresh private `Set` preserves all existing behavior. Note: only
128
+ * `trackedBlocks` is shared; `responsibilitySnapshot` stays per-monitor (it is rebalance's own
129
+ * was-responsible memory, not owned-block tracking).
130
+ */
131
+ trackedBlocks?: Set<string>
132
+ /**
133
+ * The node's key network. When provided, a block's cohort is `findCluster` on its routing key —
134
+ * the rule the writer routes by, the coordinator refuses writes by, and `RepoService` redirects
135
+ * by — so the monitor never calls a peer responsible (and pushes it a replica) that the rest of
136
+ * the node does not, nor releases a block the node is still responsible for. Omit for standalone
137
+ * construction (unit tests): the cohort is then FRET's nearest {@link RebalanceMonitor.getCohortSize}.
138
+ */
139
+ keyNetwork?: Pick<IKeyNetwork, 'findCluster'>
140
+ /**
141
+ * The node's resolved `clusterSize`. Caps {@link RebalanceMonitor.getCohortSize}: a floor wider
142
+ * than the cohort can never be confirmed (a lost block would never be released), and on the
143
+ * standalone path it would assemble peers outside the configured cohort.
144
+ */
145
+ clusterSize?: number
146
+ }
147
+
148
+ type RebalanceHandler = (event: RebalanceEvent) => void
149
+
150
+ /** Per-block rebalance/growth state (the `responsibilitySnapshot` entry). */
151
+ interface BlockGrowthState {
152
+ responsible: boolean
153
+ /**
154
+ * The growth arm's seen set: peers CONFIRMED to hold a replica of this block (or satisfied
155
+ * another way — floor met, or nothing local to push). Peers enter ONLY via
156
+ * {@link RebalanceMonitor.recordGrowthOutcome} or an acknowledged commit's holders
157
+ * ({@link RebalanceMonitor.recordCommittedHolders}), never at report time.
158
+ */
159
+ cohortPeers: Set<string>
160
+ /** Peers reported grown at the last emitting check whose confirmation is still outstanding. */
161
+ pendingPeers: Set<string>
162
+ /** Consecutive incomplete growth outcomes — the give-up counter against `growthMaxAttempts`. */
163
+ growthAttempts: number
164
+ /**
165
+ * Peers given up on after `growthMaxAttempts` incomplete outcomes excluded from growth reports
166
+ * until they leave the cohort and rejoin (each check intersects this with the current cohort).
167
+ */
168
+ abandonedPeers: Set<string>
169
+ }
170
+
171
+ const emptyGrowthState = (responsible: boolean): BlockGrowthState => ({
172
+ responsible,
173
+ cohortPeers: new Set<string>(),
174
+ pendingPeers: new Set<string>(),
175
+ growthAttempts: 0,
176
+ abandonedPeers: new Set<string>()
177
+ })
178
+
179
+ const intersect = (remembered: Set<string> | undefined, current: Set<string>): Set<string> =>
180
+ new Set([...(remembered ?? [])].filter(id => current.has(id)))
181
+
182
+ /**
183
+ * Carry a still-responsible block's growth state into the next check, intersecting both remembered
184
+ * peer sets against the CURRENT cohort: a peer that left drops out of `cohortPeers` (so its return
185
+ * is re-detected — the departure self-heal) and out of `abandonedPeers` (so a rejoin is retried from
186
+ * scratch). Commit evidence gathered since the last check is folded into `cohortPeers` first.
187
+ * `pendingPeers` is always rebuilt from this check's own report, never carried.
188
+ */
189
+ const carryGrowthState = (
190
+ prior: BlockGrowthState | undefined,
191
+ currentPeers: Set<string>,
192
+ evidence: CommitEvidence | undefined
193
+ ): BlockGrowthState => {
194
+ const confirmed = new Set([...(prior?.cohortPeers ?? []), ...(evidence?.holders ?? [])])
195
+ for (const peerId of evidence?.unconfirmed ?? []) confirmed.delete(peerId)
196
+ return {
197
+ responsible: true,
198
+ cohortPeers: intersect(confirmed, currentPeers),
199
+ pendingPeers: new Set<string>(),
200
+ growthAttempts: prior?.growthAttempts ?? 0,
201
+ abandonedPeers: intersect(prior?.abandonedPeers, currentPeers)
202
+ }
203
+ }
204
+
205
+ export class RebalanceMonitor implements Startable {
206
+ private running = false
207
+ private readonly trackedBlocks: Set<string>
208
+ // Per-monitor was-responsible memory (NOT shared, unlike trackedBlocks). When the shared
209
+ // trackedBlocks set is mutated externally — spread's no-local-data self-prune, or the node's
210
+ // responsibility-loss eviction going through untrackBlock — a snapshot entry for a since-removed
211
+ // block may linger here. That is acceptable: performRebalanceCheck only iterates trackedBlocks, so
212
+ // a lingering entry is inert; if the block is later re-fed, its responsibility is simply re-derived.
213
+ //
214
+ // `cohortPeers` is the growth arm's CONFIRMED-co-responsible memory. A MISSING entry is
215
+ // deliberately treated as "prior cohort = empty", so the first check after a topology event
216
+ // reports the whole non-self cohort as grown — that is what heals the founder case (A alone
217
+ // commits; B joins → first check pushes to B) and the restarted-holder case (snapshot memory is
218
+ // process-local, so a restarted holder re-pushes to everyone once). Peers enter the set only
219
+ // through recordGrowthOutcome or a commit's confirmed holders (recordCommittedHolders) — a
220
+ // reported-but-unconfirmed peer stays out, so a failed push is re-detected on the next check
221
+ // instead of being recorded as done.
222
+ private readonly responsibilitySnapshot = new Map<string, BlockGrowthState>()
223
+ // Commit evidence (recordCommittedHolders) waiting for the next check, which consumes each block's
224
+ // entry once its cohort lookup succeeds and drops entries for blocks no longer tracked. Held apart
225
+ // from the snapshot because a commit does not say whether this node is responsible — only a check does.
226
+ private readonly commitEvidence = new Map<string, CommitEvidence>()
227
+ private readonly handlers: RebalanceHandler[] = []
228
+ private debounceTimer: ReturnType<typeof setTimeout> | null = null
229
+ private recheckTimer: ReturnType<typeof setTimeout> | null = null
230
+ private lastRebalanceAt = 0
231
+ private pendingTopologyChange = false
232
+ private topologyChangeTimestamp = 0
233
+ private lastGrowthDeferred = 0
234
+
235
+ private readonly debounceMs: number
236
+ private readonly minRebalanceIntervalMs: number
237
+ private readonly suppressDuringPartition: boolean
238
+ private readonly growthBlockBudget: number
239
+ private readonly growthMaxAttempts: number
240
+ private readonly growthRecheckIntervalMs: number
241
+
242
+ private readonly onConnectionOpen: () => void
243
+ private readonly onConnectionClose: () => void
244
+
245
+ constructor(
246
+ private readonly deps: RebalanceMonitorDeps,
247
+ config: RebalanceMonitorConfig = {}
248
+ ) {
249
+ // Share the injected owned-block set when present (so spread + rebalance never drift);
250
+ // otherwise own a private set (standalone construction / unit tests). Only trackedBlocks is
251
+ // shared — responsibilitySnapshot stays per-monitor.
252
+ this.trackedBlocks = deps.trackedBlocks ?? new Set<string>()
253
+ this.debounceMs = config.debounceMs ?? 5000
254
+ this.minRebalanceIntervalMs = config.minRebalanceIntervalMs ?? 60000
255
+ this.suppressDuringPartition = config.suppressDuringPartition ?? true
256
+ this.growthBlockBudget = config.growthBlockBudget ?? 64
257
+ this.growthMaxAttempts = config.growthMaxAttempts ?? 5
258
+ this.growthRecheckIntervalMs = config.growthRecheckIntervalMs ?? this.minRebalanceIntervalMs
259
+
260
+ this.onConnectionOpen = () => this.handleTopologyChange()
261
+ this.onConnectionClose = () => this.handleTopologyChange()
262
+ }
263
+
264
+ async start(): Promise<void> {
265
+ if (this.running) return
266
+ this.running = true
267
+
268
+ this.deps.libp2p.addEventListener('connection:open', this.onConnectionOpen)
269
+ this.deps.libp2p.addEventListener('connection:close', this.onConnectionClose)
270
+
271
+ log('started, tracking %d blocks', this.trackedBlocks.size)
272
+ }
273
+
274
+ async stop(): Promise<void> {
275
+ if (!this.running) return
276
+ this.running = false
277
+
278
+ this.deps.libp2p.removeEventListener('connection:open', this.onConnectionOpen)
279
+ this.deps.libp2p.removeEventListener('connection:close', this.onConnectionClose)
280
+
281
+ if (this.debounceTimer) {
282
+ clearTimeout(this.debounceTimer)
283
+ this.debounceTimer = null
284
+ }
285
+ if (this.recheckTimer) {
286
+ clearTimeout(this.recheckTimer)
287
+ this.recheckTimer = null
288
+ }
289
+
290
+ this.pendingTopologyChange = false
291
+ this.commitEvidence.clear()
292
+ log('stopped')
293
+ }
294
+
295
+ onRebalance(handler: RebalanceHandler): void {
296
+ this.handlers.push(handler)
297
+ }
298
+
299
+ trackBlock(blockId: string): void {
300
+ this.trackedBlocks.add(blockId)
301
+ }
302
+
303
+ untrackBlock(blockId: string): void {
304
+ this.trackedBlocks.delete(blockId)
305
+ this.responsibilitySnapshot.delete(blockId)
306
+ this.commitEvidence.delete(blockId)
307
+ }
308
+
309
+ getTrackedBlockCount(): number {
310
+ return this.trackedBlocks.size
311
+ }
312
+
313
+ async checkNow(): Promise<RebalanceEvent | null> {
314
+ return this.performRebalanceCheck(Date.now())
315
+ }
316
+
317
+ /**
318
+ * Feedback from the growth reaction for one block reported `grown`. `satisfiedPeers` enter the
319
+ * block's seen set; an incomplete outcome counts an attempt against `growthMaxAttempts`, and on
320
+ * reaching the bound the block's still-unsatisfied reported peers are abandoned (no longer
321
+ * pushed to until they leave the cohort and rejoin). Never called for a block the reaction had
322
+ * no information about (a confirm deduped against one already in flight) a missing outcome
323
+ * leaves the block's state untouched so the next check retries.
324
+ */
325
+ recordGrowthOutcome(blockId: string, outcome: GrowthOutcome): void {
326
+ const state = this.responsibilitySnapshot.get(blockId)
327
+ // No state (untracked since) or responsibility lost since the report: the growth state was
328
+ // cleared, and recording into it would survive the clear and suppress the regain re-push.
329
+ if (!state || !state.responsible) return
330
+
331
+ for (const peerId of outcome.satisfiedPeers) {
332
+ state.cohortPeers.add(peerId)
333
+ state.pendingPeers.delete(peerId)
334
+ }
335
+
336
+ if (outcome.complete) {
337
+ state.growthAttempts = 0
338
+ state.pendingPeers.clear()
339
+ } else {
340
+ // NOTE: growthMaxAttempts is a floor on the retry count, not an exact one. `pendingPeers` is
341
+ // rebuilt from each check's own report, so two cases blunt the bound: a check that defers
342
+ // this block on growthBlockBudget clears pendingPeers, and a give-up landing right then
343
+ // abandons nobody while still resetting the counter; and two checks racing (the second
344
+ // re-reporting the same peer after the first's confirm left `inFlight` but before its outcome
345
+ // landed) double-count one attempt. Both are rare, both only change how many pushes a doomed
346
+ // peer absorbs. If a deployment ever tracks far more blocks than growthBlockBudget, deferral
347
+ // stops being rare — carry the report's peer list on GrowthOutcome and match it against
348
+ // pendingPeers instead of trusting the latest report.
349
+ state.growthAttempts++
350
+ if (state.growthAttempts >= this.growthMaxAttempts) {
351
+ for (const peerId of state.pendingPeers) {
352
+ state.abandonedPeers.add(peerId)
353
+ }
354
+ log('growth give-up: block=%s abandoning %d unsatisfied peer(s) after %d attempts',
355
+ blockId, state.pendingPeers.size, state.growthAttempts)
356
+ state.pendingPeers.clear()
357
+ state.growthAttempts = 0
358
+ }
359
+ }
360
+
361
+ this.updateRecheckTimer()
362
+ }
363
+
364
+ /**
365
+ * Evidence from an acknowledged commit that this node's storage holds: `holders` are recorded as
366
+ * confirmed co-holders of each block at the next check, and the block is not reported `gained`
367
+ * then (this node already holds it, so there is nothing to pull). Without this, every freshly
368
+ * committed block has no growth memory, so the next check reports the whole cohort grown and
369
+ * pushes the block back to the members that stored it as part of the commit — and reports it
370
+ * gained, pulling it back from them.
371
+ *
372
+ * Later evidence for the same block overrides earlier evidence about the same peer, so the
373
+ * coordinator's durability-checked answer corrects the member-side signer list recorded moments
374
+ * before it on the coordinating node. Ignored while stopped: nothing would consume it.
375
+ *
376
+ * Does not affect the founder case: a commit no other peer confirmed names no holders, so the
377
+ * cohort is still reported grown once peers appear.
378
+ */
379
+ recordCommittedHolders(committed: CommittedHolders): void {
380
+ if (!this.running) return
381
+ for (const blockId of committed.blockIds) {
382
+ let evidence = this.commitEvidence.get(blockId)
383
+ if (!evidence) {
384
+ evidence = { holders: new Set<string>(), unconfirmed: new Set<string>() }
385
+ this.commitEvidence.set(blockId, evidence)
386
+ }
387
+ for (const peerId of committed.holders) {
388
+ evidence.holders.add(peerId)
389
+ evidence.unconfirmed.delete(peerId)
390
+ }
391
+ for (const peerId of committed.unconfirmed ?? []) {
392
+ evidence.unconfirmed.add(peerId)
393
+ evidence.holders.delete(peerId)
394
+ }
395
+ }
396
+ }
397
+
398
+ /**
399
+ * Growth-arm observability: how many tracked blocks still await confirmation on reported peers,
400
+ * how many (block, peer) pairs have been given up on, and whether the re-check timer is armed.
401
+ */
402
+ getGrowthDiagnostics(): { blocksAwaitingConfirmation: number; abandonedPairs: number; recheckArmed: boolean } {
403
+ let blocksAwaitingConfirmation = 0
404
+ let abandonedPairs = 0
405
+ for (const [blockId, state] of this.responsibilitySnapshot) {
406
+ if (!state.responsible || !this.trackedBlocks.has(blockId)) continue
407
+ if (state.pendingPeers.size > 0) blocksAwaitingConfirmation++
408
+ abandonedPairs += state.abandonedPeers.size
409
+ }
410
+ return { blocksAwaitingConfirmation, abandonedPairs, recheckArmed: this.recheckTimer !== null }
411
+ }
412
+
413
+ /** Growth work is outstanding while any reported peer is unconfirmed or blocks were budget-deferred. */
414
+ private hasOutstandingGrowthWork(): boolean {
415
+ if (this.lastGrowthDeferred > 0) return true
416
+ for (const [blockId, state] of this.responsibilitySnapshot) {
417
+ if (state.responsible && state.pendingPeers.size > 0 && this.trackedBlocks.has(blockId)) {
418
+ return true
419
+ }
420
+ }
421
+ return false
422
+ }
423
+
424
+ /**
425
+ * Arm the growth re-check timer while work is outstanding; disarm it when there is none. The
426
+ * timer fires maybeRebalance(), so minRebalanceIntervalMs still bounds the push rate, and it
427
+ * re-arms itself after firing for as long as work remains. unref'd so it never holds the
428
+ * process open; stop() clears it.
429
+ */
430
+ private updateRecheckTimer(): void {
431
+ if (this.growthRecheckIntervalMs <= 0) return
432
+
433
+ if (!this.running || !this.hasOutstandingGrowthWork()) {
434
+ if (this.recheckTimer) {
435
+ clearTimeout(this.recheckTimer)
436
+ this.recheckTimer = null
437
+ }
438
+ return
439
+ }
440
+
441
+ if (this.recheckTimer) return // already armed
442
+
443
+ this.recheckTimer = setTimeout(() => {
444
+ this.recheckTimer = null
445
+ void this.maybeRebalance()
446
+ .catch(err => { log('recheck error: %O', err) })
447
+ .finally(() => this.updateRecheckTimer())
448
+ }, this.growthRecheckIntervalMs)
449
+ ;(this.recheckTimer as unknown as { unref?: () => void }).unref?.()
450
+ }
451
+
452
+ private handleTopologyChange(): void {
453
+ if (!this.running) return
454
+
455
+ if (!this.pendingTopologyChange) {
456
+ this.topologyChangeTimestamp = Date.now()
457
+ }
458
+ this.pendingTopologyChange = true
459
+
460
+ if (this.debounceTimer) {
461
+ clearTimeout(this.debounceTimer)
462
+ }
463
+
464
+ // NOTE: the check samples FRET's cohort once, debounceMs after the last connection event. A joiner
465
+ // FRET has not admitted by then is not reported grown, and nothing re-checks until the next
466
+ // connection event outside minRebalanceIntervalMs — so a machine that wrote alone would not push its
467
+ // blocks to its new backup. Over loopback FRET admits well inside the default (the backup phase of
468
+ // `small-deployment-lifecycle.integration.spec.ts` lands in ~10s); if a slower link, such as a phone
469
+ // pair over a relay, ever shows a joiner that never receives the founder's blocks, re-check when FRET
470
+ // reports the peer instead of on this timer alone.
471
+ this.debounceTimer = setTimeout(() => {
472
+ this.debounceTimer = null
473
+ this.pendingTopologyChange = false
474
+ this.maybeRebalance()
475
+ }, this.debounceMs)
476
+ }
477
+
478
+ private async maybeRebalance(): Promise<void> {
479
+ if (!this.running) return
480
+
481
+ const now = Date.now()
482
+ const elapsed = now - this.lastRebalanceAt
483
+ if (elapsed < this.minRebalanceIntervalMs) {
484
+ log('throttled, %dms since last rebalance', elapsed)
485
+ return
486
+ }
487
+
488
+ const event = await this.performRebalanceCheck(this.topologyChangeTimestamp || now)
489
+ if (event) {
490
+ this.emitEvent(event)
491
+ }
492
+ }
493
+
494
+ private async performRebalanceCheck(triggeredAt: number): Promise<RebalanceEvent | null> {
495
+ if (this.suppressDuringPartition && this.deps.partitionDetector.detectPartition()) {
496
+ log('partition detected, suppressing rebalance')
497
+ return null
498
+ }
499
+
500
+ if (this.trackedBlocks.size === 0) {
501
+ this.lastRebalanceAt = Date.now()
502
+ // Nothing left to grow, so any prior deferral is moot — drop it and let the re-check timer
503
+ // disarm, rather than re-arming forever against blocks that were untracked out from under it.
504
+ this.lastGrowthDeferred = 0
505
+ this.commitEvidence.clear()
506
+ this.updateRecheckTimer()
507
+ return null
508
+ }
509
+
510
+ const selfId = this.deps.libp2p.peerId.toString()
511
+ const gained: string[] = []
512
+ const lost: string[] = []
513
+ const newOwners = new Map<string, string[]>()
514
+ const grown = new Map<string, string[]>()
515
+ let growthDeferred = 0
516
+ let lookupFailed = 0
517
+ const growthCandidates: Array<{ blockId: string; newPeers: string[]; state: BlockGrowthState }> = []
518
+
519
+ for (const blockId of this.trackedBlocks) {
520
+ let cohort: string[]
521
+ try {
522
+ cohort = await this.cohortFor(blockId)
523
+ } catch (err) {
524
+ // Responsibility is undetermined: leave this block's state untouched (never guess lost or
525
+ // gained off a failed lookup) and count it as outstanding so the re-check timer retries it.
526
+ log('cohort lookup failed: block=%s %o', blockId, err)
527
+ lookupFailed++
528
+ continue
529
+ }
530
+ const isResponsible = cohort.includes(selfId)
531
+ const prior = this.responsibilitySnapshot.get(blockId)
532
+ const wasResponsible = prior?.responsible ?? false
533
+ // Consumed only once the lookup succeeded, so a failed lookup leaves it for the next check.
534
+ // NOTE: a block enters trackedBlocks at storage apply, but its evidence arrives only after the
535
+ // durable verdict (member) or cohort acknowledgement (coordinator); a check landing in that
536
+ // window still reports it gained + grown, the pre-evidence behaviour. If relay traces show
537
+ // post-commit transfers surviving, skip a block's first check while a commit touching it is in flight.
538
+ const evidence = this.commitEvidence.get(blockId)
539
+ this.commitEvidence.delete(blockId)
540
+
541
+ if (isResponsible && !wasResponsible && !evidence) {
542
+ gained.push(blockId)
543
+ } else if (!isResponsible && wasResponsible) {
544
+ lost.push(blockId)
545
+ // The cohort members are the new owners
546
+ newOwners.set(blockId, cohort.filter(id => id !== selfId))
547
+ }
548
+
549
+ // Growth arm: while this node STAYS responsible, any cohort peer not yet CONFIRMED to hold
550
+ // the block (and not abandoned) gets it pushed (up to the floor). Runs on every responsible
551
+ // check — NOT gated on wasResponsible — so a first observation (no snapshot entry) treats
552
+ // the whole non-self cohort as new; see the responsibilitySnapshot comment for why that is
553
+ // load-bearing. Not responsible ⇒ arm skipped, so `lost` ∩ `grown` is impossible.
554
+ //
555
+ // Reporting a peer does NOT record it as seen — only recordGrowthOutcome does — so a push
556
+ // that fails (dial timeout, receiver refused to persist, partition mid-reaction, reaction
557
+ // threw) leaves the peer un-seen and the next check re-detects it.
558
+ let state: BlockGrowthState
559
+ if (isResponsible) {
560
+ const currentSet = new Set(cohort.filter(id => id !== selfId))
561
+ state = carryGrowthState(prior, currentSet, evidence)
562
+ const newPeers = [...currentSet].filter(id => !state.cohortPeers.has(id) && !state.abandonedPeers.has(id))
563
+ if (newPeers.length > 0) {
564
+ growthCandidates.push({ blockId, newPeers, state })
565
+ } else {
566
+ // Nothing owed for this block. growthAttempts counts CONSECUTIVE incomplete outcomes
567
+ // against outstanding growth, so it must not carry across a quiet stretch — otherwise a
568
+ // block that failed a few times, then had that peer leave, would spend the leftovers on
569
+ // whichever peer joins next and abandon it early.
570
+ state.growthAttempts = 0
571
+ }
572
+ } else {
573
+ // Not responsible: clear ALL growth state (seen set, attempts, abandoned peers), so a
574
+ // later regain re-pushes to the whole cohort (benign when the local data is gone — the
575
+ // push finds nothing and no-ops).
576
+ state = emptyGrowthState(false)
577
+ }
578
+
579
+ this.responsibilitySnapshot.set(blockId, state)
580
+ }
581
+
582
+ // Evidence for a block that is not tracked has no consumer: a commit this node does not hold
583
+ // as an owned block, or one untracked since.
584
+ for (const blockId of [...this.commitEvidence.keys()]) {
585
+ if (!this.trackedBlocks.has(blockId)) this.commitEvidence.delete(blockId)
586
+ }
587
+
588
+ // Fill the growth budget in two passes: fresh growth (no failed attempts yet) first, retrying
589
+ // blocks with what remains — otherwise a stuck retry set at the front of the tracked-block
590
+ // insertion order would starve peers that just joined.
591
+ for (const candidate of [
592
+ ...growthCandidates.filter(c => c.state.growthAttempts === 0),
593
+ ...growthCandidates.filter(c => c.state.growthAttempts > 0)
594
+ ]) {
595
+ if (grown.size < this.growthBlockBudget) {
596
+ grown.set(candidate.blockId, candidate.newPeers)
597
+ candidate.state.pendingPeers = new Set(candidate.newPeers)
598
+ } else {
599
+ // Budget-dropped: the seen set was not touched, so the next check re-detects the same
600
+ // growth — a deferral, not a loss.
601
+ growthDeferred++
602
+ }
603
+ }
604
+ this.lastGrowthDeferred = growthDeferred + lookupFailed
605
+
606
+ this.lastRebalanceAt = Date.now()
607
+
608
+ if (growthDeferred > 0) {
609
+ // Deferred blocks drain one budget-full per check. Checks fire on libp2p connection events
610
+ // AND — while growth work is outstanding — on the growthRecheckIntervalMs timer armed
611
+ // below, so a backlog drains even on a quiet network.
612
+ log('growth budget reached: %d blocks deferred to the next check (budget=%d)',
613
+ growthDeferred, this.growthBlockBudget)
614
+ }
615
+
616
+ this.updateRecheckTimer()
617
+
618
+ if (gained.length === 0 && lost.length === 0 && grown.size === 0) {
619
+ return null
620
+ }
621
+
622
+ log('rebalance check: gained=%d lost=%d grown=%d', gained.length, lost.length, grown.size)
623
+
624
+ return { gained, lost, newOwners, grown, floor: this.getCohortSize(), triggeredAt }
625
+ }
626
+
627
+ /**
628
+ * The peers responsible for a block, nearest first. With a key network wired this is its
629
+ * `findCluster` cohort (see {@link RebalanceMonitorDeps.keyNetwork}); otherwise FRET's nearest
630
+ * {@link getCohortSize} ring members.
631
+ *
632
+ * NOTE: one `findCluster` (peer-store reads for each cohort member) per tracked block per pass, where
633
+ * FRET's cohort was one synchronous call; fine under the pass throttle; if passes on nodes holding many
634
+ * blocks ever show up as slow, share the lookup with the coordinator's responsibility cache.
635
+ */
636
+ private async cohortFor(blockId: string): Promise<string[]> {
637
+ const key = routingKeyForBlock(blockId)
638
+ if (this.deps.keyNetwork) {
639
+ return Object.keys(await this.deps.keyNetwork.findCluster(key))
640
+ }
641
+ return this.deps.fret.assembleCohort(await hashKey(key), this.getCohortSize())
642
+ }
643
+
644
+ /**
645
+ * The replication floor `N`. Public so the ring-shift handoff and the rebalance reaction can gate
646
+ * release on confirming replication to this many holders (`docs/arachnode-ring-handoff.md`
647
+ * § Replication floor). Derives from FRET's network-size estimate: `clamp(ceil(sqrt(n_est)), 1, 3)`,
648
+ * defaulting to 3 when no confident estimate exists — and never above the configured `clusterSize`,
649
+ * since no block has more responsible holders than that to confirm on.
650
+ */
651
+ getCohortSize(): number {
652
+ const diag: any = (this.deps.fret as any).getDiagnostics?.()
653
+ const estimate = diag?.estimate ?? diag?.n
654
+ const size = typeof estimate === 'number' && Number.isFinite(estimate) && estimate > 0
655
+ ? Math.max(1, Math.min(3, Math.ceil(Math.sqrt(estimate))))
656
+ : 3
657
+ const cap = this.deps.clusterSize
658
+ return cap != null && cap >= 1 ? Math.min(size, cap) : size
659
+ }
660
+
661
+ private emitEvent(event: RebalanceEvent): void {
662
+ for (const handler of this.handlers) {
663
+ try {
664
+ handler(event)
665
+ } catch (err) {
666
+ log('handler error: %O', err)
667
+ }
668
+ }
669
+ }
670
+
671
+ /**
672
+ * Update ArachnodeInfo status through the fret adapter.
673
+ */
674
+ setStatus(status: ArachnodeInfo['status']): void {
675
+ this.deps.fretAdapter.setStatus(status)
676
+ }
677
+ }