@optimystic/db-p2p 1.0.0-beta.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cluster/block-transfer-service.d.ts +73 -1
- package/dist/src/cluster/block-transfer-service.d.ts.map +1 -1
- package/dist/src/cluster/block-transfer-service.js +70 -3
- package/dist/src/cluster/block-transfer-service.js.map +1 -1
- package/dist/src/cluster/block-transfer.d.ts +16 -1
- package/dist/src/cluster/block-transfer.d.ts.map +1 -1
- package/dist/src/cluster/block-transfer.js +73 -72
- package/dist/src/cluster/block-transfer.js.map +1 -1
- package/dist/src/cluster/client.d.ts +4 -6
- package/dist/src/cluster/client.d.ts.map +1 -1
- package/dist/src/cluster/client.js +8 -13
- package/dist/src/cluster/client.js.map +1 -1
- package/dist/src/cluster/cluster-policy.d.ts +2 -1
- package/dist/src/cluster/cluster-policy.d.ts.map +1 -1
- package/dist/src/cluster/cluster-policy.js.map +1 -1
- package/dist/src/cluster/cluster-repo.d.ts +172 -17
- package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
- package/dist/src/cluster/cluster-repo.js +553 -149
- package/dist/src/cluster/cluster-repo.js.map +1 -1
- package/dist/src/cluster/commit-proof.js +2 -2
- package/dist/src/cluster/race-resolution.d.ts +51 -7
- package/dist/src/cluster/race-resolution.d.ts.map +1 -1
- package/dist/src/cluster/race-resolution.js +71 -7
- package/dist/src/cluster/race-resolution.js.map +1 -1
- package/dist/src/cluster/rebalance-monitor.d.ts +67 -5
- package/dist/src/cluster/rebalance-monitor.d.ts.map +1 -1
- package/dist/src/cluster/rebalance-monitor.js +120 -26
- package/dist/src/cluster/rebalance-monitor.js.map +1 -1
- package/dist/src/cluster/spread-on-churn.d.ts.map +1 -1
- package/dist/src/cluster/spread-on-churn.js +34 -47
- package/dist/src/cluster/spread-on-churn.js.map +1 -1
- package/dist/src/cohort-topic/host.d.ts.map +1 -1
- package/dist/src/cohort-topic/host.js +8 -6
- package/dist/src/cohort-topic/host.js.map +1 -1
- package/dist/src/cohort-topic/membership-source.js +1 -1
- package/dist/src/cohort-topic/membership-source.js.map +1 -1
- package/dist/src/cohort-topic/peer-codec.d.ts.map +1 -1
- package/dist/src/cohort-topic/peer-codec.js +5 -2
- package/dist/src/cohort-topic/peer-codec.js.map +1 -1
- package/dist/src/cohort-topic/stream-util.d.ts +24 -3
- package/dist/src/cohort-topic/stream-util.d.ts.map +1 -1
- package/dist/src/cohort-topic/stream-util.js +36 -3
- package/dist/src/cohort-topic/stream-util.js.map +1 -1
- package/dist/src/cohort-topic/topic-router.d.ts +9 -0
- package/dist/src/cohort-topic/topic-router.d.ts.map +1 -1
- package/dist/src/cohort-topic/topic-router.js +12 -2
- package/dist/src/cohort-topic/topic-router.js.map +1 -1
- package/dist/src/dispute/client.d.ts.map +1 -1
- package/dist/src/dispute/client.js +15 -3
- package/dist/src/dispute/client.js.map +1 -1
- package/dist/src/index.d.ts +5 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +5 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/libp2p-key-network.d.ts +78 -17
- package/dist/src/libp2p-key-network.d.ts.map +1 -1
- package/dist/src/libp2p-key-network.js +241 -147
- package/dist/src/libp2p-key-network.js.map +1 -1
- package/dist/src/libp2p-node-base.d.ts +35 -1
- package/dist/src/libp2p-node-base.d.ts.map +1 -1
- package/dist/src/libp2p-node-base.js +151 -26
- package/dist/src/libp2p-node-base.js.map +1 -1
- package/dist/src/libp2p-node-rn.d.ts +2 -2
- package/dist/src/libp2p-node-rn.d.ts.map +1 -1
- package/dist/src/libp2p-node-rn.js.map +1 -1
- package/dist/src/libp2p-node.d.ts +2 -2
- package/dist/src/libp2p-node.d.ts.map +1 -1
- package/dist/src/libp2p-node.js.map +1 -1
- package/dist/src/logger.d.ts.map +1 -1
- package/dist/src/logger.js +19 -7
- package/dist/src/logger.js.map +1 -1
- package/dist/src/matchmaking/query-transport.d.ts +3 -4
- package/dist/src/matchmaking/query-transport.d.ts.map +1 -1
- package/dist/src/matchmaking/query-transport.js +9 -9
- package/dist/src/matchmaking/query-transport.js.map +1 -1
- package/dist/src/network/network-manager-service.d.ts +10 -5
- package/dist/src/network/network-manager-service.d.ts.map +1 -1
- package/dist/src/network/network-manager-service.js +11 -2
- package/dist/src/network/network-manager-service.js.map +1 -1
- package/dist/src/network/relay-reservation.d.ts +170 -0
- package/dist/src/network/relay-reservation.d.ts.map +1 -0
- package/dist/src/network/relay-reservation.js +527 -0
- package/dist/src/network/relay-reservation.js.map +1 -0
- package/dist/src/pend-validation.d.ts +3 -2
- package/dist/src/pend-validation.d.ts.map +1 -1
- package/dist/src/pend-validation.js +3 -2
- package/dist/src/pend-validation.js.map +1 -1
- package/dist/src/protocol-client.d.ts +8 -1
- package/dist/src/protocol-client.d.ts.map +1 -1
- package/dist/src/protocol-client.js +11 -0
- package/dist/src/protocol-client.js.map +1 -1
- package/dist/src/reactivity/forwarder-host.d.ts +2 -1
- package/dist/src/reactivity/forwarder-host.d.ts.map +1 -1
- package/dist/src/reactivity/forwarder-host.js +2 -1
- package/dist/src/reactivity/forwarder-host.js.map +1 -1
- package/dist/src/reactivity/origination-manager.d.ts +3 -3
- package/dist/src/reactivity/origination-manager.js +3 -3
- package/dist/src/reactivity/recover-transport.d.ts +22 -14
- package/dist/src/reactivity/recover-transport.d.ts.map +1 -1
- package/dist/src/reactivity/recover-transport.js +32 -19
- package/dist/src/reactivity/recover-transport.js.map +1 -1
- package/dist/src/reactivity/subscription-manager.d.ts +2 -2
- package/dist/src/reactivity/topic-bytes.d.ts +6 -7
- package/dist/src/reactivity/topic-bytes.d.ts.map +1 -1
- package/dist/src/reactivity/topic-bytes.js +8 -9
- package/dist/src/reactivity/topic-bytes.js.map +1 -1
- package/dist/src/repo/client.d.ts +5 -2
- package/dist/src/repo/client.d.ts.map +1 -1
- package/dist/src/repo/client.js +49 -40
- package/dist/src/repo/client.js.map +1 -1
- package/dist/src/repo/cluster-coordinator.d.ts +119 -5
- package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
- package/dist/src/repo/cluster-coordinator.js +181 -29
- package/dist/src/repo/cluster-coordinator.js.map +1 -1
- package/dist/src/repo/coordinator-repo.d.ts +259 -76
- package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
- package/dist/src/repo/coordinator-repo.js +716 -328
- package/dist/src/repo/coordinator-repo.js.map +1 -1
- package/dist/src/repo/i-under-replication-ledger.d.ts +87 -0
- package/dist/src/repo/i-under-replication-ledger.d.ts.map +1 -0
- package/dist/src/repo/i-under-replication-ledger.js +2 -0
- package/dist/src/repo/i-under-replication-ledger.js.map +1 -0
- package/dist/src/repo/kv-under-replication-ledger.d.ts +91 -0
- package/dist/src/repo/kv-under-replication-ledger.d.ts.map +1 -0
- package/dist/src/repo/kv-under-replication-ledger.js +287 -0
- package/dist/src/repo/kv-under-replication-ledger.js.map +1 -0
- package/dist/src/repo/responsibility.d.ts +37 -0
- package/dist/src/repo/responsibility.d.ts.map +1 -0
- package/dist/src/repo/responsibility.js +37 -0
- package/dist/src/repo/responsibility.js.map +1 -0
- package/dist/src/repo/service.d.ts +44 -15
- package/dist/src/repo/service.d.ts.map +1 -1
- package/dist/src/repo/service.js +64 -27
- package/dist/src/repo/service.js.map +1 -1
- package/dist/src/repo/stuck-reservation.d.ts +127 -0
- package/dist/src/repo/stuck-reservation.d.ts.map +1 -0
- package/dist/src/repo/stuck-reservation.js +179 -0
- package/dist/src/repo/stuck-reservation.js.map +1 -0
- package/dist/src/repo/under-replication-drain.d.ts +208 -0
- package/dist/src/repo/under-replication-drain.d.ts.map +1 -0
- package/dist/src/repo/under-replication-drain.js +461 -0
- package/dist/src/repo/under-replication-drain.js.map +1 -0
- package/dist/src/rn.d.ts +5 -0
- package/dist/src/rn.d.ts.map +1 -1
- package/dist/src/rn.js +5 -0
- package/dist/src/rn.js.map +1 -1
- package/dist/src/storage/block-latch.d.ts +3 -2
- package/dist/src/storage/block-latch.d.ts.map +1 -1
- package/dist/src/storage/block-latch.js +7 -4
- package/dist/src/storage/block-latch.js.map +1 -1
- package/dist/src/storage/block-storage.d.ts +45 -3
- package/dist/src/storage/block-storage.d.ts.map +1 -1
- package/dist/src/storage/block-storage.js +156 -9
- package/dist/src/storage/block-storage.js.map +1 -1
- package/dist/src/storage/i-block-storage.d.ts +57 -5
- package/dist/src/storage/i-block-storage.d.ts.map +1 -1
- package/dist/src/storage/i-block-storage.js.map +1 -1
- package/dist/src/storage/pending-claim.d.ts +167 -0
- package/dist/src/storage/pending-claim.d.ts.map +1 -0
- package/dist/src/storage/pending-claim.js +156 -0
- package/dist/src/storage/pending-claim.js.map +1 -0
- package/dist/src/storage/raw-store-codec.d.ts +13 -0
- package/dist/src/storage/raw-store-codec.d.ts.map +1 -1
- package/dist/src/storage/raw-store-codec.js +24 -3
- package/dist/src/storage/raw-store-codec.js.map +1 -1
- package/dist/src/storage/restoration-coordinator.d.ts.map +1 -1
- package/dist/src/storage/restoration-coordinator.js +2 -1
- package/dist/src/storage/restoration-coordinator.js.map +1 -1
- package/dist/src/storage/ring-shift-coordinator.d.ts.map +1 -1
- package/dist/src/storage/ring-shift-coordinator.js +3 -3
- package/dist/src/storage/ring-shift-coordinator.js.map +1 -1
- package/dist/src/storage/storage-repo.d.ts +137 -5
- package/dist/src/storage/storage-repo.d.ts.map +1 -1
- package/dist/src/storage/storage-repo.js +273 -68
- package/dist/src/storage/storage-repo.js.map +1 -1
- package/dist/src/storage/struct.d.ts +58 -0
- package/dist/src/storage/struct.d.ts.map +1 -1
- package/dist/src/testing/mesh-harness.d.ts +92 -11
- package/dist/src/testing/mesh-harness.d.ts.map +1 -1
- package/dist/src/testing/mesh-harness.js +246 -103
- package/dist/src/testing/mesh-harness.js.map +1 -1
- package/package.json +2 -2
- package/readme.md +27 -3
- package/src/cluster/block-transfer-service.ts +132 -4
- package/src/cluster/block-transfer.ts +79 -84
- package/src/cluster/client.ts +115 -119
- package/src/cluster/cluster-policy.ts +2 -1
- package/src/cluster/cluster-repo.ts +3115 -2671
- package/src/cluster/commit-proof.ts +2 -2
- package/src/cluster/race-resolution.ts +76 -7
- package/src/cluster/rebalance-monitor.ts +677 -539
- package/src/cluster/spread-on-churn.ts +34 -54
- package/src/cohort-topic/host.ts +8 -6
- package/src/cohort-topic/membership-source.ts +1 -1
- package/src/cohort-topic/peer-codec.ts +34 -31
- package/src/cohort-topic/stream-util.ts +39 -4
- package/src/cohort-topic/topic-router.ts +12 -2
- package/src/dispute/client.ts +17 -7
- package/src/index.ts +64 -59
- package/src/libp2p-key-network.ts +1377 -1253
- package/src/libp2p-node-base.ts +186 -27
- package/src/libp2p-node-rn.ts +31 -30
- package/src/libp2p-node.ts +37 -36
- package/src/logger.ts +20 -7
- package/src/matchmaking/query-transport.ts +9 -9
- package/src/network/network-manager-service.ts +16 -5
- package/src/network/relay-reservation.ts +636 -0
- package/src/pend-validation.ts +3 -2
- package/src/protocol-client.ts +13 -1
- package/src/reactivity/forwarder-host.ts +2 -1
- package/src/reactivity/origination-manager.ts +3 -3
- package/src/reactivity/recover-transport.ts +39 -22
- package/src/reactivity/subscription-manager.ts +2 -2
- package/src/reactivity/topic-bytes.ts +35 -37
- package/src/repo/client.ts +172 -173
- package/src/repo/cluster-coordinator.ts +1296 -1113
- package/src/repo/coordinator-repo.ts +3087 -2687
- package/src/repo/i-under-replication-ledger.ts +89 -0
- package/src/repo/kv-under-replication-ledger.ts +302 -0
- package/src/repo/responsibility.ts +48 -0
- package/src/repo/service.ts +76 -39
- package/src/repo/stuck-reservation.ts +224 -0
- package/src/repo/under-replication-drain.ts +582 -0
- package/src/rn.ts +74 -69
- package/src/storage/block-latch.ts +7 -4
- package/src/storage/block-storage.ts +161 -10
- package/src/storage/i-block-storage.ts +60 -5
- package/src/storage/pending-claim.ts +200 -0
- package/src/storage/raw-store-codec.ts +26 -3
- package/src/storage/restoration-coordinator.ts +212 -211
- package/src/storage/ring-shift-coordinator.ts +271 -271
- package/src/storage/storage-repo.ts +311 -73
- package/src/storage/struct.ts +58 -0
- package/src/testing/mesh-harness.ts +329 -115
|
@@ -1,539 +1,677 @@
|
|
|
1
|
-
import type { Startable, Libp2p } from '@libp2p/interface'
|
|
2
|
-
import { hashKey } from 'p2p-fret'
|
|
3
|
-
import type { FretService } from 'p2p-fret'
|
|
4
|
-
import
|
|
5
|
-
import type {
|
|
6
|
-
import {
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
*
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
/**
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
*
|
|
100
|
-
*
|
|
101
|
-
*
|
|
102
|
-
*
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
*
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
*
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
/**
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
return
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
if (!this.running) return
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
this.
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
const
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
//
|
|
470
|
-
//
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
if (
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
}
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
1
|
+
import type { Startable, Libp2p } from '@libp2p/interface'
|
|
2
|
+
import { hashKey } from 'p2p-fret'
|
|
3
|
+
import type { FretService } from 'p2p-fret'
|
|
4
|
+
import { routingKeyForBlock } from '@optimystic/db-core'
|
|
5
|
+
import type { IKeyNetwork } from '@optimystic/db-core'
|
|
6
|
+
import type { PartitionDetector } from './partition-detector.js'
|
|
7
|
+
import type { ArachnodeFretAdapter, ArachnodeInfo } from '../storage/arachnode-fret-adapter.js'
|
|
8
|
+
import { createLogger } from '../logger.js'
|
|
9
|
+
|
|
10
|
+
const log = createLogger('rebalance-monitor')
|
|
11
|
+
|
|
12
|
+
export interface RebalanceEvent {
|
|
13
|
+
/**
|
|
14
|
+
* Block IDs this node has gained responsibility for (the reaction pulls each). A block first seen
|
|
15
|
+
* through a commit this node holds ({@link RebalanceMonitor.recordCommittedHolders}) is not
|
|
16
|
+
* reported: there is nothing to pull.
|
|
17
|
+
*/
|
|
18
|
+
gained: string[]
|
|
19
|
+
/** Block IDs this node has lost responsibility for */
|
|
20
|
+
lost: string[]
|
|
21
|
+
/** Peers that are now closer for the lost blocks: blockId → peerId[] */
|
|
22
|
+
newOwners: Map<string, string[]>
|
|
23
|
+
/**
|
|
24
|
+
* Blocks this node KEEPS responsibility for whose cohort now contains peers it has not seen
|
|
25
|
+
* co-responsible before: blockId → the newly co-responsible peer ids (never self). This is the
|
|
26
|
+
* cohort-GROWTH arm: the founder case (a block committed while the deployment was one node) never
|
|
27
|
+
* appears in `lost` — the holder keeps the block — so without this arm nothing ever pushes the
|
|
28
|
+
* second copy and the block stays readable only by its sole holder. The reaction pushes each
|
|
29
|
+
* block to these peers (capped by the replication floor). A block can appear in both `gained`
|
|
30
|
+
* and `grown` (first observation after a restart/regain — the push then finds no local data and
|
|
31
|
+
* is a benign no-op); it can never appear in both `lost` and `grown` (lost ⇒ not responsible).
|
|
32
|
+
*/
|
|
33
|
+
grown: Map<string, string[]>
|
|
34
|
+
/**
|
|
35
|
+
* Replication floor `N` for this event — the cohort size FRET assembled at check time
|
|
36
|
+
* ({@link RebalanceMonitor.getCohortSize}). The reaction gates release of a `lost` block on
|
|
37
|
+
* confirming it replicated to this many new owners, so a lost block is never released below the
|
|
38
|
+
* floor. See `docs/arachnode-ring-handoff.md` § Part 2.
|
|
39
|
+
*/
|
|
40
|
+
floor: number
|
|
41
|
+
/** Timestamp of the topology change that triggered this */
|
|
42
|
+
triggeredAt: number
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* What the growth reaction learned about ONE block reported `grown`. Fed back to
|
|
47
|
+
* {@link RebalanceMonitor.recordGrowthOutcome} so the seen set is confirmation-driven: a peer
|
|
48
|
+
* enters a block's seen set only once a replica is confirmed on it, or once the block has
|
|
49
|
+
* otherwise reached its floor. A block the reaction had NO information about (its confirm was
|
|
50
|
+
* deduped against one already in flight) gets no outcome at all — the monitor's state stays
|
|
51
|
+
* untouched and the next check re-detects.
|
|
52
|
+
*/
|
|
53
|
+
export interface GrowthOutcome {
|
|
54
|
+
/** Newly co-responsible peers that may now be recorded as seen for this block. */
|
|
55
|
+
satisfiedPeers: string[]
|
|
56
|
+
/** True when nothing about this block is still owed a push. */
|
|
57
|
+
complete: boolean
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* What an acknowledged commit proved about who holds its blocks, reported by the node that holds them.
|
|
62
|
+
* Only ever reported for a commit THIS node's own storage durably holds, so a block named here needs
|
|
63
|
+
* no pull. See {@link RebalanceMonitor.recordCommittedHolders}.
|
|
64
|
+
*/
|
|
65
|
+
export interface CommittedHolders {
|
|
66
|
+
blockIds: readonly string[]
|
|
67
|
+
/** Peers evidenced to hold the committed revision. Self may appear; it is ignored. */
|
|
68
|
+
holders: readonly string[]
|
|
69
|
+
/**
|
|
70
|
+
* Cohort members the commit's coordinator knows did NOT confirm holding the revision. Withdraws
|
|
71
|
+
* any earlier record of them as holders, so the growth arm pushes them the block.
|
|
72
|
+
*/
|
|
73
|
+
unconfirmed?: readonly string[]
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** Evidence gathered between two checks for one block: `holders` and `unconfirmed` stay disjoint. */
|
|
77
|
+
interface CommitEvidence {
|
|
78
|
+
holders: Set<string>
|
|
79
|
+
unconfirmed: Set<string>
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export interface RebalanceMonitorConfig {
|
|
83
|
+
/** Debounce window for topology changes (ms). Default: 5000 */
|
|
84
|
+
debounceMs?: number
|
|
85
|
+
/** Maximum frequency of full rebalance scans (ms). Default: 60000 */
|
|
86
|
+
minRebalanceIntervalMs?: number
|
|
87
|
+
/** Whether to suppress rebalancing during detected partitions. Default: true */
|
|
88
|
+
suppressDuringPartition?: boolean
|
|
89
|
+
/**
|
|
90
|
+
* Maximum blocks reported `grown` per check. Bounds the work a single peer join can trigger on a
|
|
91
|
+
* node with a large owned-block set — the primary bound is already the floor (a cohort is at most
|
|
92
|
+
* floor-sized, so each grown block pushes to ≤ floor−1 peers, and the reaction stops per block
|
|
93
|
+
* once the floor is met); this cap bounds the block COUNT per pass. A block dropped by the cap is
|
|
94
|
+
* NOT recorded as seen, so the next check re-detects the same growth — deferred, never lost.
|
|
95
|
+
* Default: 64
|
|
96
|
+
*/
|
|
97
|
+
growthBlockBudget?: number
|
|
98
|
+
/**
|
|
99
|
+
* How many incomplete growth outcomes ({@link GrowthOutcome} with `complete: false`) a block
|
|
100
|
+
* absorbs before its still-unsatisfied peers are moved to a per-block abandoned set and no longer
|
|
101
|
+
* pushed to. Without this bound a peer that permanently refuses would be re-pushed on every check
|
|
102
|
+
* forever, and its block would re-consume `growthBlockBudget` slots and starve genuinely-new
|
|
103
|
+
* growth. An abandoned peer that leaves the cohort and later rejoins is retried from scratch.
|
|
104
|
+
* Default: 5
|
|
105
|
+
*/
|
|
106
|
+
growthMaxAttempts?: number
|
|
107
|
+
/**
|
|
108
|
+
* Self-arming re-check timer for outstanding growth work (reported-but-unconfirmed peers, or
|
|
109
|
+
* blocks deferred by `growthBlockBudget`). Checks otherwise fire only on libp2p connection
|
|
110
|
+
* events, so a failed push on a then-quiet network would never be retried. Armed at the end of a
|
|
111
|
+
* check only while work is outstanding; fires `maybeRebalance()` so the existing
|
|
112
|
+
* `minRebalanceIntervalMs` throttle still bounds the push rate. `0` disables.
|
|
113
|
+
* Default: `minRebalanceIntervalMs`
|
|
114
|
+
*/
|
|
115
|
+
growthRecheckIntervalMs?: number
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
export interface RebalanceMonitorDeps {
|
|
119
|
+
libp2p: Libp2p
|
|
120
|
+
fret: FretService
|
|
121
|
+
partitionDetector: PartitionDetector
|
|
122
|
+
fretAdapter: ArachnodeFretAdapter
|
|
123
|
+
/**
|
|
124
|
+
* The owned-block tracked set. When provided (e.g. the shared `ownedBlocks` set wired in
|
|
125
|
+
* `libp2p-node-base`), the monitor references this exact `Set` instead of constructing its own,
|
|
126
|
+
* so it stays in lock-step with the `SpreadOnChurnMonitor` that shares it. Omit for standalone
|
|
127
|
+
* construction (unit tests) — a fresh private `Set` preserves all existing behavior. Note: only
|
|
128
|
+
* `trackedBlocks` is shared; `responsibilitySnapshot` stays per-monitor (it is rebalance's own
|
|
129
|
+
* was-responsible memory, not owned-block tracking).
|
|
130
|
+
*/
|
|
131
|
+
trackedBlocks?: Set<string>
|
|
132
|
+
/**
|
|
133
|
+
* The node's key network. When provided, a block's cohort is `findCluster` on its routing key —
|
|
134
|
+
* the rule the writer routes by, the coordinator refuses writes by, and `RepoService` redirects
|
|
135
|
+
* by — so the monitor never calls a peer responsible (and pushes it a replica) that the rest of
|
|
136
|
+
* the node does not, nor releases a block the node is still responsible for. Omit for standalone
|
|
137
|
+
* construction (unit tests): the cohort is then FRET's nearest {@link RebalanceMonitor.getCohortSize}.
|
|
138
|
+
*/
|
|
139
|
+
keyNetwork?: Pick<IKeyNetwork, 'findCluster'>
|
|
140
|
+
/**
|
|
141
|
+
* The node's resolved `clusterSize`. Caps {@link RebalanceMonitor.getCohortSize}: a floor wider
|
|
142
|
+
* than the cohort can never be confirmed (a lost block would never be released), and on the
|
|
143
|
+
* standalone path it would assemble peers outside the configured cohort.
|
|
144
|
+
*/
|
|
145
|
+
clusterSize?: number
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
type RebalanceHandler = (event: RebalanceEvent) => void
|
|
149
|
+
|
|
150
|
+
/** Per-block rebalance/growth state (the `responsibilitySnapshot` entry). */
|
|
151
|
+
interface BlockGrowthState {
|
|
152
|
+
responsible: boolean
|
|
153
|
+
/**
|
|
154
|
+
* The growth arm's seen set: peers CONFIRMED to hold a replica of this block (or satisfied
|
|
155
|
+
* another way — floor met, or nothing local to push). Peers enter ONLY via
|
|
156
|
+
* {@link RebalanceMonitor.recordGrowthOutcome} or an acknowledged commit's holders
|
|
157
|
+
* ({@link RebalanceMonitor.recordCommittedHolders}), never at report time.
|
|
158
|
+
*/
|
|
159
|
+
cohortPeers: Set<string>
|
|
160
|
+
/** Peers reported grown at the last emitting check whose confirmation is still outstanding. */
|
|
161
|
+
pendingPeers: Set<string>
|
|
162
|
+
/** Consecutive incomplete growth outcomes — the give-up counter against `growthMaxAttempts`. */
|
|
163
|
+
growthAttempts: number
|
|
164
|
+
/**
|
|
165
|
+
* Peers given up on after `growthMaxAttempts` incomplete outcomes — excluded from growth reports
|
|
166
|
+
* until they leave the cohort and rejoin (each check intersects this with the current cohort).
|
|
167
|
+
*/
|
|
168
|
+
abandonedPeers: Set<string>
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
const emptyGrowthState = (responsible: boolean): BlockGrowthState => ({
|
|
172
|
+
responsible,
|
|
173
|
+
cohortPeers: new Set<string>(),
|
|
174
|
+
pendingPeers: new Set<string>(),
|
|
175
|
+
growthAttempts: 0,
|
|
176
|
+
abandonedPeers: new Set<string>()
|
|
177
|
+
})
|
|
178
|
+
|
|
179
|
+
const intersect = (remembered: Set<string> | undefined, current: Set<string>): Set<string> =>
|
|
180
|
+
new Set([...(remembered ?? [])].filter(id => current.has(id)))
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* Carry a still-responsible block's growth state into the next check, intersecting both remembered
|
|
184
|
+
* peer sets against the CURRENT cohort: a peer that left drops out of `cohortPeers` (so its return
|
|
185
|
+
* is re-detected — the departure self-heal) and out of `abandonedPeers` (so a rejoin is retried from
|
|
186
|
+
* scratch). Commit evidence gathered since the last check is folded into `cohortPeers` first.
|
|
187
|
+
* `pendingPeers` is always rebuilt from this check's own report, never carried.
|
|
188
|
+
*/
|
|
189
|
+
const carryGrowthState = (
|
|
190
|
+
prior: BlockGrowthState | undefined,
|
|
191
|
+
currentPeers: Set<string>,
|
|
192
|
+
evidence: CommitEvidence | undefined
|
|
193
|
+
): BlockGrowthState => {
|
|
194
|
+
const confirmed = new Set([...(prior?.cohortPeers ?? []), ...(evidence?.holders ?? [])])
|
|
195
|
+
for (const peerId of evidence?.unconfirmed ?? []) confirmed.delete(peerId)
|
|
196
|
+
return {
|
|
197
|
+
responsible: true,
|
|
198
|
+
cohortPeers: intersect(confirmed, currentPeers),
|
|
199
|
+
pendingPeers: new Set<string>(),
|
|
200
|
+
growthAttempts: prior?.growthAttempts ?? 0,
|
|
201
|
+
abandonedPeers: intersect(prior?.abandonedPeers, currentPeers)
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
export class RebalanceMonitor implements Startable {
|
|
206
|
+
private running = false
|
|
207
|
+
private readonly trackedBlocks: Set<string>
|
|
208
|
+
// Per-monitor was-responsible memory (NOT shared, unlike trackedBlocks). When the shared
|
|
209
|
+
// trackedBlocks set is mutated externally — spread's no-local-data self-prune, or the node's
|
|
210
|
+
// responsibility-loss eviction going through untrackBlock — a snapshot entry for a since-removed
|
|
211
|
+
// block may linger here. That is acceptable: performRebalanceCheck only iterates trackedBlocks, so
|
|
212
|
+
// a lingering entry is inert; if the block is later re-fed, its responsibility is simply re-derived.
|
|
213
|
+
//
|
|
214
|
+
// `cohortPeers` is the growth arm's CONFIRMED-co-responsible memory. A MISSING entry is
|
|
215
|
+
// deliberately treated as "prior cohort = empty", so the first check after a topology event
|
|
216
|
+
// reports the whole non-self cohort as grown — that is what heals the founder case (A alone
|
|
217
|
+
// commits; B joins → first check pushes to B) and the restarted-holder case (snapshot memory is
|
|
218
|
+
// process-local, so a restarted holder re-pushes to everyone once). Peers enter the set only
|
|
219
|
+
// through recordGrowthOutcome or a commit's confirmed holders (recordCommittedHolders) — a
|
|
220
|
+
// reported-but-unconfirmed peer stays out, so a failed push is re-detected on the next check
|
|
221
|
+
// instead of being recorded as done.
|
|
222
|
+
private readonly responsibilitySnapshot = new Map<string, BlockGrowthState>()
|
|
223
|
+
// Commit evidence (recordCommittedHolders) waiting for the next check, which consumes each block's
|
|
224
|
+
// entry once its cohort lookup succeeds and drops entries for blocks no longer tracked. Held apart
|
|
225
|
+
// from the snapshot because a commit does not say whether this node is responsible — only a check does.
|
|
226
|
+
private readonly commitEvidence = new Map<string, CommitEvidence>()
|
|
227
|
+
private readonly handlers: RebalanceHandler[] = []
|
|
228
|
+
private debounceTimer: ReturnType<typeof setTimeout> | null = null
|
|
229
|
+
private recheckTimer: ReturnType<typeof setTimeout> | null = null
|
|
230
|
+
private lastRebalanceAt = 0
|
|
231
|
+
private pendingTopologyChange = false
|
|
232
|
+
private topologyChangeTimestamp = 0
|
|
233
|
+
private lastGrowthDeferred = 0
|
|
234
|
+
|
|
235
|
+
private readonly debounceMs: number
|
|
236
|
+
private readonly minRebalanceIntervalMs: number
|
|
237
|
+
private readonly suppressDuringPartition: boolean
|
|
238
|
+
private readonly growthBlockBudget: number
|
|
239
|
+
private readonly growthMaxAttempts: number
|
|
240
|
+
private readonly growthRecheckIntervalMs: number
|
|
241
|
+
|
|
242
|
+
private readonly onConnectionOpen: () => void
|
|
243
|
+
private readonly onConnectionClose: () => void
|
|
244
|
+
|
|
245
|
+
constructor(
|
|
246
|
+
private readonly deps: RebalanceMonitorDeps,
|
|
247
|
+
config: RebalanceMonitorConfig = {}
|
|
248
|
+
) {
|
|
249
|
+
// Share the injected owned-block set when present (so spread + rebalance never drift);
|
|
250
|
+
// otherwise own a private set (standalone construction / unit tests). Only trackedBlocks is
|
|
251
|
+
// shared — responsibilitySnapshot stays per-monitor.
|
|
252
|
+
this.trackedBlocks = deps.trackedBlocks ?? new Set<string>()
|
|
253
|
+
this.debounceMs = config.debounceMs ?? 5000
|
|
254
|
+
this.minRebalanceIntervalMs = config.minRebalanceIntervalMs ?? 60000
|
|
255
|
+
this.suppressDuringPartition = config.suppressDuringPartition ?? true
|
|
256
|
+
this.growthBlockBudget = config.growthBlockBudget ?? 64
|
|
257
|
+
this.growthMaxAttempts = config.growthMaxAttempts ?? 5
|
|
258
|
+
this.growthRecheckIntervalMs = config.growthRecheckIntervalMs ?? this.minRebalanceIntervalMs
|
|
259
|
+
|
|
260
|
+
this.onConnectionOpen = () => this.handleTopologyChange()
|
|
261
|
+
this.onConnectionClose = () => this.handleTopologyChange()
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
async start(): Promise<void> {
|
|
265
|
+
if (this.running) return
|
|
266
|
+
this.running = true
|
|
267
|
+
|
|
268
|
+
this.deps.libp2p.addEventListener('connection:open', this.onConnectionOpen)
|
|
269
|
+
this.deps.libp2p.addEventListener('connection:close', this.onConnectionClose)
|
|
270
|
+
|
|
271
|
+
log('started, tracking %d blocks', this.trackedBlocks.size)
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
async stop(): Promise<void> {
|
|
275
|
+
if (!this.running) return
|
|
276
|
+
this.running = false
|
|
277
|
+
|
|
278
|
+
this.deps.libp2p.removeEventListener('connection:open', this.onConnectionOpen)
|
|
279
|
+
this.deps.libp2p.removeEventListener('connection:close', this.onConnectionClose)
|
|
280
|
+
|
|
281
|
+
if (this.debounceTimer) {
|
|
282
|
+
clearTimeout(this.debounceTimer)
|
|
283
|
+
this.debounceTimer = null
|
|
284
|
+
}
|
|
285
|
+
if (this.recheckTimer) {
|
|
286
|
+
clearTimeout(this.recheckTimer)
|
|
287
|
+
this.recheckTimer = null
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
this.pendingTopologyChange = false
|
|
291
|
+
this.commitEvidence.clear()
|
|
292
|
+
log('stopped')
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
onRebalance(handler: RebalanceHandler): void {
|
|
296
|
+
this.handlers.push(handler)
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
trackBlock(blockId: string): void {
|
|
300
|
+
this.trackedBlocks.add(blockId)
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
untrackBlock(blockId: string): void {
|
|
304
|
+
this.trackedBlocks.delete(blockId)
|
|
305
|
+
this.responsibilitySnapshot.delete(blockId)
|
|
306
|
+
this.commitEvidence.delete(blockId)
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
getTrackedBlockCount(): number {
|
|
310
|
+
return this.trackedBlocks.size
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
async checkNow(): Promise<RebalanceEvent | null> {
|
|
314
|
+
return this.performRebalanceCheck(Date.now())
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/**
|
|
318
|
+
* Feedback from the growth reaction for one block reported `grown`. `satisfiedPeers` enter the
|
|
319
|
+
* block's seen set; an incomplete outcome counts an attempt against `growthMaxAttempts`, and on
|
|
320
|
+
* reaching the bound the block's still-unsatisfied reported peers are abandoned (no longer
|
|
321
|
+
* pushed to until they leave the cohort and rejoin). Never called for a block the reaction had
|
|
322
|
+
* no information about (a confirm deduped against one already in flight) — a missing outcome
|
|
323
|
+
* leaves the block's state untouched so the next check retries.
|
|
324
|
+
*/
|
|
325
|
+
recordGrowthOutcome(blockId: string, outcome: GrowthOutcome): void {
|
|
326
|
+
const state = this.responsibilitySnapshot.get(blockId)
|
|
327
|
+
// No state (untracked since) or responsibility lost since the report: the growth state was
|
|
328
|
+
// cleared, and recording into it would survive the clear and suppress the regain re-push.
|
|
329
|
+
if (!state || !state.responsible) return
|
|
330
|
+
|
|
331
|
+
for (const peerId of outcome.satisfiedPeers) {
|
|
332
|
+
state.cohortPeers.add(peerId)
|
|
333
|
+
state.pendingPeers.delete(peerId)
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
if (outcome.complete) {
|
|
337
|
+
state.growthAttempts = 0
|
|
338
|
+
state.pendingPeers.clear()
|
|
339
|
+
} else {
|
|
340
|
+
// NOTE: growthMaxAttempts is a floor on the retry count, not an exact one. `pendingPeers` is
|
|
341
|
+
// rebuilt from each check's own report, so two cases blunt the bound: a check that defers
|
|
342
|
+
// this block on growthBlockBudget clears pendingPeers, and a give-up landing right then
|
|
343
|
+
// abandons nobody while still resetting the counter; and two checks racing (the second
|
|
344
|
+
// re-reporting the same peer after the first's confirm left `inFlight` but before its outcome
|
|
345
|
+
// landed) double-count one attempt. Both are rare, both only change how many pushes a doomed
|
|
346
|
+
// peer absorbs. If a deployment ever tracks far more blocks than growthBlockBudget, deferral
|
|
347
|
+
// stops being rare — carry the report's peer list on GrowthOutcome and match it against
|
|
348
|
+
// pendingPeers instead of trusting the latest report.
|
|
349
|
+
state.growthAttempts++
|
|
350
|
+
if (state.growthAttempts >= this.growthMaxAttempts) {
|
|
351
|
+
for (const peerId of state.pendingPeers) {
|
|
352
|
+
state.abandonedPeers.add(peerId)
|
|
353
|
+
}
|
|
354
|
+
log('growth give-up: block=%s abandoning %d unsatisfied peer(s) after %d attempts',
|
|
355
|
+
blockId, state.pendingPeers.size, state.growthAttempts)
|
|
356
|
+
state.pendingPeers.clear()
|
|
357
|
+
state.growthAttempts = 0
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
this.updateRecheckTimer()
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
/**
|
|
365
|
+
* Evidence from an acknowledged commit that this node's storage holds: `holders` are recorded as
|
|
366
|
+
* confirmed co-holders of each block at the next check, and the block is not reported `gained`
|
|
367
|
+
* then (this node already holds it, so there is nothing to pull). Without this, every freshly
|
|
368
|
+
* committed block has no growth memory, so the next check reports the whole cohort grown and
|
|
369
|
+
* pushes the block back to the members that stored it as part of the commit — and reports it
|
|
370
|
+
* gained, pulling it back from them.
|
|
371
|
+
*
|
|
372
|
+
* Later evidence for the same block overrides earlier evidence about the same peer, so the
|
|
373
|
+
* coordinator's durability-checked answer corrects the member-side signer list recorded moments
|
|
374
|
+
* before it on the coordinating node. Ignored while stopped: nothing would consume it.
|
|
375
|
+
*
|
|
376
|
+
* Does not affect the founder case: a commit no other peer confirmed names no holders, so the
|
|
377
|
+
* cohort is still reported grown once peers appear.
|
|
378
|
+
*/
|
|
379
|
+
recordCommittedHolders(committed: CommittedHolders): void {
|
|
380
|
+
if (!this.running) return
|
|
381
|
+
for (const blockId of committed.blockIds) {
|
|
382
|
+
let evidence = this.commitEvidence.get(blockId)
|
|
383
|
+
if (!evidence) {
|
|
384
|
+
evidence = { holders: new Set<string>(), unconfirmed: new Set<string>() }
|
|
385
|
+
this.commitEvidence.set(blockId, evidence)
|
|
386
|
+
}
|
|
387
|
+
for (const peerId of committed.holders) {
|
|
388
|
+
evidence.holders.add(peerId)
|
|
389
|
+
evidence.unconfirmed.delete(peerId)
|
|
390
|
+
}
|
|
391
|
+
for (const peerId of committed.unconfirmed ?? []) {
|
|
392
|
+
evidence.unconfirmed.add(peerId)
|
|
393
|
+
evidence.holders.delete(peerId)
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
/**
|
|
399
|
+
* Growth-arm observability: how many tracked blocks still await confirmation on reported peers,
|
|
400
|
+
* how many (block, peer) pairs have been given up on, and whether the re-check timer is armed.
|
|
401
|
+
*/
|
|
402
|
+
getGrowthDiagnostics(): { blocksAwaitingConfirmation: number; abandonedPairs: number; recheckArmed: boolean } {
|
|
403
|
+
let blocksAwaitingConfirmation = 0
|
|
404
|
+
let abandonedPairs = 0
|
|
405
|
+
for (const [blockId, state] of this.responsibilitySnapshot) {
|
|
406
|
+
if (!state.responsible || !this.trackedBlocks.has(blockId)) continue
|
|
407
|
+
if (state.pendingPeers.size > 0) blocksAwaitingConfirmation++
|
|
408
|
+
abandonedPairs += state.abandonedPeers.size
|
|
409
|
+
}
|
|
410
|
+
return { blocksAwaitingConfirmation, abandonedPairs, recheckArmed: this.recheckTimer !== null }
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
/** Growth work is outstanding while any reported peer is unconfirmed or blocks were budget-deferred. */
|
|
414
|
+
private hasOutstandingGrowthWork(): boolean {
|
|
415
|
+
if (this.lastGrowthDeferred > 0) return true
|
|
416
|
+
for (const [blockId, state] of this.responsibilitySnapshot) {
|
|
417
|
+
if (state.responsible && state.pendingPeers.size > 0 && this.trackedBlocks.has(blockId)) {
|
|
418
|
+
return true
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
return false
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
/**
|
|
425
|
+
* Arm the growth re-check timer while work is outstanding; disarm it when there is none. The
|
|
426
|
+
* timer fires maybeRebalance(), so minRebalanceIntervalMs still bounds the push rate, and it
|
|
427
|
+
* re-arms itself after firing for as long as work remains. unref'd so it never holds the
|
|
428
|
+
* process open; stop() clears it.
|
|
429
|
+
*/
|
|
430
|
+
private updateRecheckTimer(): void {
|
|
431
|
+
if (this.growthRecheckIntervalMs <= 0) return
|
|
432
|
+
|
|
433
|
+
if (!this.running || !this.hasOutstandingGrowthWork()) {
|
|
434
|
+
if (this.recheckTimer) {
|
|
435
|
+
clearTimeout(this.recheckTimer)
|
|
436
|
+
this.recheckTimer = null
|
|
437
|
+
}
|
|
438
|
+
return
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
if (this.recheckTimer) return // already armed
|
|
442
|
+
|
|
443
|
+
this.recheckTimer = setTimeout(() => {
|
|
444
|
+
this.recheckTimer = null
|
|
445
|
+
void this.maybeRebalance()
|
|
446
|
+
.catch(err => { log('recheck error: %O', err) })
|
|
447
|
+
.finally(() => this.updateRecheckTimer())
|
|
448
|
+
}, this.growthRecheckIntervalMs)
|
|
449
|
+
;(this.recheckTimer as unknown as { unref?: () => void }).unref?.()
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
private handleTopologyChange(): void {
|
|
453
|
+
if (!this.running) return
|
|
454
|
+
|
|
455
|
+
if (!this.pendingTopologyChange) {
|
|
456
|
+
this.topologyChangeTimestamp = Date.now()
|
|
457
|
+
}
|
|
458
|
+
this.pendingTopologyChange = true
|
|
459
|
+
|
|
460
|
+
if (this.debounceTimer) {
|
|
461
|
+
clearTimeout(this.debounceTimer)
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
// NOTE: the check samples FRET's cohort once, debounceMs after the last connection event. A joiner
|
|
465
|
+
// FRET has not admitted by then is not reported grown, and nothing re-checks until the next
|
|
466
|
+
// connection event outside minRebalanceIntervalMs — so a machine that wrote alone would not push its
|
|
467
|
+
// blocks to its new backup. Over loopback FRET admits well inside the default (the backup phase of
|
|
468
|
+
// `small-deployment-lifecycle.integration.spec.ts` lands in ~10s); if a slower link, such as a phone
|
|
469
|
+
// pair over a relay, ever shows a joiner that never receives the founder's blocks, re-check when FRET
|
|
470
|
+
// reports the peer instead of on this timer alone.
|
|
471
|
+
this.debounceTimer = setTimeout(() => {
|
|
472
|
+
this.debounceTimer = null
|
|
473
|
+
this.pendingTopologyChange = false
|
|
474
|
+
this.maybeRebalance()
|
|
475
|
+
}, this.debounceMs)
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
private async maybeRebalance(): Promise<void> {
|
|
479
|
+
if (!this.running) return
|
|
480
|
+
|
|
481
|
+
const now = Date.now()
|
|
482
|
+
const elapsed = now - this.lastRebalanceAt
|
|
483
|
+
if (elapsed < this.minRebalanceIntervalMs) {
|
|
484
|
+
log('throttled, %dms since last rebalance', elapsed)
|
|
485
|
+
return
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
const event = await this.performRebalanceCheck(this.topologyChangeTimestamp || now)
|
|
489
|
+
if (event) {
|
|
490
|
+
this.emitEvent(event)
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
private async performRebalanceCheck(triggeredAt: number): Promise<RebalanceEvent | null> {
|
|
495
|
+
if (this.suppressDuringPartition && this.deps.partitionDetector.detectPartition()) {
|
|
496
|
+
log('partition detected, suppressing rebalance')
|
|
497
|
+
return null
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
if (this.trackedBlocks.size === 0) {
|
|
501
|
+
this.lastRebalanceAt = Date.now()
|
|
502
|
+
// Nothing left to grow, so any prior deferral is moot — drop it and let the re-check timer
|
|
503
|
+
// disarm, rather than re-arming forever against blocks that were untracked out from under it.
|
|
504
|
+
this.lastGrowthDeferred = 0
|
|
505
|
+
this.commitEvidence.clear()
|
|
506
|
+
this.updateRecheckTimer()
|
|
507
|
+
return null
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
const selfId = this.deps.libp2p.peerId.toString()
|
|
511
|
+
const gained: string[] = []
|
|
512
|
+
const lost: string[] = []
|
|
513
|
+
const newOwners = new Map<string, string[]>()
|
|
514
|
+
const grown = new Map<string, string[]>()
|
|
515
|
+
let growthDeferred = 0
|
|
516
|
+
let lookupFailed = 0
|
|
517
|
+
const growthCandidates: Array<{ blockId: string; newPeers: string[]; state: BlockGrowthState }> = []
|
|
518
|
+
|
|
519
|
+
for (const blockId of this.trackedBlocks) {
|
|
520
|
+
let cohort: string[]
|
|
521
|
+
try {
|
|
522
|
+
cohort = await this.cohortFor(blockId)
|
|
523
|
+
} catch (err) {
|
|
524
|
+
// Responsibility is undetermined: leave this block's state untouched (never guess lost or
|
|
525
|
+
// gained off a failed lookup) and count it as outstanding so the re-check timer retries it.
|
|
526
|
+
log('cohort lookup failed: block=%s %o', blockId, err)
|
|
527
|
+
lookupFailed++
|
|
528
|
+
continue
|
|
529
|
+
}
|
|
530
|
+
const isResponsible = cohort.includes(selfId)
|
|
531
|
+
const prior = this.responsibilitySnapshot.get(blockId)
|
|
532
|
+
const wasResponsible = prior?.responsible ?? false
|
|
533
|
+
// Consumed only once the lookup succeeded, so a failed lookup leaves it for the next check.
|
|
534
|
+
// NOTE: a block enters trackedBlocks at storage apply, but its evidence arrives only after the
|
|
535
|
+
// durable verdict (member) or cohort acknowledgement (coordinator); a check landing in that
|
|
536
|
+
// window still reports it gained + grown, the pre-evidence behaviour. If relay traces show
|
|
537
|
+
// post-commit transfers surviving, skip a block's first check while a commit touching it is in flight.
|
|
538
|
+
const evidence = this.commitEvidence.get(blockId)
|
|
539
|
+
this.commitEvidence.delete(blockId)
|
|
540
|
+
|
|
541
|
+
if (isResponsible && !wasResponsible && !evidence) {
|
|
542
|
+
gained.push(blockId)
|
|
543
|
+
} else if (!isResponsible && wasResponsible) {
|
|
544
|
+
lost.push(blockId)
|
|
545
|
+
// The cohort members are the new owners
|
|
546
|
+
newOwners.set(blockId, cohort.filter(id => id !== selfId))
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
// Growth arm: while this node STAYS responsible, any cohort peer not yet CONFIRMED to hold
|
|
550
|
+
// the block (and not abandoned) gets it pushed (up to the floor). Runs on every responsible
|
|
551
|
+
// check — NOT gated on wasResponsible — so a first observation (no snapshot entry) treats
|
|
552
|
+
// the whole non-self cohort as new; see the responsibilitySnapshot comment for why that is
|
|
553
|
+
// load-bearing. Not responsible ⇒ arm skipped, so `lost` ∩ `grown` is impossible.
|
|
554
|
+
//
|
|
555
|
+
// Reporting a peer does NOT record it as seen — only recordGrowthOutcome does — so a push
|
|
556
|
+
// that fails (dial timeout, receiver refused to persist, partition mid-reaction, reaction
|
|
557
|
+
// threw) leaves the peer un-seen and the next check re-detects it.
|
|
558
|
+
let state: BlockGrowthState
|
|
559
|
+
if (isResponsible) {
|
|
560
|
+
const currentSet = new Set(cohort.filter(id => id !== selfId))
|
|
561
|
+
state = carryGrowthState(prior, currentSet, evidence)
|
|
562
|
+
const newPeers = [...currentSet].filter(id => !state.cohortPeers.has(id) && !state.abandonedPeers.has(id))
|
|
563
|
+
if (newPeers.length > 0) {
|
|
564
|
+
growthCandidates.push({ blockId, newPeers, state })
|
|
565
|
+
} else {
|
|
566
|
+
// Nothing owed for this block. growthAttempts counts CONSECUTIVE incomplete outcomes
|
|
567
|
+
// against outstanding growth, so it must not carry across a quiet stretch — otherwise a
|
|
568
|
+
// block that failed a few times, then had that peer leave, would spend the leftovers on
|
|
569
|
+
// whichever peer joins next and abandon it early.
|
|
570
|
+
state.growthAttempts = 0
|
|
571
|
+
}
|
|
572
|
+
} else {
|
|
573
|
+
// Not responsible: clear ALL growth state (seen set, attempts, abandoned peers), so a
|
|
574
|
+
// later regain re-pushes to the whole cohort (benign when the local data is gone — the
|
|
575
|
+
// push finds nothing and no-ops).
|
|
576
|
+
state = emptyGrowthState(false)
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
this.responsibilitySnapshot.set(blockId, state)
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
// Evidence for a block that is not tracked has no consumer: a commit this node does not hold
|
|
583
|
+
// as an owned block, or one untracked since.
|
|
584
|
+
for (const blockId of [...this.commitEvidence.keys()]) {
|
|
585
|
+
if (!this.trackedBlocks.has(blockId)) this.commitEvidence.delete(blockId)
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
// Fill the growth budget in two passes: fresh growth (no failed attempts yet) first, retrying
|
|
589
|
+
// blocks with what remains — otherwise a stuck retry set at the front of the tracked-block
|
|
590
|
+
// insertion order would starve peers that just joined.
|
|
591
|
+
for (const candidate of [
|
|
592
|
+
...growthCandidates.filter(c => c.state.growthAttempts === 0),
|
|
593
|
+
...growthCandidates.filter(c => c.state.growthAttempts > 0)
|
|
594
|
+
]) {
|
|
595
|
+
if (grown.size < this.growthBlockBudget) {
|
|
596
|
+
grown.set(candidate.blockId, candidate.newPeers)
|
|
597
|
+
candidate.state.pendingPeers = new Set(candidate.newPeers)
|
|
598
|
+
} else {
|
|
599
|
+
// Budget-dropped: the seen set was not touched, so the next check re-detects the same
|
|
600
|
+
// growth — a deferral, not a loss.
|
|
601
|
+
growthDeferred++
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
this.lastGrowthDeferred = growthDeferred + lookupFailed
|
|
605
|
+
|
|
606
|
+
this.lastRebalanceAt = Date.now()
|
|
607
|
+
|
|
608
|
+
if (growthDeferred > 0) {
|
|
609
|
+
// Deferred blocks drain one budget-full per check. Checks fire on libp2p connection events
|
|
610
|
+
// AND — while growth work is outstanding — on the growthRecheckIntervalMs timer armed
|
|
611
|
+
// below, so a backlog drains even on a quiet network.
|
|
612
|
+
log('growth budget reached: %d blocks deferred to the next check (budget=%d)',
|
|
613
|
+
growthDeferred, this.growthBlockBudget)
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
this.updateRecheckTimer()
|
|
617
|
+
|
|
618
|
+
if (gained.length === 0 && lost.length === 0 && grown.size === 0) {
|
|
619
|
+
return null
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
log('rebalance check: gained=%d lost=%d grown=%d', gained.length, lost.length, grown.size)
|
|
623
|
+
|
|
624
|
+
return { gained, lost, newOwners, grown, floor: this.getCohortSize(), triggeredAt }
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
/**
|
|
628
|
+
* The peers responsible for a block, nearest first. With a key network wired this is its
|
|
629
|
+
* `findCluster` cohort (see {@link RebalanceMonitorDeps.keyNetwork}); otherwise FRET's nearest
|
|
630
|
+
* {@link getCohortSize} ring members.
|
|
631
|
+
*
|
|
632
|
+
* NOTE: one `findCluster` (peer-store reads for each cohort member) per tracked block per pass, where
|
|
633
|
+
* FRET's cohort was one synchronous call; fine under the pass throttle; if passes on nodes holding many
|
|
634
|
+
* blocks ever show up as slow, share the lookup with the coordinator's responsibility cache.
|
|
635
|
+
*/
|
|
636
|
+
private async cohortFor(blockId: string): Promise<string[]> {
|
|
637
|
+
const key = routingKeyForBlock(blockId)
|
|
638
|
+
if (this.deps.keyNetwork) {
|
|
639
|
+
return Object.keys(await this.deps.keyNetwork.findCluster(key))
|
|
640
|
+
}
|
|
641
|
+
return this.deps.fret.assembleCohort(await hashKey(key), this.getCohortSize())
|
|
642
|
+
}
|
|
643
|
+
|
|
644
|
+
/**
|
|
645
|
+
* The replication floor `N`. Public so the ring-shift handoff and the rebalance reaction can gate
|
|
646
|
+
* release on confirming replication to this many holders (`docs/arachnode-ring-handoff.md`
|
|
647
|
+
* § Replication floor). Derives from FRET's network-size estimate: `clamp(ceil(sqrt(n_est)), 1, 3)`,
|
|
648
|
+
* defaulting to 3 when no confident estimate exists — and never above the configured `clusterSize`,
|
|
649
|
+
* since no block has more responsible holders than that to confirm on.
|
|
650
|
+
*/
|
|
651
|
+
getCohortSize(): number {
|
|
652
|
+
const diag: any = (this.deps.fret as any).getDiagnostics?.()
|
|
653
|
+
const estimate = diag?.estimate ?? diag?.n
|
|
654
|
+
const size = typeof estimate === 'number' && Number.isFinite(estimate) && estimate > 0
|
|
655
|
+
? Math.max(1, Math.min(3, Math.ceil(Math.sqrt(estimate))))
|
|
656
|
+
: 3
|
|
657
|
+
const cap = this.deps.clusterSize
|
|
658
|
+
return cap != null && cap >= 1 ? Math.min(size, cap) : size
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
private emitEvent(event: RebalanceEvent): void {
|
|
662
|
+
for (const handler of this.handlers) {
|
|
663
|
+
try {
|
|
664
|
+
handler(event)
|
|
665
|
+
} catch (err) {
|
|
666
|
+
log('handler error: %O', err)
|
|
667
|
+
}
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
/**
|
|
672
|
+
* Update ArachnodeInfo status through the fret adapter.
|
|
673
|
+
*/
|
|
674
|
+
setStatus(status: ArachnodeInfo['status']): void {
|
|
675
|
+
this.deps.fretAdapter.setStatus(status)
|
|
676
|
+
}
|
|
677
|
+
}
|