@serfab/cadre-core 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +255 -255
- package/dist/announce-addrs.d.ts +49 -0
- package/dist/announce-addrs.js +78 -0
- package/dist/announce-addrs.js.map +1 -0
- package/dist/cadre-node.d.ts +484 -58
- package/dist/cadre-node.js +953 -182
- package/dist/cadre-node.js.map +1 -1
- package/dist/control-cohort.d.ts +26 -0
- package/dist/control-cohort.js +26 -0
- package/dist/control-cohort.js.map +1 -1
- package/dist/control-database.d.ts +222 -125
- package/dist/control-database.js +356 -266
- package/dist/control-database.js.map +1 -1
- package/dist/control-formation-recorder.d.ts +8 -7
- package/dist/control-formation-recorder.js +8 -7
- package/dist/control-formation-recorder.js.map +1 -1
- package/dist/control-read-retry.d.ts +103 -0
- package/dist/control-read-retry.js +145 -0
- package/dist/control-read-retry.js.map +1 -0
- package/dist/control-retry.d.ts +101 -0
- package/dist/control-retry.js +108 -0
- package/dist/control-retry.js.map +1 -0
- package/dist/control-schema.d.ts +1 -1
- package/dist/control-schema.js +805 -774
- package/dist/control-schema.js.map +1 -1
- package/dist/control-write-retry.d.ts +65 -31
- package/dist/control-write-retry.js +33 -103
- package/dist/control-write-retry.js.map +1 -1
- package/dist/delegate-admission.d.ts +17 -15
- package/dist/delegate-admission.js +17 -15
- package/dist/delegate-admission.js.map +1 -1
- package/dist/index.d.ts +7 -5
- package/dist/index.js +18 -9
- package/dist/index.js.map +1 -1
- package/dist/membership-connection-gater.d.ts +211 -25
- package/dist/membership-connection-gater.js +309 -29
- package/dist/membership-connection-gater.js.map +1 -1
- package/dist/peer-addr-book.d.ts +92 -0
- package/dist/peer-addr-book.js +220 -0
- package/dist/peer-addr-book.js.map +1 -0
- package/dist/peer-record.d.ts +26 -0
- package/dist/peer-record.js +32 -0
- package/dist/peer-record.js.map +1 -1
- package/dist/relay-addrs.d.ts +76 -26
- package/dist/relay-addrs.js +115 -27
- package/dist/relay-addrs.js.map +1 -1
- package/dist/relay-reservation.d.ts +20 -15
- package/dist/relay-reservation.js +18 -14
- package/dist/relay-reservation.js.map +1 -1
- package/dist/seed-bootstrap.d.ts +7 -0
- package/dist/seed-bootstrap.js +15 -13
- package/dist/seed-bootstrap.js.map +1 -1
- package/dist/strand-addr-protocol.js +16 -0
- package/dist/strand-addr-protocol.js.map +1 -1
- package/dist/strand-backfill.js +13 -4
- package/dist/strand-backfill.js.map +1 -1
- package/dist/strand-cohort.d.ts +4 -37
- package/dist/strand-cohort.js +4 -15
- package/dist/strand-cohort.js.map +1 -1
- package/dist/strand-database.d.ts +11 -16
- package/dist/strand-database.js +19 -7
- package/dist/strand-database.js.map +1 -1
- package/dist/strand-formation-manager.d.ts +18 -17
- package/dist/strand-formation-manager.js +25 -27
- package/dist/strand-formation-manager.js.map +1 -1
- package/dist/strand-formation-protocol.d.ts +7 -7
- package/dist/strand-formation-protocol.js +8 -8
- package/dist/strand-formation-protocol.js.map +1 -1
- package/dist/strand-instance-manager.d.ts +53 -21
- package/dist/strand-instance-manager.js +128 -37
- package/dist/strand-instance-manager.js.map +1 -1
- package/dist/strand-membership-writer.d.ts +113 -19
- package/dist/strand-membership-writer.js +219 -51
- package/dist/strand-membership-writer.js.map +1 -1
- package/dist/strand-solicitation.d.ts +3 -3
- package/dist/strand-solicitation.js +4 -5
- package/dist/strand-solicitation.js.map +1 -1
- package/dist/strand-wake-protocol.d.ts +47 -2
- package/dist/strand-wake-protocol.js +76 -6
- package/dist/strand-wake-protocol.js.map +1 -1
- package/dist/types.d.ts +107 -54
- package/dist/types.js.map +1 -1
- package/package.json +8 -7
- package/src/announce-addrs.ts +93 -0
- package/src/arachnode-stub.ts +159 -159
- package/src/bootstrap-peer-store-file.ts +44 -44
- package/src/bootstrap-peer-store.ts +172 -172
- package/src/cadre-node.ts +1026 -194
- package/src/canonical-json.ts +30 -30
- package/src/control-authorization.ts +114 -114
- package/src/control-cohort.ts +130 -103
- package/src/control-database.ts +376 -298
- package/src/control-formation-recorder.ts +305 -304
- package/src/control-read-retry.ts +167 -0
- package/src/control-retry.ts +190 -0
- package/src/control-schema.ts +817 -786
- package/src/control-stream.ts +205 -205
- package/src/control-write-retry.ts +383 -473
- package/src/delegate-admission.ts +304 -302
- package/src/device-token.ts +94 -94
- package/src/diagnostics/connection-path.ts +277 -277
- package/src/diagnostics/webrtc-turn-tracker.ts +222 -222
- package/src/ed25519-key.ts +134 -134
- package/src/enrollment.ts +199 -199
- package/src/file-durable-slot.ts +61 -61
- package/src/formation-approval.ts +604 -604
- package/src/fs-atomic.ts +107 -107
- package/src/hibernation-manager.ts +437 -437
- package/src/identity-key.ts +109 -109
- package/src/index.ts +435 -406
- package/src/key-store-file.ts +132 -132
- package/src/key-store.ts +99 -99
- package/src/membership-connection-gater.ts +368 -39
- package/src/node-local-snapshot.ts +260 -260
- package/src/peer-addr-book.ts +250 -0
- package/src/peer-authorization.ts +259 -259
- package/src/peer-record.ts +197 -164
- package/src/push-credentials.ts +84 -84
- package/src/push-node.ts +64 -64
- package/src/push-notifier-apns.ts +247 -247
- package/src/push-notifier-fcm.ts +220 -220
- package/src/push-notifier-shared.ts +28 -28
- package/src/push-notifier.ts +52 -52
- package/src/relay-addrs.ts +134 -27
- package/src/relay-reservation.ts +22 -17
- package/src/seed-bootstrap.ts +15 -15
- package/src/seed-trust-policy.ts +140 -140
- package/src/strand-addr-protocol.ts +458 -441
- package/src/strand-backfill.ts +28 -5
- package/src/strand-cohort.ts +38 -77
- package/src/strand-database.ts +202 -204
- package/src/strand-formation-manager.ts +26 -28
- package/src/strand-formation-protocol.ts +692 -692
- package/src/strand-instance-manager.ts +636 -553
- package/src/strand-member-key.ts +39 -39
- package/src/strand-member-registry.ts +182 -182
- package/src/strand-membership-writer.ts +1627 -1430
- package/src/strand-solicitation.ts +471 -472
- package/src/strand-transport-key.ts +58 -58
- package/src/strand-wake-payload.ts +32 -32
- package/src/strand-wake-protocol.ts +400 -314
- package/src/strand-watcher.ts +314 -314
- package/src/trusted-owner-store-file.ts +41 -41
- package/src/trusted-owner-store.ts +171 -171
- package/src/types.ts +1354 -1303
|
@@ -1,553 +1,636 @@
|
|
|
1
|
-
import debug from 'debug';
|
|
2
|
-
import type { PrivateKey } from '@libp2p/interface';
|
|
3
|
-
import { createLibp2pNode, type IRawStorage } from '@optimystic/db-p2p';
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
Libp2pNodeWithRepo
|
|
19
|
-
} from './types.js';
|
|
20
|
-
import { resolveStrandClusterSize, STRAND_CLUSTER_POLICY } from './types.js';
|
|
21
|
-
import { resolveListenAddrs } from './relay-addrs.js';
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
const
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
*
|
|
56
|
-
* {@link
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
*
|
|
61
|
-
*
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
*
|
|
66
|
-
*
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
*
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
*
|
|
94
|
-
*
|
|
95
|
-
*
|
|
96
|
-
*
|
|
97
|
-
*
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
//
|
|
116
|
-
const
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
*
|
|
129
|
-
*
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
return
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
/**
|
|
146
|
-
* Manages individual strand instances - creates and destroys isolated libp2p nodes
|
|
147
|
-
* for each strand the cadre participates in.
|
|
148
|
-
*/
|
|
149
|
-
export class StrandInstanceManager {
|
|
150
|
-
private instances: Map<string, StrandInstance> = new Map();
|
|
151
|
-
/**
|
|
152
|
-
* Retained launch config per strand, captured in `startStrand` and cleared in
|
|
153
|
-
* `stopStrand`. `resumeStrand` reuses it to rebuild a quiesced strand's runtime
|
|
154
|
-
* without the caller re-threading storage/network/profile/key/sApp config.
|
|
155
|
-
*/
|
|
156
|
-
private launchConfigs: Map<string, StartStrandConfig> = new Map();
|
|
157
|
-
/**
|
|
158
|
-
* The per-strand peer-join block catch-up, keyed by strand id. Private (not on
|
|
159
|
-
* the public {@link StrandInstance}) because it is runtime plumbing with the
|
|
160
|
-
* same lifetime as the strand's libp2p node: created in `buildStrandRuntime`,
|
|
161
|
-
* stopped and dropped in `releaseRuntime` — so quiesce → resume rebuilds it
|
|
162
|
-
* with a fresh caught-up-peer memo, which is intended (a resumed node may have
|
|
163
|
-
* missed writes).
|
|
164
|
-
*/
|
|
165
|
-
private backfills: Map<string, StrandBackfill> = new Map();
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
*
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
*
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
*
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
strandId
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
}
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
}
|
|
391
|
-
}
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
instance.
|
|
395
|
-
|
|
396
|
-
//
|
|
397
|
-
//
|
|
398
|
-
//
|
|
399
|
-
//
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
*
|
|
464
|
-
*
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
this.stopping
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
1
|
+
import debug from 'debug';
|
|
2
|
+
import type { PrivateKey } from '@libp2p/interface';
|
|
3
|
+
import { createLibp2pNode, type IRawStorage } from '@optimystic/db-p2p';
|
|
4
|
+
import { wrapStorageWithCache, disposeStorageCache } from '@serfab/quereus-plugin-sereus';
|
|
5
|
+
import { StrandDatabase } from './strand-database.js';
|
|
6
|
+
import { StrandBackfill, type StrandBackfillConfig } from './strand-backfill.js';
|
|
7
|
+
import { assertSchemaSignature } from './schema-verification.js';
|
|
8
|
+
import type {
|
|
9
|
+
StrandInstance,
|
|
10
|
+
StrandRow,
|
|
11
|
+
StorageConfig,
|
|
12
|
+
NetworkConfig,
|
|
13
|
+
LatencyHint,
|
|
14
|
+
NodeProfile,
|
|
15
|
+
SAppConfig,
|
|
16
|
+
SAppInfo,
|
|
17
|
+
RawStorageProvider,
|
|
18
|
+
Libp2pNodeWithRepo
|
|
19
|
+
} from './types.js';
|
|
20
|
+
import { resolveStrandClusterSize, STRAND_CLUSTER_POLICY } from './types.js';
|
|
21
|
+
import { resolveListenAddrs } from './relay-addrs.js';
|
|
22
|
+
import { resolveAnnounceAddrs } from './announce-addrs.js';
|
|
23
|
+
|
|
24
|
+
const log = debug('sereus:cadre:strand-manager');
|
|
25
|
+
const timing = debug('sereus:cadre:timing');
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Configuration for starting a strand instance
|
|
29
|
+
*/
|
|
30
|
+
export interface StartStrandConfig {
|
|
31
|
+
strandRow: StrandRow;
|
|
32
|
+
/** sApp configuration provided by the hosting application */
|
|
33
|
+
sAppConfig: SAppConfig;
|
|
34
|
+
storage?: StorageConfig;
|
|
35
|
+
network?: NetworkConfig;
|
|
36
|
+
profile: NodeProfile;
|
|
37
|
+
defaultLatencyHint: LatencyHint;
|
|
38
|
+
privateKey?: PrivateKey;
|
|
39
|
+
/** Cohort-derived discovery seed (multiaddr strings). Defaults to [] when omitted. */
|
|
40
|
+
bootstrapNodes?: string[];
|
|
41
|
+
/**
|
|
42
|
+
* Require a valid author signature on the sApp schema before bring-up.
|
|
43
|
+
* Defaults to true (fail closed) when omitted; set false only for dev/test
|
|
44
|
+
* with unsigned demo schemas. Mirrors {@link CadreNodeConfig.requireSignedSchemas}.
|
|
45
|
+
*/
|
|
46
|
+
requireSignedSchemas?: boolean;
|
|
47
|
+
/**
|
|
48
|
+
* Whether this node founds the strand (vs. joins it). Forwarded to the
|
|
49
|
+
* StrandDatabase so the founder bootstrap (Header, founding Member/Manager)
|
|
50
|
+
* runs once at bring-up. Joiners leave this unset and write nothing. See
|
|
51
|
+
* {@link StrandConfig.founder}.
|
|
52
|
+
*/
|
|
53
|
+
founder?: boolean;
|
|
54
|
+
/**
|
|
55
|
+
* Number of nodes Optimystic is told this strand's replication cluster should
|
|
56
|
+
* have. Same rule as {@link CadreNodeConfig.strandClusterSize}, which CadreNode
|
|
57
|
+
* forwards here: every node on the strand should use the same value, and it is
|
|
58
|
+
* frozen when the strand's libp2p node is created. Defaults to
|
|
59
|
+
* `DEFAULT_STRAND_CLUSTER_SIZE` (4); values below `MIN_CLUSTER_SIZE` (2) are
|
|
60
|
+
* rejected by `resolveStrandClusterSize`. Strand-only — the
|
|
61
|
+
* control network's breadth is the fixed `CONTROL_REPLICATION_BREADTH`.
|
|
62
|
+
*/
|
|
63
|
+
clusterSize?: number;
|
|
64
|
+
/**
|
|
65
|
+
* Tuning for the strand peer-join block catch-up ({@link StrandBackfill}),
|
|
66
|
+
* forwarded from {@link CadreNodeConfig.strandBackfill}. When the strand's
|
|
67
|
+
* libp2p node connects to a peer this runtime has not yet caught up, every
|
|
68
|
+
* block in the strand's own raw store is pushed to it. Runs only on strands
|
|
69
|
+
* with per-strand storage; `{ enabled: false }` restores the pre-existing
|
|
70
|
+
* no-backfill behaviour.
|
|
71
|
+
*/
|
|
72
|
+
backfill?: StrandBackfillConfig;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Volatile inputs re-resolved when resuming a quiesced strand. The cohort
|
|
77
|
+
* discovery seed is the only one — it grows as peers are learned since the
|
|
78
|
+
* strand first launched.
|
|
79
|
+
*/
|
|
80
|
+
export interface ResumeStrandOverrides {
|
|
81
|
+
/** Freshly-resolved cohort discovery seed (multiaddr strings). */
|
|
82
|
+
bootstrapNodes?: string[];
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Get the isolated storage path for a specific strand.
|
|
87
|
+
*
|
|
88
|
+
* @deprecated This helper is Node-only and throws in React Native (it assumes a
|
|
89
|
+
* filesystem layout). Use a storage provider factory function instead, which
|
|
90
|
+
* receives the strandId and can create strand-specific storage paths using
|
|
91
|
+
* platform-appropriate methods.
|
|
92
|
+
*
|
|
93
|
+
* @example
|
|
94
|
+
* // Instead of using getStrandStoragePath, use a storage provider factory:
|
|
95
|
+
* const storage = {
|
|
96
|
+
* provider: (strandId: string) => new FileRawStorage(`./data/strands/${strandId}`)
|
|
97
|
+
* };
|
|
98
|
+
*/
|
|
99
|
+
export function getStrandStoragePath(basePath: string, strandId: string): string {
|
|
100
|
+
// Check if we're in a Node.js environment
|
|
101
|
+
if (typeof process === 'undefined' || !process.versions?.node) {
|
|
102
|
+
throw new Error(
|
|
103
|
+
'getStrandStoragePath is not available in React Native. ' +
|
|
104
|
+
'Use a storage provider factory function instead.'
|
|
105
|
+
);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
// Sanitize strandId for filesystem safety (UUIDs should be safe, but just in case)
|
|
109
|
+
const safeId = strandId.replace(/[^a-zA-Z0-9-]/g, '_');
|
|
110
|
+
|
|
111
|
+
// Build the path with plain string joins rather than the Node `path` module.
|
|
112
|
+
// A static `require('path')` forces RN bundlers (e.g. Metro) to *resolve* the
|
|
113
|
+
// module at bundle time even though this Node-only helper throws above before
|
|
114
|
+
// ever reaching here — joining by hand keeps the module free of any Node
|
|
115
|
+
// built-in reference, so RN bundles need no `path` shim.
|
|
116
|
+
const trimmedBase = basePath.replace(/[\\/]+$/, '');
|
|
117
|
+
return `${trimmedBase}/strands/${safeId}`;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Resolve a storage provider for a specific strand.
|
|
122
|
+
* If the provider is a factory function, call it with the strandId.
|
|
123
|
+
*
|
|
124
|
+
* Called only from {@link StrandInstanceManager.startStrand}, which owns the result
|
|
125
|
+
* for the instance's lifetime — see `strandStorages`.
|
|
126
|
+
*
|
|
127
|
+
* @param provider - Storage provider (instance or factory)
|
|
128
|
+
* @param strandId - The strand ID to create storage for
|
|
129
|
+
* @returns The resolved IRawStorage instance, or undefined if no provider
|
|
130
|
+
*/
|
|
131
|
+
function resolveStrandStorage(
|
|
132
|
+
provider: RawStorageProvider | undefined,
|
|
133
|
+
strandId: string
|
|
134
|
+
): IRawStorage | undefined {
|
|
135
|
+
if (!provider) {
|
|
136
|
+
return undefined;
|
|
137
|
+
}
|
|
138
|
+
const storage = typeof provider === 'function' ? provider(strandId) : provider;
|
|
139
|
+
// Wrapped in the write-through raw-storage cache (quereus-plugin-sereus's cached-storage.ts).
|
|
140
|
+
// Called ONCE per strand launch — `startStrand` keeps the result for the instance's
|
|
141
|
+
// lifetime — so the wrap survives every runtime rebuild the instance goes through.
|
|
142
|
+
return wrapStorageWithCache(storage, strandId);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* Manages individual strand instances - creates and destroys isolated libp2p nodes
|
|
147
|
+
* for each strand the cadre participates in.
|
|
148
|
+
*/
|
|
149
|
+
export class StrandInstanceManager {
|
|
150
|
+
private instances: Map<string, StrandInstance> = new Map();
|
|
151
|
+
/**
|
|
152
|
+
* Retained launch config per strand, captured in `startStrand` and cleared in
|
|
153
|
+
* `stopStrand`. `resumeStrand` reuses it to rebuild a quiesced strand's runtime
|
|
154
|
+
* without the caller re-threading storage/network/profile/key/sApp config.
|
|
155
|
+
*/
|
|
156
|
+
private launchConfigs: Map<string, StartStrandConfig> = new Map();
|
|
157
|
+
/**
|
|
158
|
+
* The per-strand peer-join block catch-up, keyed by strand id. Private (not on
|
|
159
|
+
* the public {@link StrandInstance}) because it is runtime plumbing with the
|
|
160
|
+
* same lifetime as the strand's libp2p node: created in `buildStrandRuntime`,
|
|
161
|
+
* stopped and dropped in `releaseRuntime` — so quiesce → resume rebuilds it
|
|
162
|
+
* with a fresh caught-up-peer memo, which is intended (a resumed node may have
|
|
163
|
+
* missed writes).
|
|
164
|
+
*/
|
|
165
|
+
private backfills: Map<string, StrandBackfill> = new Map();
|
|
166
|
+
/**
|
|
167
|
+
* The resolved (cache-wrapped) raw storage per strand id — the instance's OWN
|
|
168
|
+
* store, resolved once in `startStrand` and held until `stopStrand` disposes it.
|
|
169
|
+
* Private for the same reason `backfills` is: runtime plumbing, not part of the
|
|
170
|
+
* public {@link StrandInstance}.
|
|
171
|
+
*
|
|
172
|
+
* Deliberately NOT touched by `releaseRuntime`: that is what lets a quiesce →
|
|
173
|
+
* resume cycle rebuild the libp2p node over the SAME store, keeping its
|
|
174
|
+
* write-through cache warm (and, on an in-memory backend, keeping the strand's
|
|
175
|
+
* blocks at all). An entry exists iff `instances` does AND the launch config
|
|
176
|
+
* supplied a storage provider.
|
|
177
|
+
*
|
|
178
|
+
* NOTE: a hibernating strand therefore keeps its store — and its share of the
|
|
179
|
+
* process-wide cache pool — resident for as long as the instance is tracked. That
|
|
180
|
+
* is the point (a warm wake), and the pool evicts under pressure, so the cost is
|
|
181
|
+
* one map entry per hibernating strand today. If a device ever hibernates strands
|
|
182
|
+
* by the hundred, revisit: dropping the store at quiesce and paying for a cold
|
|
183
|
+
* wake becomes the better trade.
|
|
184
|
+
*/
|
|
185
|
+
private strandStorages: Map<string, IRawStorage> = new Map();
|
|
186
|
+
private stopping = false;
|
|
187
|
+
|
|
188
|
+
constructor() {
|
|
189
|
+
log('StrandInstanceManager created');
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/**
|
|
193
|
+
* Get all current strand instances
|
|
194
|
+
*/
|
|
195
|
+
getInstances(): Map<string, StrandInstance> {
|
|
196
|
+
return new Map(this.instances);
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* Get a specific strand instance
|
|
201
|
+
*/
|
|
202
|
+
getInstance(strandId: string): StrandInstance | undefined {
|
|
203
|
+
return this.instances.get(strandId);
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
/**
|
|
207
|
+
* Check if a strand is currently running
|
|
208
|
+
*/
|
|
209
|
+
hasStrand(strandId: string): boolean {
|
|
210
|
+
return this.instances.has(strandId);
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* Start a new strand instance.
|
|
215
|
+
*
|
|
216
|
+
* A failed launch leaves NOTHING tracked: the instance and its retained launch
|
|
217
|
+
* config are both dropped before the error is rethrown, so the strand id is
|
|
218
|
+
* free for a genuine retry. This matches the pre-registration failure path
|
|
219
|
+
* (a rejected schema signature, which throws before anything is recorded) —
|
|
220
|
+
* both failure modes of this call leave the same residue: none. Callers learn
|
|
221
|
+
* of the failure from the rejected promise (and, on the control-discovered
|
|
222
|
+
* path, from CadreNode's `strand:error` event), not from an error record left
|
|
223
|
+
* behind in `instances`.
|
|
224
|
+
*
|
|
225
|
+
* The strand's raw storage is resolved HERE, once, and held for the instance's
|
|
226
|
+
* lifetime (see `strandStorages`) — `buildStrandRuntime` only reads it, so a
|
|
227
|
+
* hibernation wake never re-enters the embedder's provider.
|
|
228
|
+
*/
|
|
229
|
+
async startStrand(config: StartStrandConfig): Promise<StrandInstance> {
|
|
230
|
+
const { strandRow, sAppConfig } = config;
|
|
231
|
+
const strandId = strandRow.Id;
|
|
232
|
+
|
|
233
|
+
if (this.stopping) {
|
|
234
|
+
throw new Error('StrandInstanceManager is stopping');
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
if (this.instances.has(strandId)) {
|
|
238
|
+
log('Strand %s already running', strandId);
|
|
239
|
+
return this.instances.get(strandId)!;
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
log('Starting strand instance: %s (sApp: %s v%s)', strandId, sAppConfig.id, sAppConfig.version);
|
|
243
|
+
const tTotal = performance.now();
|
|
244
|
+
|
|
245
|
+
// Verify schema signature before proceeding (fail-closed by default)
|
|
246
|
+
const requireSignature = config.requireSignedSchemas ?? true;
|
|
247
|
+
assertSchemaSignature(sAppConfig, { requireSignature });
|
|
248
|
+
log('Strand %s sApp schema signature verified (author: %s)', strandId, sAppConfig.id);
|
|
249
|
+
|
|
250
|
+
// Convert SAppConfig to SAppInfo for the instance
|
|
251
|
+
const sAppInfo: SAppInfo = {
|
|
252
|
+
id: sAppConfig.id,
|
|
253
|
+
version: sAppConfig.version,
|
|
254
|
+
schema: sAppConfig.schema,
|
|
255
|
+
signature: sAppConfig.signature
|
|
256
|
+
};
|
|
257
|
+
|
|
258
|
+
// Resolve this strand's storage ONCE, before anything is recorded. If a factory
|
|
259
|
+
// function is provided, it is called with the strandId to create strand-specific
|
|
260
|
+
// storage (e.g. strand-isolated directories). A provider that throws therefore
|
|
261
|
+
// fails the launch alongside the schema-signature check, leaving nothing tracked.
|
|
262
|
+
const strandStorage = resolveStrandStorage(config.storage?.provider, strandId);
|
|
263
|
+
if (strandStorage) {
|
|
264
|
+
log('Strand %s using provided storage provider', strandId);
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
// Determine latency hint: sApp config > default
|
|
268
|
+
const latencyHint = sAppConfig.latencyHint ?? config.defaultLatencyHint;
|
|
269
|
+
|
|
270
|
+
const instance: StrandInstance = {
|
|
271
|
+
strandId,
|
|
272
|
+
status: 'starting',
|
|
273
|
+
sAppInfo,
|
|
274
|
+
memberPrivateKey: strandRow.MemberPrivateKey ?? undefined,
|
|
275
|
+
connectedPeers: 0,
|
|
276
|
+
lastActivity: new Date(),
|
|
277
|
+
latencyHint
|
|
278
|
+
};
|
|
279
|
+
|
|
280
|
+
this.instances.set(strandId, instance);
|
|
281
|
+
this.launchConfigs.set(strandId, config);
|
|
282
|
+
if (strandStorage) {
|
|
283
|
+
this.strandStorages.set(strandId, strandStorage);
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
try {
|
|
287
|
+
await this.buildStrandRuntime(instance, config);
|
|
288
|
+
timing('[startStrand:%s] total: %dms', strandId, Math.round(performance.now() - tTotal));
|
|
289
|
+
log('Strand %s started successfully with sApp %s', strandId, sAppConfig.id);
|
|
290
|
+
return instance;
|
|
291
|
+
} catch (error) {
|
|
292
|
+
// Status/error first — the (now discarded) record is still what `log` reports on.
|
|
293
|
+
instance.status = 'error';
|
|
294
|
+
instance.error = error instanceof Error ? error.message : String(error);
|
|
295
|
+
log('Failed to start strand %s: %s', strandId, instance.error);
|
|
296
|
+
// Drop the dead record so this strand id can be launched again. Keep the
|
|
297
|
+
// `launchConfigs` has an entry iff `instances` does invariant — resumeStrand
|
|
298
|
+
// reads both, and a config without an instance would strand the config.
|
|
299
|
+
this.instances.delete(strandId);
|
|
300
|
+
this.launchConfigs.delete(strandId);
|
|
301
|
+
// The store this launch resolved goes with it: nothing owns it any more, and a
|
|
302
|
+
// retained cache wrapper would be handed back (already retired) on a retry.
|
|
303
|
+
await this.disposeStrandStorage(strandId);
|
|
304
|
+
throw error;
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
/**
|
|
309
|
+
* Build (or rebuild) the libp2p node + StrandDatabase for an instance and
|
|
310
|
+
* attach them, transitioning it to `active`. Shared by `startStrand` (fresh
|
|
311
|
+
* launch) and `resumeStrand` (rehydrating a quiesced instance). Reads all
|
|
312
|
+
* volatile inputs (bootstrapNodes, network, profile, privateKey, sApp config)
|
|
313
|
+
* from `config`, so the caller controls the cohort-derived values. Storage is the
|
|
314
|
+
* one input it does NOT re-read from `config`: that belongs to the instance and
|
|
315
|
+
* comes from `strandStorages`.
|
|
316
|
+
*/
|
|
317
|
+
private async buildStrandRuntime(instance: StrandInstance, config: StartStrandConfig): Promise<void> {
|
|
318
|
+
const strandId = instance.strandId;
|
|
319
|
+
const { sAppConfig } = config;
|
|
320
|
+
|
|
321
|
+
// The store the instance OWNS (resolved once in `startStrand`), not a fresh
|
|
322
|
+
// resolution: a rebuild must reach the same backend through the same warm cache.
|
|
323
|
+
const strandStorage = this.strandStorages.get(strandId);
|
|
324
|
+
|
|
325
|
+
// db-p2p namespaces every one of the node's protocol ids by network name
|
|
326
|
+
// (`/optimystic/<networkName>/...`), so anything dialing this node's own
|
|
327
|
+
// services must derive its prefix from the SAME string the node was built
|
|
328
|
+
// with — hence one binding for both, not two literals that can drift.
|
|
329
|
+
const networkName = `strand-${strandId}`;
|
|
330
|
+
const protocolPrefix = `/optimystic/${networkName}`;
|
|
331
|
+
|
|
332
|
+
// Determine relay mode: if explicitly set in config, use that;
|
|
333
|
+
// otherwise default to true for storage profile nodes.
|
|
334
|
+
const enableRelay = config.network?.enableRelay ?? (config.profile === 'storage');
|
|
335
|
+
// The CONFIGURED relay route (the default), deliberately NOT the control node's
|
|
336
|
+
// `'search'` route: nothing drives an explicit reservation for a strand node, so
|
|
337
|
+
// a bare `/p2p-circuit` search entry would register a pending reservation that
|
|
338
|
+
// never gets filled and leave the strand node undialable. The ordering hazard
|
|
339
|
+
// that pushed the control node onto the search route does not exist here — a
|
|
340
|
+
// strand node's protocol ids are namespaced `/optimystic/strand-<id>/…`, so a
|
|
341
|
+
// relay dialed from inside `libp2p.start()` is never in the strand's cohort and
|
|
342
|
+
// cannot refuse its database bring-up. See `relay-addrs.ts`.
|
|
343
|
+
const listenAddrs = resolveListenAddrs(config.network);
|
|
344
|
+
|
|
345
|
+
try {
|
|
346
|
+
let t0 = performance.now();
|
|
347
|
+
const node = await createLibp2pNode({
|
|
348
|
+
port: 0, // Random port
|
|
349
|
+
bootstrapNodes: config.bootstrapNodes ?? [],
|
|
350
|
+
networkName,
|
|
351
|
+
storage: strandStorage,
|
|
352
|
+
fretProfile: config.profile === 'storage' ? 'core' : 'edge',
|
|
353
|
+
relay: enableRelay,
|
|
354
|
+
clusterSize: resolveStrandClusterSize(config.clusterSize),
|
|
355
|
+
// Deliberately NOT CONTROL_CLUSTER_POLICY: a strand is application data with its own
|
|
356
|
+
// breadth reasoning, and the shape match with the control policy is a coincidence.
|
|
357
|
+
clusterPolicy: STRAND_CLUSTER_POLICY,
|
|
358
|
+
arachnode: {
|
|
359
|
+
enableRingZulu: config.profile === 'storage'
|
|
360
|
+
},
|
|
361
|
+
...(config.privateKey && { privateKey: config.privateKey }),
|
|
362
|
+
...(config.network?.transports && { transports: config.network.transports }),
|
|
363
|
+
// NOTE: strand nodes receive the same RESOLVED listen addrs as the
|
|
364
|
+
// control node. A fixed-port listen addr (e.g. cadre-cli's example
|
|
365
|
+
// `/ip4/0.0.0.0/tcp/4001`) would have control + strand nodes racing to
|
|
366
|
+
// bind one port — EADDRINUSE. Unverified; if a deployment configures a
|
|
367
|
+
// fixed port and strands fail to start, rewrite the port per node here.
|
|
368
|
+
// An inherited configured `/p2p-circuit` addr, by contrast, is deliberate —
|
|
369
|
+
// it is what gives a NAT'd strand node a reachable relay slot — and works
|
|
370
|
+
// because the launch path announces this strand's derived peerId to
|
|
371
|
+
// the relay first (delegate admission; see cadre-node.ts). Those circuit
|
|
372
|
+
// entries come either from a hand-written `network.listenAddrs` or from
|
|
373
|
+
// `network.relayAddrs`, which `resolveListenAddrs` folds into the same list
|
|
374
|
+
// on the configured route this call takes (see the comment above it).
|
|
375
|
+
...(listenAddrs && { listenAddrs }),
|
|
376
|
+
// A strand node behind a reverse proxy has the same advertising need as the
|
|
377
|
+
// control node, and inherits the same `NetworkConfig` — so it announces the same
|
|
378
|
+
// addresses.
|
|
379
|
+
//
|
|
380
|
+
// NOTE: an announce addr naming a PORT therefore names the control node's port,
|
|
381
|
+
// not this strand node's — the same second-node-on-one-host problem the listen
|
|
382
|
+
// NOTE above describes, and the reason both belong to one fix. It is currently
|
|
383
|
+
// unreachable: naming a fixed public port means configuring a fixed
|
|
384
|
+
// `listenAddrs` port, and that collides at bind time before any strand node gets
|
|
385
|
+
// far enough to advertise anything. Whatever gives strand nodes their own listen
|
|
386
|
+
// port has to give them their own announce addrs in the same pass, or a strand
|
|
387
|
+
// will advertise an address that reaches the control node. Tracked as an arm of
|
|
388
|
+
// `tickets/backlog/strand-network-nat-relay-reachability.md`.
|
|
389
|
+
...resolveAnnounceAddrs(config.network),
|
|
390
|
+
...(config.network?.connectionGater && { connectionGater: config.network.connectionGater })
|
|
391
|
+
}) as Libp2pNodeWithRepo;
|
|
392
|
+
timing('[buildStrandRuntime:%s] createLibp2pNode: %dms', strandId, Math.round(performance.now() - t0));
|
|
393
|
+
|
|
394
|
+
instance.libp2pNode = node;
|
|
395
|
+
|
|
396
|
+
// Create and initialize the StrandDatabase.
|
|
397
|
+
//
|
|
398
|
+
// Attach before initialize so a failed init is cleaned up by
|
|
399
|
+
// releaseRuntime below (close() is safe on a partially-initialized db).
|
|
400
|
+
t0 = performance.now();
|
|
401
|
+
const strandDb = new StrandDatabase({
|
|
402
|
+
strandId,
|
|
403
|
+
sAppConfig,
|
|
404
|
+
libp2pNode: node,
|
|
405
|
+
coordinatedRepo: node.coordinatedRepo,
|
|
406
|
+
// Founder bootstrap inputs: the strand's type drives which membership rows
|
|
407
|
+
// are written, and the closed-strand MemberPrivateKey derives the founding
|
|
408
|
+
// Member/Manager key. Both come off the control-network strand row.
|
|
409
|
+
strandType: config.strandRow.Type,
|
|
410
|
+
memberPrivateKey: config.strandRow.MemberPrivateKey ?? undefined,
|
|
411
|
+
founder: config.founder
|
|
412
|
+
});
|
|
413
|
+
instance.database = strandDb;
|
|
414
|
+
await strandDb.initialize();
|
|
415
|
+
timing('[buildStrandRuntime:%s] strandDatabase.initialize: %dms', strandId, Math.round(performance.now() - t0));
|
|
416
|
+
|
|
417
|
+
// Peer-join block catch-up: push this strand's own blocks to each newly
|
|
418
|
+
// connected peer, so a machine that joined after blocks were committed
|
|
419
|
+
// still ends up physically holding them (without per-strand storage there
|
|
420
|
+
// is nothing to copy). Armed for EVERY stored strand: `StrandBackfill`
|
|
421
|
+
// only does work when the strand's libp2p node reports a peer connection,
|
|
422
|
+
// so on a device that is genuinely alone it is inert, and arming it at
|
|
423
|
+
// launch is what closes the "founded alone, never replicates" hole — a
|
|
424
|
+
// peer that joins later gets the founder's blocks without any relaunch.
|
|
425
|
+
//
|
|
426
|
+
// NOTE: cost is one StrandBackfill object + one `connection:open` listener
|
|
427
|
+
// per running strand — linear in strand count, negligible at the handful a
|
|
428
|
+
// device or host runs today. If a node ever hosts strands by the hundred,
|
|
429
|
+
// move to one shared listener that dispatches by strand id.
|
|
430
|
+
if (strandStorage && config.backfill?.enabled !== false) {
|
|
431
|
+
if (node.keyNetwork) {
|
|
432
|
+
const backfill = new StrandBackfill({
|
|
433
|
+
strandId,
|
|
434
|
+
libp2p: node,
|
|
435
|
+
peerNetwork: node.keyNetwork,
|
|
436
|
+
storage: strandStorage,
|
|
437
|
+
// The same prefix the receiver registered its block-transfer handler
|
|
438
|
+
// under — derived from networkName above, never re-spelled here.
|
|
439
|
+
protocolPrefix
|
|
440
|
+
}, config.backfill);
|
|
441
|
+
backfill.start();
|
|
442
|
+
this.backfills.set(strandId, backfill);
|
|
443
|
+
} else {
|
|
444
|
+
log('Strand %s: libp2p node exposes no keyNetwork; peer-join block catch-up is inert', strandId);
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
instance.status = 'active';
|
|
449
|
+
instance.lastActivity = new Date();
|
|
450
|
+
} catch (error) {
|
|
451
|
+
// Roll back any partially-attached runtime so the instance is left with
|
|
452
|
+
// NEITHER handle. Otherwise the `libp2pNode || database` "already live"
|
|
453
|
+
// guard in resumeStrand/handleStrandWake would treat a half-built strand
|
|
454
|
+
// as healthy — leaking the libp2p node and never retrying the rebuild.
|
|
455
|
+
await this.releaseRuntime(instance).catch((cleanupErr) => {
|
|
456
|
+
log('buildStrandRuntime cleanup for strand %s also failed: %o', strandId, cleanupErr);
|
|
457
|
+
});
|
|
458
|
+
throw error;
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
/**
|
|
463
|
+
* Release an instance's strand-network runtime: close the StrandDatabase, then
|
|
464
|
+
* stop the libp2p node (construction order in reverse), clearing both fields
|
|
465
|
+
* and zeroing connectedPeers. Tolerant of partially-built state — either handle
|
|
466
|
+
* may be absent — so it doubles as rollback for a failed `buildStrandRuntime`.
|
|
467
|
+
* Shared by `quiesceStrand`, `stopStrand`, and that rollback path.
|
|
468
|
+
*
|
|
469
|
+
* Leaves `strandStorages` untouched by design — the store outlives the runtime it
|
|
470
|
+
* was built into, which is what makes a resume warm. Only `stopStrand` disposes it.
|
|
471
|
+
*/
|
|
472
|
+
private async releaseRuntime(instance: StrandInstance): Promise<void> {
|
|
473
|
+
// Backfill first — before the database closes and the libp2p node stops — so
|
|
474
|
+
// no NEW catch-up push is issued against a torn-down transport. A push already
|
|
475
|
+
// in flight is not awaited; it fails into the module's own per-chunk catch.
|
|
476
|
+
const backfill = this.backfills.get(instance.strandId);
|
|
477
|
+
if (backfill) {
|
|
478
|
+
backfill.stop();
|
|
479
|
+
this.backfills.delete(instance.strandId);
|
|
480
|
+
}
|
|
481
|
+
if (instance.database) {
|
|
482
|
+
await instance.database.close();
|
|
483
|
+
instance.database = undefined;
|
|
484
|
+
}
|
|
485
|
+
if (instance.libp2pNode) {
|
|
486
|
+
await instance.libp2pNode.stop();
|
|
487
|
+
instance.libp2pNode = undefined;
|
|
488
|
+
}
|
|
489
|
+
instance.connectedPeers = 0;
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
/**
|
|
493
|
+
* Drop the strand's owned store and release this strand's claim on its cache. The
|
|
494
|
+
* wrapper counts holders, so the cache is emptied and unregistered from the shared
|
|
495
|
+
* pool only if no other scope still holds it. Called only where the strand's whole
|
|
496
|
+
* lifetime ends (`stopStrand`, and the failed-launch rollback) — never on a quiesce.
|
|
497
|
+
*
|
|
498
|
+
* A dispose failure is logged, not thrown: the store is already unreferenced here,
|
|
499
|
+
* and failing the stop over a cache-bookkeeping error would leave the caller unable
|
|
500
|
+
* to tear the strand down. `disposeStorageCache` no-ops for an unwrapped store
|
|
501
|
+
* (e.g. `MemoryRawStorage`), so no instanceof test is needed.
|
|
502
|
+
*/
|
|
503
|
+
private async disposeStrandStorage(strandId: string): Promise<void> {
|
|
504
|
+
const storage = this.strandStorages.get(strandId);
|
|
505
|
+
if (!storage) {
|
|
506
|
+
return;
|
|
507
|
+
}
|
|
508
|
+
this.strandStorages.delete(strandId);
|
|
509
|
+
try {
|
|
510
|
+
await disposeStorageCache(storage);
|
|
511
|
+
} catch (error) {
|
|
512
|
+
log('Failed to dispose storage cache for strand %s: %o', strandId, error);
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
/**
|
|
517
|
+
* Quiesce a strand: release its strand-network resources (stop the libp2p node,
|
|
518
|
+
* close the StrandDatabase) while RETAINING the instance record — identity,
|
|
519
|
+
* sAppInfo, keys, latency hint, metadata — and its launch config so it can be
|
|
520
|
+
* resumed later. Mechanically this is `stopStrand` minus the instance/config
|
|
521
|
+
* deletion. The caller sets the post-quiesce status (e.g. `hibernating`).
|
|
522
|
+
* No-ops when the strand is missing or already quiesced.
|
|
523
|
+
*/
|
|
524
|
+
async quiesceStrand(strandId: string): Promise<void> {
|
|
525
|
+
const instance = this.instances.get(strandId);
|
|
526
|
+
if (!instance) {
|
|
527
|
+
log('quiesceStrand: strand %s not found', strandId);
|
|
528
|
+
return;
|
|
529
|
+
}
|
|
530
|
+
if (!instance.libp2pNode && !instance.database) {
|
|
531
|
+
log('quiesceStrand: strand %s already quiesced', strandId);
|
|
532
|
+
return;
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
log('Quiescing strand instance: %s', strandId);
|
|
536
|
+
await this.releaseRuntime(instance);
|
|
537
|
+
log('Strand %s quiesced (resources released, instance retained)', strandId);
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
/**
|
|
541
|
+
* Resume a previously-quiesced strand: rebuild its libp2p node + StrandDatabase
|
|
542
|
+
* from the retained launch config and re-attach them, transitioning it back to
|
|
543
|
+
* `active`. `overrides` re-applies volatile inputs that may have changed since
|
|
544
|
+
* launch (the cohort `bootstrapNodes` seed) and updates the retained config so
|
|
545
|
+
* a later resume reuses the latest values. Returns the live instance unchanged
|
|
546
|
+
* if it is already running.
|
|
547
|
+
*/
|
|
548
|
+
async resumeStrand(strandId: string, overrides?: ResumeStrandOverrides): Promise<StrandInstance> {
|
|
549
|
+
if (this.stopping) {
|
|
550
|
+
throw new Error('StrandInstanceManager is stopping');
|
|
551
|
+
}
|
|
552
|
+
const instance = this.instances.get(strandId);
|
|
553
|
+
if (!instance) {
|
|
554
|
+
throw new Error(`Cannot resume strand ${strandId}: not tracked`);
|
|
555
|
+
}
|
|
556
|
+
const launchConfig = this.launchConfigs.get(strandId);
|
|
557
|
+
if (!launchConfig) {
|
|
558
|
+
throw new Error(`Cannot resume strand ${strandId}: no retained launch config`);
|
|
559
|
+
}
|
|
560
|
+
if (instance.libp2pNode || instance.database) {
|
|
561
|
+
log('resumeStrand: strand %s already live', strandId);
|
|
562
|
+
return instance;
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
log('Resuming strand instance: %s', strandId);
|
|
566
|
+
const tTotal = performance.now();
|
|
567
|
+
|
|
568
|
+
// Re-apply volatile inputs and persist them so a subsequent resume reuses them.
|
|
569
|
+
const resumeConfig: StartStrandConfig = {
|
|
570
|
+
...launchConfig,
|
|
571
|
+
bootstrapNodes: overrides?.bootstrapNodes ?? launchConfig.bootstrapNodes
|
|
572
|
+
};
|
|
573
|
+
this.launchConfigs.set(strandId, resumeConfig);
|
|
574
|
+
|
|
575
|
+
instance.status = 'starting';
|
|
576
|
+
try {
|
|
577
|
+
await this.buildStrandRuntime(instance, resumeConfig);
|
|
578
|
+
timing('[resumeStrand:%s] total: %dms', strandId, Math.round(performance.now() - tTotal));
|
|
579
|
+
log('Strand %s resumed successfully', strandId);
|
|
580
|
+
return instance;
|
|
581
|
+
} catch (error) {
|
|
582
|
+
instance.status = 'error';
|
|
583
|
+
instance.error = error instanceof Error ? error.message : String(error);
|
|
584
|
+
log('Failed to resume strand %s: %s', strandId, instance.error);
|
|
585
|
+
throw error;
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
/**
|
|
590
|
+
* Stop a strand instance
|
|
591
|
+
*/
|
|
592
|
+
async stopStrand(strandId: string): Promise<void> {
|
|
593
|
+
const instance = this.instances.get(strandId);
|
|
594
|
+
if (!instance) {
|
|
595
|
+
log('Strand %s not found', strandId);
|
|
596
|
+
return;
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
log('Stopping strand instance: %s', strandId);
|
|
600
|
+
instance.status = 'stopping';
|
|
601
|
+
|
|
602
|
+
try {
|
|
603
|
+
await this.releaseRuntime(instance);
|
|
604
|
+
instance.status = 'stopped';
|
|
605
|
+
this.instances.delete(strandId);
|
|
606
|
+
this.launchConfigs.delete(strandId);
|
|
607
|
+
// Storage is released only here — NOT in releaseRuntime, which a quiesce shares.
|
|
608
|
+
await this.disposeStrandStorage(strandId);
|
|
609
|
+
log('Strand %s stopped successfully', strandId);
|
|
610
|
+
} catch (error) {
|
|
611
|
+
instance.status = 'error';
|
|
612
|
+
instance.error = error instanceof Error ? error.message : String(error);
|
|
613
|
+
log('Error stopping strand %s: %s', strandId, instance.error);
|
|
614
|
+
throw error;
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
/**
|
|
619
|
+
* Stop all strand instances
|
|
620
|
+
*/
|
|
621
|
+
async stopAll(): Promise<void> {
|
|
622
|
+
this.stopping = true;
|
|
623
|
+
log('Stopping all strand instances (%d)', this.instances.size);
|
|
624
|
+
|
|
625
|
+
const stopPromises = Array.from(this.instances.keys()).map(id =>
|
|
626
|
+
this.stopStrand(id).catch(err => {
|
|
627
|
+
log('Error stopping strand %s during shutdown: %s', id, err);
|
|
628
|
+
})
|
|
629
|
+
);
|
|
630
|
+
|
|
631
|
+
await Promise.all(stopPromises);
|
|
632
|
+
this.stopping = false;
|
|
633
|
+
log('All strand instances stopped');
|
|
634
|
+
}
|
|
635
|
+
}
|
|
636
|
+
|