@serfab/cadre-core 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/dist/cadre-node.d.ts +328 -25
- package/dist/cadre-node.js +760 -59
- package/dist/cadre-node.js.map +1 -1
- package/dist/control-database.d.ts +19 -0
- package/dist/control-database.js +51 -9
- package/dist/control-database.js.map +1 -1
- package/dist/control-schema.d.ts +1 -1
- package/dist/control-schema.js +30 -2
- package/dist/control-schema.js.map +1 -1
- package/dist/enrolled-machine-store-file.d.ts +13 -0
- package/dist/enrolled-machine-store-file.js +41 -0
- package/dist/enrolled-machine-store-file.js.map +1 -0
- package/dist/enrolled-machine-store.d.ts +81 -0
- package/dist/enrolled-machine-store.js +225 -0
- package/dist/enrolled-machine-store.js.map +1 -0
- package/dist/file-durable-slot.js +5 -3
- package/dist/file-durable-slot.js.map +1 -1
- package/dist/index.d.ts +5 -3
- package/dist/index.js +23 -5
- package/dist/index.js.map +1 -1
- package/dist/membership-connection-gater.d.ts +15 -1
- package/dist/membership-connection-gater.js +9 -3
- package/dist/membership-connection-gater.js.map +1 -1
- package/dist/node-local-snapshot.d.ts +6 -0
- package/dist/node-local-snapshot.js +7 -1
- package/dist/node-local-snapshot.js.map +1 -1
- package/dist/{strand-backfill.d.ts → peer-join-backfill.d.ts} +49 -20
- package/dist/{strand-backfill.js → peer-join-backfill.js} +133 -52
- package/dist/peer-join-backfill.js.map +1 -0
- package/dist/relay-addrs.d.ts +73 -0
- package/dist/relay-addrs.js +195 -0
- package/dist/relay-addrs.js.map +1 -1
- package/dist/strand-database.d.ts +12 -0
- package/dist/strand-database.js +16 -0
- package/dist/strand-database.js.map +1 -1
- package/dist/strand-formation-manager.d.ts +12 -0
- package/dist/strand-formation-manager.js +10 -3
- package/dist/strand-formation-manager.js.map +1 -1
- package/dist/strand-formation-protocol.d.ts +76 -2
- package/dist/strand-formation-protocol.js +78 -2
- package/dist/strand-formation-protocol.js.map +1 -1
- package/dist/strand-instance-manager.d.ts +147 -14
- package/dist/strand-instance-manager.js +225 -45
- package/dist/strand-instance-manager.js.map +1 -1
- package/dist/strand-membership-writer.d.ts +12 -0
- package/dist/strand-membership-writer.js +23 -1
- package/dist/strand-membership-writer.js.map +1 -1
- package/dist/strand-network-config.d.ts +84 -0
- package/dist/strand-network-config.js +136 -0
- package/dist/strand-network-config.js.map +1 -0
- package/dist/strand-revocation-enforcer.d.ts +392 -0
- package/dist/strand-revocation-enforcer.js +533 -0
- package/dist/strand-revocation-enforcer.js.map +1 -0
- package/dist/strand-solicitation.d.ts +7 -0
- package/dist/strand-solicitation.js +8 -2
- package/dist/strand-solicitation.js.map +1 -1
- package/dist/strand-watcher.js +6 -1
- package/dist/strand-watcher.js.map +1 -1
- package/dist/types.d.ts +210 -16
- package/dist/types.js +7 -1
- package/dist/types.js.map +1 -1
- package/package.json +16 -8
- package/src/cadre-node.ts +6353 -5603
- package/src/control-database.ts +53 -9
- package/src/control-schema.ts +30 -2
- package/src/enrolled-machine-store-file.ts +43 -0
- package/src/enrolled-machine-store.ts +280 -0
- package/src/file-durable-slot.ts +5 -3
- package/src/index.ts +52 -10
- package/src/membership-connection-gater.ts +9 -3
- package/src/node-local-snapshot.ts +13 -1
- package/src/peer-addr-book.ts +251 -250
- package/src/{strand-backfill.ts → peer-join-backfill.ts} +478 -389
- package/src/relay-addrs.ts +218 -0
- package/src/strand-database.ts +17 -0
- package/src/strand-formation-manager.ts +22 -3
- package/src/strand-formation-protocol.ts +808 -692
- package/src/strand-instance-manager.ts +317 -54
- package/src/strand-membership-writer.ts +23 -1
- package/src/strand-network-config.ts +172 -0
- package/src/strand-revocation-enforcer.ts +653 -0
- package/src/strand-solicitation.ts +484 -471
- package/src/strand-watcher.ts +319 -314
- package/src/types.ts +1554 -1354
- package/dist/strand-backfill.js.map +0 -1
|
@@ -3,7 +3,13 @@ import type { PrivateKey } from '@libp2p/interface';
|
|
|
3
3
|
import { createLibp2pNode, type IRawStorage } from '@optimystic/db-p2p';
|
|
4
4
|
import { wrapStorageWithCache, disposeStorageCache } from '@serfab/quereus-plugin-sereus';
|
|
5
5
|
import { StrandDatabase } from './strand-database.js';
|
|
6
|
-
import {
|
|
6
|
+
import { PeerJoinBackfill, type PeerJoinBackfillConfig } from './peer-join-backfill.js';
|
|
7
|
+
import {
|
|
8
|
+
StrandRevocationEnforcer,
|
|
9
|
+
createRevocationConnectionGater,
|
|
10
|
+
readStrandRevocationRows,
|
|
11
|
+
type StrandRevocationEnforcementConfig
|
|
12
|
+
} from './strand-revocation-enforcer.js';
|
|
7
13
|
import { assertSchemaSignature } from './schema-verification.js';
|
|
8
14
|
import type {
|
|
9
15
|
StrandInstance,
|
|
@@ -17,9 +23,8 @@ import type {
|
|
|
17
23
|
RawStorageProvider,
|
|
18
24
|
Libp2pNodeWithRepo
|
|
19
25
|
} from './types.js';
|
|
20
|
-
import { resolveStrandClusterSize,
|
|
21
|
-
import {
|
|
22
|
-
import { resolveAnnounceAddrs } from './announce-addrs.js';
|
|
26
|
+
import { resolveStrandClusterSize, strandClusterPolicy } from './types.js';
|
|
27
|
+
import { strandNodeAddrs } from './strand-network-config.js';
|
|
23
28
|
|
|
24
29
|
const log = debug('sereus:cadre:strand-manager');
|
|
25
30
|
const timing = debug('sereus:cadre:timing');
|
|
@@ -47,8 +52,12 @@ export interface StartStrandConfig {
|
|
|
47
52
|
/**
|
|
48
53
|
* Whether this node founds the strand (vs. joins it). Forwarded to the
|
|
49
54
|
* StrandDatabase so the founder bootstrap (Header, founding Member/Manager)
|
|
50
|
-
* runs once at bring-up. Joiners leave this unset and write nothing.
|
|
51
|
-
*
|
|
55
|
+
* runs once at bring-up. Joiners leave this unset and write nothing. Callers
|
|
56
|
+
* resolve it BEFORE calling `startStrand` (`CadreNode.launchStrand` derives it
|
|
57
|
+
* from the row's `FounderOwnerKey` when no explicit flag is given — see
|
|
58
|
+
* {@link StrandConfig.founder}); a founder request that arrives while the
|
|
59
|
+
* instance is already tracked goes through {@link StrandInstanceManager.foundExistingStrand},
|
|
60
|
+
* never through a repeat `startStrand`.
|
|
52
61
|
*/
|
|
53
62
|
founder?: boolean;
|
|
54
63
|
/**
|
|
@@ -62,24 +71,87 @@ export interface StartStrandConfig {
|
|
|
62
71
|
*/
|
|
63
72
|
clusterSize?: number;
|
|
64
73
|
/**
|
|
65
|
-
*
|
|
74
|
+
* Machines that SERVE THIS STRAND (this node included), for the strand node's
|
|
75
|
+
* block-repair corroboration yardstick — see `resolveRepairYardstick`. It must be
|
|
76
|
+
* an authenticated per-strand count: the machines that actually run a node for this
|
|
77
|
+
* strand, not the machines that exist.
|
|
78
|
+
*
|
|
79
|
+
* **The party's enrolled-machine count is NOT this number and must never be passed
|
|
80
|
+
* here.** A strand launches only on machines whose embedding app registered its sApp
|
|
81
|
+
* config (`CadreNode.addStrand`), so a closed strand shared by two machines of a
|
|
82
|
+
* three-machine party is served by two. Passing three over-declares, and
|
|
83
|
+
* over-declaring is the unsafe direction: at a declaration of 3 or more Optimystic
|
|
84
|
+
* pins the repair corroboration floor at two corroborating peers, which a cohort
|
|
85
|
+
* that can only ever field one peer can never reach — so the strand can never repair
|
|
86
|
+
* a block (`cluster-fetch:no-quorum`, surfacing as reads failing with `Missing
|
|
87
|
+
* block`). That regression is why this field was renamed off "enrolledMachines":
|
|
88
|
+
* `bug-strand-yardstick-counts-party-machines`.
|
|
89
|
+
*
|
|
90
|
+
* **Nothing feeds it in production today.** No authenticated per-strand serving count
|
|
91
|
+
* exists yet, so `CadreNode` passes nothing and the strand node runs the frozen
|
|
92
|
+
* `STRAND_CLUSTER_POLICY` — declaring no yardstick, which leaves the known,
|
|
93
|
+
* upstream-tracked single-voter exposure
|
|
94
|
+
* (`backlog/debt-read-repair-single-voter-corroboration`). Building the count is
|
|
95
|
+
* `backlog/feat-strand-yardstick-from-serving-machines`; this field and its
|
|
96
|
+
* threading are the seam it plugs into.
|
|
97
|
+
*
|
|
98
|
+
* Volatile when a source does exist: re-resolve it on every resume beside the cohort
|
|
99
|
+
* seed, since machines join and leave a strand while it hibernates. Omitting it means
|
|
100
|
+
* "this node does not know", which declares nothing.
|
|
101
|
+
*/
|
|
102
|
+
servingMachines?: number;
|
|
103
|
+
/**
|
|
104
|
+
* Tuning for the strand peer-join block catch-up ({@link PeerJoinBackfill}),
|
|
66
105
|
* forwarded from {@link CadreNodeConfig.strandBackfill}. When the strand's
|
|
67
106
|
* libp2p node connects to a peer this runtime has not yet caught up, every
|
|
68
107
|
* block in the strand's own raw store is pushed to it. Runs only on strands
|
|
69
108
|
* with per-strand storage; `{ enabled: false }` restores the pre-existing
|
|
70
109
|
* no-backfill behaviour.
|
|
71
110
|
*/
|
|
72
|
-
backfill?:
|
|
111
|
+
backfill?: PeerJoinBackfillConfig;
|
|
112
|
+
/**
|
|
113
|
+
* Tuning for the CLOSED-strand revoked-peer gate
|
|
114
|
+
* ({@link StrandRevocationEnforcer}), forwarded from
|
|
115
|
+
* {@link CadreNodeConfig.strandRevocationEnforcement}. Armed only for
|
|
116
|
+
* `strandRow.Type === 'c'` (an open strand has no membership rows to derive a
|
|
117
|
+
* deny set from); `{ enabled: false }` restores the pre-existing behaviour
|
|
118
|
+
* (a removed party's peers keep being served).
|
|
119
|
+
*/
|
|
120
|
+
revocationEnforcement?: StrandRevocationEnforcementConfig;
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Called when THIS node's own peer id turns up in the strand's revoked set —
|
|
124
|
+
* this node's party was removed from the closed strand, or left it. Fires once
|
|
125
|
+
* per removal (re-armed if the party is re-admitted; a resume also rebuilds the
|
|
126
|
+
* enforcer and may legitimately re-fire), and only for closed strands with the
|
|
127
|
+
* gate armed. `CadreNode`
|
|
128
|
+
* wires it to its `strand:revoked` event; nothing is stopped or torn down on
|
|
129
|
+
* this node's behalf — what to do about it is the app's call.
|
|
130
|
+
*/
|
|
131
|
+
onSelfRevoked?: (strandId: string) => void;
|
|
73
132
|
}
|
|
74
133
|
|
|
75
134
|
/**
|
|
76
|
-
* Volatile inputs re-resolved when resuming a quiesced strand
|
|
77
|
-
*
|
|
78
|
-
*
|
|
135
|
+
* Volatile inputs re-resolved when resuming a quiesced strand — the ones that can
|
|
136
|
+
* have moved since it last ran, and that the rebuilt libp2p node freezes again. Each
|
|
137
|
+
* one omitted keeps the value the retained launch config already holds, so a resume
|
|
138
|
+
* that passes nothing rebuilds the strand exactly as it last ran.
|
|
79
139
|
*/
|
|
80
140
|
export interface ResumeStrandOverrides {
|
|
81
|
-
/**
|
|
141
|
+
/**
|
|
142
|
+
* Freshly-resolved cohort discovery seed (multiaddr strings). Grows as peers are
|
|
143
|
+
* learned since the strand first launched.
|
|
144
|
+
*/
|
|
82
145
|
bootstrapNodes?: string[];
|
|
146
|
+
/**
|
|
147
|
+
* Freshly-read count of the machines serving this strand, for the repair yardstick
|
|
148
|
+
* (see {@link StartStrandConfig.servingMachines} — including why the party's
|
|
149
|
+
* enrolled-machine count is not it). Moves whenever a machine starts or stops
|
|
150
|
+
* serving the strand, which a hibernating strand does not otherwise notice. Nothing
|
|
151
|
+
* passes it today. Omitting it here does NOT mean "unknown" the way omitting it from
|
|
152
|
+
* {@link StartStrandConfig} does — it retains the last value; see `resumeStrand`.
|
|
153
|
+
*/
|
|
154
|
+
servingMachines?: number;
|
|
83
155
|
}
|
|
84
156
|
|
|
85
157
|
/**
|
|
@@ -162,7 +234,14 @@ export class StrandInstanceManager {
|
|
|
162
234
|
* with a fresh caught-up-peer memo, which is intended (a resumed node may have
|
|
163
235
|
* missed writes).
|
|
164
236
|
*/
|
|
165
|
-
private backfills: Map<string,
|
|
237
|
+
private backfills: Map<string, PeerJoinBackfill> = new Map();
|
|
238
|
+
/**
|
|
239
|
+
* The per-strand revoked-peer enforcer (closed strands only), keyed by strand
|
|
240
|
+
* id. Same lifecycle rationale as {@link backfills}: created in
|
|
241
|
+
* `buildStrandRuntime`, stopped and dropped in `releaseRuntime`, so quiesce →
|
|
242
|
+
* resume rebuilds it with a fresh (initially empty, fail-open) snapshot.
|
|
243
|
+
*/
|
|
244
|
+
private revocationEnforcers: Map<string, StrandRevocationEnforcer> = new Map();
|
|
166
245
|
/**
|
|
167
246
|
* The resolved (cache-wrapped) raw storage per strand id — the instance's OWN
|
|
168
247
|
* store, resolved once in `startStrand` and held until `stopStrand` disposes it.
|
|
@@ -235,6 +314,14 @@ export class StrandInstanceManager {
|
|
|
235
314
|
}
|
|
236
315
|
|
|
237
316
|
if (this.instances.has(strandId)) {
|
|
317
|
+
// Callers resolve founder-ness BEFORE reaching here (CadreNode.launchStrand) and
|
|
318
|
+
// honor a founder request on a tracked instance via foundExistingStrand — so a
|
|
319
|
+
// founder flag arriving at this early return against a non-founder retained
|
|
320
|
+
// config is a dropped bootstrap, and must never again be silent.
|
|
321
|
+
if (config.founder === true && this.launchConfigs.get(strandId)?.founder !== true) {
|
|
322
|
+
log('startStrand: strand %s is already running but was NOT launched as founder — ' +
|
|
323
|
+
'this early return DROPS the founder request; use foundExistingStrand', strandId);
|
|
324
|
+
}
|
|
238
325
|
log('Strand %s already running', strandId);
|
|
239
326
|
return this.instances.get(strandId)!;
|
|
240
327
|
}
|
|
@@ -309,8 +396,9 @@ export class StrandInstanceManager {
|
|
|
309
396
|
* Build (or rebuild) the libp2p node + StrandDatabase for an instance and
|
|
310
397
|
* attach them, transitioning it to `active`. Shared by `startStrand` (fresh
|
|
311
398
|
* launch) and `resumeStrand` (rehydrating a quiesced instance). Reads all
|
|
312
|
-
* volatile inputs (bootstrapNodes, network, profile,
|
|
313
|
-
* from `config`, so the caller controls the
|
|
399
|
+
* volatile inputs (bootstrapNodes, servingMachines, network, profile,
|
|
400
|
+
* privateKey, sApp config) from `config`, so the caller controls the
|
|
401
|
+
* cohort-derived values. Storage is the
|
|
314
402
|
* one input it does NOT re-read from `config`: that belongs to the instance and
|
|
315
403
|
* comes from `strandStorages`.
|
|
316
404
|
*/
|
|
@@ -340,9 +428,65 @@ export class StrandInstanceManager {
|
|
|
340
428
|
// strand node's protocol ids are namespaced `/optimystic/strand-<id>/…`, so a
|
|
341
429
|
// relay dialed from inside `libp2p.start()` is never in the strand's cohort and
|
|
342
430
|
// cannot refuse its database bring-up. See `relay-addrs.ts`.
|
|
343
|
-
|
|
431
|
+
//
|
|
432
|
+
// The strand-node VIEW of the machine's one `NetworkConfig`, not the control
|
|
433
|
+
// node's resolution: fixed direct listen ports become ephemeral (two nodes cannot
|
|
434
|
+
// bind one port) and the announce config is dropped (it names the control node's
|
|
435
|
+
// address). See `strand-network-config.ts` for both, and for the tradeoff.
|
|
436
|
+
const addrOptions = strandNodeAddrs(config.network);
|
|
437
|
+
|
|
438
|
+
// The CLOSED-strand revoked-peer gate (see strand-revocation-enforcer.ts).
|
|
439
|
+
// Constructed BEFORE the libp2p node because the node's options embed its
|
|
440
|
+
// predicates; its deny set starts empty (admit everything — fail-open, same
|
|
441
|
+
// posture as bring-up everywhere here) and is first populated after the
|
|
442
|
+
// strand database initializes below. Registered in the map immediately so
|
|
443
|
+
// the failure rollback (releaseRuntime) tears it down like every other
|
|
444
|
+
// runtime component. Open strands skip it entirely: their Member/MemberPeer
|
|
445
|
+
// tables are empty by schema (`OnlyClosed`), so there is nothing to derive
|
|
446
|
+
// a deny set from — arming would be pointless polling.
|
|
447
|
+
const revocationEnforcer =
|
|
448
|
+
config.strandRow.Type === 'c' && config.revocationEnforcement?.enabled !== false
|
|
449
|
+
? new StrandRevocationEnforcer({
|
|
450
|
+
label: strandId,
|
|
451
|
+
readRows: () => {
|
|
452
|
+
const database = instance.database;
|
|
453
|
+
if (!database) {
|
|
454
|
+
throw new Error(`strand ${strandId} has no live database`);
|
|
455
|
+
}
|
|
456
|
+
return readStrandRevocationRows(database.getDatabase());
|
|
457
|
+
},
|
|
458
|
+
// Read per sweep, never captured: this closure is built BEFORE the
|
|
459
|
+
// libp2p node exists (its options embed the enforcer's predicates),
|
|
460
|
+
// and `releaseRuntime` clears the field again on quiesce. Undefined
|
|
461
|
+
// therefore means "no transport right now", which the enforcer
|
|
462
|
+
// treats as "nothing to tear down".
|
|
463
|
+
getNetwork: () => instance.libp2pNode,
|
|
464
|
+
onSelfRevoked: () => config.onSelfRevoked?.(strandId)
|
|
465
|
+
}, config.revocationEnforcement)
|
|
466
|
+
: undefined;
|
|
467
|
+
if (revocationEnforcer) {
|
|
468
|
+
this.revocationEnforcers.set(strandId, revocationEnforcer);
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
// Closed strands compose revoked-peer denial onto the caller's gater and arm
|
|
472
|
+
// the fail-closed per-stream gate; open strands keep the raw configured gater
|
|
473
|
+
// (their peers are legitimately cross-party and nothing is ever revoked).
|
|
474
|
+
const revocationGateOptions = revocationEnforcer
|
|
475
|
+
? {
|
|
476
|
+
connectionGater: createRevocationConnectionGater(revocationEnforcer, config.network?.connectionGater),
|
|
477
|
+
authorizeInboundStream: (remotePeerId: string, protocol: string) =>
|
|
478
|
+
revocationEnforcer.authorizeStream(remotePeerId, protocol)
|
|
479
|
+
}
|
|
480
|
+
: config.network?.connectionGater
|
|
481
|
+
? { connectionGater: config.network.connectionGater }
|
|
482
|
+
: {};
|
|
344
483
|
|
|
345
484
|
try {
|
|
485
|
+
// Bound once: the breadth is also the ceiling on the repair yardstick below, and the
|
|
486
|
+
// two must be derived from the same resolution. Inside the `try` deliberately — a
|
|
487
|
+
// rejected clusterSize is a build failure that runs the same cleanup as any other.
|
|
488
|
+
const strandClusterSize = resolveStrandClusterSize(config.clusterSize);
|
|
489
|
+
|
|
346
490
|
let t0 = performance.now();
|
|
347
491
|
const node = await createLibp2pNode({
|
|
348
492
|
port: 0, // Random port
|
|
@@ -351,43 +495,38 @@ export class StrandInstanceManager {
|
|
|
351
495
|
storage: strandStorage,
|
|
352
496
|
fretProfile: config.profile === 'storage' ? 'core' : 'edge',
|
|
353
497
|
relay: enableRelay,
|
|
354
|
-
clusterSize:
|
|
498
|
+
clusterSize: strandClusterSize,
|
|
355
499
|
// Deliberately NOT CONTROL_CLUSTER_POLICY: a strand is application data with its own
|
|
356
500
|
// breadth reasoning, and the shape match with the control policy is a coincidence.
|
|
357
|
-
|
|
501
|
+
//
|
|
502
|
+
// The builder declares this node's block-repair corroboration yardstick from the
|
|
503
|
+
// count of machines SERVING this strand, capped at the breadth above (a block
|
|
504
|
+
// never lives on more machines than the cohort is wide). Given no count — the
|
|
505
|
+
// production path today, since no per-strand serving count exists yet — it
|
|
506
|
+
// returns the frozen STRAND_CLUSTER_POLICY itself, declaring nothing. Resolved
|
|
507
|
+
// HERE rather than at `startStrand`, so a wake from hibernation would pick up a
|
|
508
|
+
// serving set that changed while the strand slept.
|
|
509
|
+
clusterPolicy: strandClusterPolicy(strandClusterSize, config.servingMachines),
|
|
358
510
|
arachnode: {
|
|
359
511
|
enableRingZulu: config.profile === 'storage'
|
|
360
512
|
},
|
|
361
513
|
...(config.privateKey && { privateKey: config.privateKey }),
|
|
362
514
|
...(config.network?.transports && { transports: config.network.transports }),
|
|
363
|
-
//
|
|
364
|
-
//
|
|
365
|
-
//
|
|
366
|
-
//
|
|
367
|
-
//
|
|
368
|
-
//
|
|
369
|
-
//
|
|
370
|
-
//
|
|
371
|
-
//
|
|
372
|
-
//
|
|
373
|
-
|
|
374
|
-
//
|
|
375
|
-
|
|
376
|
-
//
|
|
377
|
-
|
|
378
|
-
// addresses.
|
|
379
|
-
//
|
|
380
|
-
// NOTE: an announce addr naming a PORT therefore names the control node's port,
|
|
381
|
-
// not this strand node's — the same second-node-on-one-host problem the listen
|
|
382
|
-
// NOTE above describes, and the reason both belong to one fix. It is currently
|
|
383
|
-
// unreachable: naming a fixed public port means configuring a fixed
|
|
384
|
-
// `listenAddrs` port, and that collides at bind time before any strand node gets
|
|
385
|
-
// far enough to advertise anything. Whatever gives strand nodes their own listen
|
|
386
|
-
// port has to give them their own announce addrs in the same pass, or a strand
|
|
387
|
-
// will advertise an address that reaches the control node. Tracked as an arm of
|
|
388
|
-
// `tickets/backlog/strand-network-nat-relay-reachability.md`.
|
|
389
|
-
...resolveAnnounceAddrs(config.network),
|
|
390
|
-
...(config.network?.connectionGater && { connectionGater: config.network.connectionGater })
|
|
515
|
+
// Listen entries plus the WebSocket transport switch they imply — a strand node
|
|
516
|
+
// announces nothing the operator configured (`strand-network-config.ts`), and
|
|
517
|
+
// spreads AFTER `transports` above because the switch is a no-op whenever the
|
|
518
|
+
// embedder supplied its own factories. An inherited configured `/p2p-circuit` entry is
|
|
519
|
+
// deliberate and survives the derivation untouched: it is what gives a NAT'd
|
|
520
|
+
// strand node a reachable relay slot, and it works because the launch path
|
|
521
|
+
// announces this strand's derived peerId to the relay first (delegate admission;
|
|
522
|
+
// see cadre-node.ts). Those circuit entries come either from a hand-written
|
|
523
|
+
// `network.listenAddrs` or from `network.relayAddrs`, which the resolution folds
|
|
524
|
+
// into the same list on the configured route this call takes (see above).
|
|
525
|
+
...addrOptions,
|
|
526
|
+
// The raw configured gater (open strands), or the revocation-composed
|
|
527
|
+
// gater plus the fail-closed per-stream revoked-peer gate (closed
|
|
528
|
+
// strands) — resolved above, before the try.
|
|
529
|
+
...revocationGateOptions
|
|
391
530
|
}) as Libp2pNodeWithRepo;
|
|
392
531
|
timing('[buildStrandRuntime:%s] createLibp2pNode: %dms', strandId, Math.round(performance.now() - t0));
|
|
393
532
|
|
|
@@ -414,23 +553,32 @@ export class StrandInstanceManager {
|
|
|
414
553
|
await strandDb.initialize();
|
|
415
554
|
timing('[buildStrandRuntime:%s] strandDatabase.initialize: %dms', strandId, Math.round(performance.now() - t0));
|
|
416
555
|
|
|
556
|
+
// Arm the revoked-peer deny-set poll now that the database it reads
|
|
557
|
+
// exists. start() kicks an immediate refresh WITHOUT awaiting it —
|
|
558
|
+
// bring-up and resume are never blocked on a membership read; until the
|
|
559
|
+
// first read lands the empty snapshot admits everyone (fail-open).
|
|
560
|
+
revocationEnforcer?.start();
|
|
561
|
+
|
|
417
562
|
// Peer-join block catch-up: push this strand's own blocks to each newly
|
|
418
563
|
// connected peer, so a machine that joined after blocks were committed
|
|
419
564
|
// still ends up physically holding them (without per-strand storage there
|
|
420
|
-
// is nothing to copy). Armed for EVERY stored strand: `
|
|
565
|
+
// is nothing to copy). Armed for EVERY stored strand: `PeerJoinBackfill`
|
|
421
566
|
// only does work when the strand's libp2p node reports a peer connection,
|
|
422
567
|
// so on a device that is genuinely alone it is inert, and arming it at
|
|
423
568
|
// launch is what closes the "founded alone, never replicates" hole — a
|
|
424
569
|
// peer that joins later gets the founder's blocks without any relaunch.
|
|
570
|
+
// No `authorizePeer` gate, deliberately — see the module comment in
|
|
571
|
+
// peer-join-backfill.ts for the strand-side argument (and why the control
|
|
572
|
+
// network, which DOES gate, is different).
|
|
425
573
|
//
|
|
426
|
-
// NOTE: cost is one
|
|
574
|
+
// NOTE: cost is one PeerJoinBackfill object + one `connection:open` listener
|
|
427
575
|
// per running strand — linear in strand count, negligible at the handful a
|
|
428
576
|
// device or host runs today. If a node ever hosts strands by the hundred,
|
|
429
577
|
// move to one shared listener that dispatches by strand id.
|
|
430
578
|
if (strandStorage && config.backfill?.enabled !== false) {
|
|
431
579
|
if (node.keyNetwork) {
|
|
432
|
-
const backfill = new
|
|
433
|
-
strandId,
|
|
580
|
+
const backfill = new PeerJoinBackfill({
|
|
581
|
+
label: strandId,
|
|
434
582
|
libp2p: node,
|
|
435
583
|
peerNetwork: node.keyNetwork,
|
|
436
584
|
storage: strandStorage,
|
|
@@ -478,6 +626,13 @@ export class StrandInstanceManager {
|
|
|
478
626
|
backfill.stop();
|
|
479
627
|
this.backfills.delete(instance.strandId);
|
|
480
628
|
}
|
|
629
|
+
// The revoked-peer enforcer goes with the runtime it gated: a resume
|
|
630
|
+
// rebuilds it with a fresh (initially empty, fail-open) snapshot.
|
|
631
|
+
const revocationEnforcer = this.revocationEnforcers.get(instance.strandId);
|
|
632
|
+
if (revocationEnforcer) {
|
|
633
|
+
revocationEnforcer.stop();
|
|
634
|
+
this.revocationEnforcers.delete(instance.strandId);
|
|
635
|
+
}
|
|
481
636
|
if (instance.database) {
|
|
482
637
|
await instance.database.close();
|
|
483
638
|
instance.database = undefined;
|
|
@@ -513,6 +668,32 @@ export class StrandInstanceManager {
|
|
|
513
668
|
}
|
|
514
669
|
}
|
|
515
670
|
|
|
671
|
+
/**
|
|
672
|
+
* Refresh a strand's revoked-peer deny set NOW — and, with it, run the
|
|
673
|
+
* teardown sweep that hangs up every connected revoked peer.
|
|
674
|
+
*
|
|
675
|
+
* The gate is otherwise poll-driven (default
|
|
676
|
+
* `DEFAULT_REVOCATION_POLL_INTERVAL_MS`), because neither a replicated
|
|
677
|
+
* revocation nor a local `revokeMember`/`leaveStrand` raises anything this
|
|
678
|
+
* runtime can hook. A caller that just wrote a revocation against the
|
|
679
|
+
* strand's database should follow it with this call, which makes the cut
|
|
680
|
+
* immediate instead of waiting out the interval.
|
|
681
|
+
*
|
|
682
|
+
* Quiet no-op — matching `quiesceStrand`'s posture — when the strand is not
|
|
683
|
+
* tracked, is quiesced, is open, or has the gate disabled: in every one of
|
|
684
|
+
* those cases there is no enforcer to refresh. Never rejects (the enforcer's
|
|
685
|
+
* refresh contains its own failures), and resolves only once the sweep has
|
|
686
|
+
* finished.
|
|
687
|
+
*/
|
|
688
|
+
async refreshRevocationEnforcement(strandId: string): Promise<void> {
|
|
689
|
+
const enforcer = this.revocationEnforcers.get(strandId);
|
|
690
|
+
if (!enforcer) {
|
|
691
|
+
log('refreshRevocationEnforcement: strand %s has no armed revocation enforcer', strandId);
|
|
692
|
+
return;
|
|
693
|
+
}
|
|
694
|
+
await enforcer.refresh();
|
|
695
|
+
}
|
|
696
|
+
|
|
516
697
|
/**
|
|
517
698
|
* Quiesce a strand: release its strand-network resources (stop the libp2p node,
|
|
518
699
|
* close the StrandDatabase) while RETAINING the instance record — identity,
|
|
@@ -541,9 +722,9 @@ export class StrandInstanceManager {
|
|
|
541
722
|
* Resume a previously-quiesced strand: rebuild its libp2p node + StrandDatabase
|
|
542
723
|
* from the retained launch config and re-attach them, transitioning it back to
|
|
543
724
|
* `active`. `overrides` re-applies volatile inputs that may have changed since
|
|
544
|
-
* launch (the cohort `bootstrapNodes` seed
|
|
545
|
-
* a later resume reuses the latest
|
|
546
|
-
* if it is already running.
|
|
725
|
+
* launch (the cohort `bootstrapNodes` seed and the strand's `servingMachines`
|
|
726
|
+
* count) and updates the retained config so a later resume reuses the latest
|
|
727
|
+
* values. Returns the live instance unchanged if it is already running.
|
|
547
728
|
*/
|
|
548
729
|
async resumeStrand(strandId: string, overrides?: ResumeStrandOverrides): Promise<StrandInstance> {
|
|
549
730
|
if (this.stopping) {
|
|
@@ -566,9 +747,20 @@ export class StrandInstanceManager {
|
|
|
566
747
|
const tTotal = performance.now();
|
|
567
748
|
|
|
568
749
|
// Re-apply volatile inputs and persist them so a subsequent resume reuses them.
|
|
750
|
+
// Each `??` matters: a resume that passes no override must keep the retained value,
|
|
751
|
+
// and a resume that passes one must leave it retained for the next resume — otherwise
|
|
752
|
+
// a later no-override wake silently reverts to whatever launch time saw.
|
|
569
753
|
const resumeConfig: StartStrandConfig = {
|
|
570
754
|
...launchConfig,
|
|
571
|
-
bootstrapNodes: overrides?.bootstrapNodes ?? launchConfig.bootstrapNodes
|
|
755
|
+
bootstrapNodes: overrides?.bootstrapNodes ?? launchConfig.bootstrapNodes,
|
|
756
|
+
// NOTE: `??` retains, so an override can raise or lower the count but cannot CLEAR it
|
|
757
|
+
// back to "this node no longer knows" — the direction that would declare nothing. Moot
|
|
758
|
+
// while nothing feeds `servingMachines` at all; if a source lands
|
|
759
|
+
// (`backlog/feat-strand-yardstick-from-serving-machines`) that can legitimately lose the
|
|
760
|
+
// count — a strand whose member rows became unreadable — this merge must gain an explicit
|
|
761
|
+
// clear rather than silently declaring a stale number over a serving set it can no longer
|
|
762
|
+
// see. Same applies to `bootstrapNodes` above, where a stale seed is harmless.
|
|
763
|
+
servingMachines: overrides?.servingMachines ?? launchConfig.servingMachines
|
|
572
764
|
};
|
|
573
765
|
this.launchConfigs.set(strandId, resumeConfig);
|
|
574
766
|
|
|
@@ -586,6 +778,77 @@ export class StrandInstanceManager {
|
|
|
586
778
|
}
|
|
587
779
|
}
|
|
588
780
|
|
|
781
|
+
/**
|
|
782
|
+
* Honor a founder request against an ALREADY-TRACKED strand — the seam that
|
|
783
|
+
* closes the "whoever launches first decides whether the bootstrap runs" gap:
|
|
784
|
+
* an instance first launched as a joiner (an app's own attach, or a watcher
|
|
785
|
+
* poll winning the launch race) used to swallow a later founder request
|
|
786
|
+
* silently, leaving the strand active with no `Strand.Header`.
|
|
787
|
+
*
|
|
788
|
+
* Flips the RETAINED launch config's `founder` to true, so every later
|
|
789
|
+
* quiesce → resume rebuild founds as well (the bootstrap is insert-if-absent —
|
|
790
|
+
* {@link StrandDatabase.ensureFounderBootstrap} — so re-running it per rebuild
|
|
791
|
+
* writes nothing twice), and runs the bootstrap against the live database now.
|
|
792
|
+
*
|
|
793
|
+
* @returns how the request resolved:
|
|
794
|
+
* - `'already-founder'` — the retained config already founds; nothing to do.
|
|
795
|
+
* - `'bootstrapped'` — config flipped and the live database ran the bootstrap.
|
|
796
|
+
* - `'needs-resume'` — config flipped, but the instance is quiesced (no live
|
|
797
|
+
* database), so the bootstrap could not run here: the CALLER must wake the
|
|
798
|
+
* strand (`CadreNode.wakeStrand`, which owns the hibernation bookkeeping this
|
|
799
|
+
* manager does not) so the rebuild — which now founds — runs it.
|
|
800
|
+
* @throws when the strand is not tracked — this seam exists only for the
|
|
801
|
+
* tracked-instance launch path; an untracked id is a caller bug.
|
|
802
|
+
*/
|
|
803
|
+
async foundExistingStrand(strandId: string): Promise<'already-founder' | 'bootstrapped' | 'needs-resume'> {
|
|
804
|
+
const instance = this.instances.get(strandId);
|
|
805
|
+
const config = this.launchConfigs.get(strandId);
|
|
806
|
+
if (!instance || !config) {
|
|
807
|
+
throw new Error(`Cannot found strand ${strandId}: not tracked`);
|
|
808
|
+
}
|
|
809
|
+
if (config.founder === true) {
|
|
810
|
+
return 'already-founder';
|
|
811
|
+
}
|
|
812
|
+
// A fresh object rather than mutating in place: startStrand retains the CALLER'S
|
|
813
|
+
// config object, which is not ours to rewrite.
|
|
814
|
+
this.launchConfigs.set(strandId, { ...config, founder: true });
|
|
815
|
+
if (!instance.database) {
|
|
816
|
+
return 'needs-resume';
|
|
817
|
+
}
|
|
818
|
+
await this.ensureFounderBootstrap(strandId);
|
|
819
|
+
return 'bootstrapped';
|
|
820
|
+
}
|
|
821
|
+
|
|
822
|
+
/**
|
|
823
|
+
* Run the (idempotent) founder bootstrap against a tracked strand's LIVE
|
|
824
|
+
* database, independently of what the retained launch config says.
|
|
825
|
+
*
|
|
826
|
+
* Separate from {@link foundExistingStrand} because a caller that resolved
|
|
827
|
+
* `'needs-resume'` and woke the strand must not assume the wake's own rebuild
|
|
828
|
+
* founded it: `HibernationManager` COALESCES wakes, so a wake already in flight
|
|
829
|
+
* when the config flipped had already read the PRE-flip config and rebuilt as a
|
|
830
|
+
* joiner. Re-running the bootstrap costs one insert-if-absent probe per table
|
|
831
|
+
* and is the only thing that makes "founding resolves once the Header is
|
|
832
|
+
* written" true on that path.
|
|
833
|
+
*
|
|
834
|
+
* @throws when the strand is not tracked, or is still quiesced (no live
|
|
835
|
+
* database) — both mean the bootstrap did NOT run, which a founder request
|
|
836
|
+
* must never swallow.
|
|
837
|
+
*/
|
|
838
|
+
async ensureFounderBootstrap(strandId: string): Promise<void> {
|
|
839
|
+
const instance = this.instances.get(strandId);
|
|
840
|
+
if (!instance) {
|
|
841
|
+
throw new Error(`Cannot run the founder bootstrap for strand ${strandId}: not tracked`);
|
|
842
|
+
}
|
|
843
|
+
if (!instance.database) {
|
|
844
|
+
throw new Error(
|
|
845
|
+
`Cannot run the founder bootstrap for strand ${strandId}: it is quiesced, so there ` +
|
|
846
|
+
'is no live database to write to — resume it first.'
|
|
847
|
+
);
|
|
848
|
+
}
|
|
849
|
+
await instance.database.ensureFounderBootstrap();
|
|
850
|
+
}
|
|
851
|
+
|
|
589
852
|
/**
|
|
590
853
|
* Stop a strand instance
|
|
591
854
|
*/
|
|
@@ -852,6 +852,13 @@ export interface RevokeMemberParams {
|
|
|
852
852
|
* (`Member.MinOneMember`, a local-count floor with the cross-node caveat its
|
|
853
853
|
* schema NOTE states).
|
|
854
854
|
*
|
|
855
|
+
* Callers running inside a cadre runtime should follow this write with
|
|
856
|
+
* `CadreNode.refreshRevocationEnforcement(strandId)`. The removed party's
|
|
857
|
+
* machines are refused at the network layer by the strand's revoked-peer gate
|
|
858
|
+
* (`strand-revocation-enforcer.ts`), which polls for membership changes because
|
|
859
|
+
* this bare `Database` handle raises no event the runtime can hook — so without
|
|
860
|
+
* that call the cut lands up to one poll interval (default 30 s) later.
|
|
861
|
+
*
|
|
855
862
|
* @param db - The closed strand's database.
|
|
856
863
|
* @param params - The revoking manager keypair and the target member key.
|
|
857
864
|
* @throws If `Member.Authorized` rejects (a non-manager or same-transaction
|
|
@@ -907,6 +914,11 @@ export interface LeaveStrandParams {
|
|
|
907
914
|
* The same floors as {@link revokeMember} apply: a manager must resign first
|
|
908
915
|
* (`NotAManager`) and the last member cannot leave (`MinOneMember`).
|
|
909
916
|
*
|
|
917
|
+
* Like {@link revokeMember}, a caller inside a cadre runtime should follow this
|
|
918
|
+
* with `CadreNode.refreshRevocationEnforcement(strandId)` — here it is what
|
|
919
|
+
* makes the departing node notice its own departure promptly (the gate emits
|
|
920
|
+
* `strand:revoked` when it finds this node's own peer id revoked).
|
|
921
|
+
*
|
|
910
922
|
* @param db - The closed strand's database.
|
|
911
923
|
* @param params - The departing member's own keypair.
|
|
912
924
|
* @throws If `Member.Authorized`, `Member.NotAManager`, or `Member.MinOneMember`
|
|
@@ -1207,7 +1219,17 @@ async function deleteOwnMemberPeer(db: Database, params: RemoveOwnPeerParams, st
|
|
|
1207
1219
|
await insertRevocation(db, 'MemberPeer', stampId, memberKeyPair);
|
|
1208
1220
|
}
|
|
1209
1221
|
|
|
1210
|
-
/**
|
|
1222
|
+
/**
|
|
1223
|
+
* The manager branch: a manager signs the manager-remove-tagged digest + tombstone.
|
|
1224
|
+
*
|
|
1225
|
+
* NOTE: clearing an ORPHANED binding (one whose member was already removed) also
|
|
1226
|
+
* forgets the NETWORK DENIAL of that peer id. The orphaned `MemberPeer` row IS
|
|
1227
|
+
* the record the strand revoked-peer gate derives its deny set from
|
|
1228
|
+
* (`strand-revocation-enforcer.ts`) — the `Strand.Revocation` tombstone keeps
|
|
1229
|
+
* only the stamp, not the peer id — so once the orphan is cleared, remaining
|
|
1230
|
+
* members' nodes resume answering and dialing that peer. Only clear a binding
|
|
1231
|
+
* that should never have existed, or a peer that is truly gone.
|
|
1232
|
+
*/
|
|
1211
1233
|
async function deleteMemberPeerByManager(
|
|
1212
1234
|
db: Database,
|
|
1213
1235
|
params: RemoveMemberPeerByManagerParams,
|