@serfab/cadre-core 1.8.0 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cadre-node.d.ts +301 -121
- package/dist/cadre-node.js +588 -244
- package/dist/cadre-node.js.map +1 -1
- package/dist/control-authorization.d.ts +2 -2
- package/dist/control-authorization.js +1 -0
- package/dist/control-authorization.js.map +1 -1
- package/dist/control-cohort.d.ts +1 -1
- package/dist/control-cohort.js +1 -1
- package/dist/control-database.d.ts +105 -14
- package/dist/control-database.js +242 -73
- package/dist/control-database.js.map +1 -1
- package/dist/control-formation-recorder.d.ts +6 -6
- package/dist/control-formation-recorder.js +7 -8
- package/dist/control-formation-recorder.js.map +1 -1
- package/dist/control-schema.d.ts +1 -1
- package/dist/control-schema.js +90 -3
- package/dist/control-schema.js.map +1 -1
- package/dist/control-write-retry.d.ts +21 -15
- package/dist/control-write-retry.js +21 -15
- package/dist/control-write-retry.js.map +1 -1
- package/dist/index.d.ts +7 -7
- package/dist/index.js +16 -13
- package/dist/index.js.map +1 -1
- package/dist/invitation-bootstrap.d.ts +32 -0
- package/dist/invitation-bootstrap.js +30 -0
- package/dist/invitation-bootstrap.js.map +1 -0
- package/dist/link-budget.d.ts +204 -119
- package/dist/link-budget.js +208 -120
- package/dist/link-budget.js.map +1 -1
- package/dist/membership-connection-gater.d.ts +3 -3
- package/dist/membership-connection-gater.js +3 -3
- package/dist/node-local-snapshot.d.ts +6 -4
- package/dist/node-local-snapshot.js +7 -3
- package/dist/node-local-snapshot.js.map +1 -1
- package/dist/peer-dial.d.ts +18 -14
- package/dist/peer-dial.js +18 -14
- package/dist/peer-dial.js.map +1 -1
- package/dist/peer-join-backfill.d.ts +3 -12
- package/dist/peer-join-backfill.js +2 -3
- package/dist/peer-join-backfill.js.map +1 -1
- package/dist/pending-join-runner.d.ts +186 -0
- package/dist/pending-join-runner.js +649 -0
- package/dist/pending-join-runner.js.map +1 -0
- package/dist/push-fanout.d.ts +1 -1
- package/dist/push-fanout.js +1 -1
- package/dist/seed-bootstrap.d.ts +2 -2
- package/dist/seed-bootstrap.js +1 -1
- package/dist/strand-addr-protocol.js +2 -2
- package/dist/strand-formation-deadlines.d.ts +5 -5
- package/dist/strand-formation-deadlines.js +5 -5
- package/dist/strand-formation-manager.d.ts +18 -4
- package/dist/strand-formation-manager.js +48 -33
- package/dist/strand-formation-manager.js.map +1 -1
- package/dist/strand-formation-protocol.d.ts +71 -25
- package/dist/strand-formation-protocol.js +225 -51
- package/dist/strand-formation-protocol.js.map +1 -1
- package/dist/strand-formation-rejection.d.ts +94 -0
- package/dist/strand-formation-rejection.js +77 -0
- package/dist/strand-formation-rejection.js.map +1 -0
- package/dist/strand-fret-addrs.d.ts +54 -0
- package/dist/strand-fret-addrs.js +108 -0
- package/dist/strand-fret-addrs.js.map +1 -0
- package/dist/strand-instance-manager.d.ts +9 -38
- package/dist/strand-instance-manager.js +17 -74
- package/dist/strand-instance-manager.js.map +1 -1
- package/dist/strand-network-state-file.d.ts +12 -0
- package/dist/strand-network-state-file.js +38 -0
- package/dist/strand-network-state-file.js.map +1 -0
- package/dist/strand-network-state.d.ts +111 -0
- package/dist/strand-network-state.js +191 -0
- package/dist/strand-network-state.js.map +1 -0
- package/dist/strand-solicitation.d.ts +11 -3
- package/dist/strand-solicitation.js +9 -0
- package/dist/strand-solicitation.js.map +1 -1
- package/dist/types.d.ts +123 -40
- package/dist/types.js +5 -9
- package/dist/types.js.map +1 -1
- package/package.json +25 -24
- package/src/cadre-node.ts +637 -251
- package/src/control-authorization.ts +117 -116
- package/src/control-cohort.ts +1 -1
- package/src/control-database.ts +303 -82
- package/src/control-formation-recorder.ts +8 -9
- package/src/control-schema.ts +90 -3
- package/src/control-write-retry.ts +21 -15
- package/src/index.ts +36 -56
- package/src/invitation-bootstrap.ts +40 -0
- package/src/link-budget.ts +221 -120
- package/src/membership-connection-gater.ts +3 -3
- package/src/node-local-snapshot.ts +12 -7
- package/src/peer-dial.ts +18 -14
- package/src/peer-join-backfill.ts +5 -6
- package/src/pending-join-runner.ts +735 -0
- package/src/push-fanout.ts +1 -1
- package/src/seed-bootstrap.ts +2 -2
- package/src/strand-addr-protocol.ts +2 -2
- package/src/strand-formation-deadlines.ts +5 -5
- package/src/strand-formation-manager.ts +62 -37
- package/src/strand-formation-protocol.ts +1036 -803
- package/src/strand-formation-rejection.ts +129 -0
- package/src/strand-fret-addrs.ts +145 -0
- package/src/strand-instance-manager.ts +24 -97
- package/src/strand-network-state-file.ts +40 -0
- package/src/strand-network-state.ts +243 -0
- package/src/strand-solicitation.ts +17 -4
- package/src/types.ts +119 -40
- package/dist/strand-peer-book-file.d.ts +0 -12
- package/dist/strand-peer-book-file.js +0 -37
- package/dist/strand-peer-book-file.js.map +0 -1
- package/dist/strand-peer-book-protocol.d.ts +0 -168
- package/dist/strand-peer-book-protocol.js +0 -356
- package/dist/strand-peer-book-protocol.js.map +0 -1
- package/dist/strand-peer-book-swap.d.ts +0 -104
- package/dist/strand-peer-book-swap.js +0 -284
- package/dist/strand-peer-book-swap.js.map +0 -1
- package/dist/strand-peer-book.d.ts +0 -159
- package/dist/strand-peer-book.js +0 -361
- package/dist/strand-peer-book.js.map +0 -1
- package/dist/strand-peer-observer.d.ts +0 -88
- package/dist/strand-peer-observer.js +0 -180
- package/dist/strand-peer-observer.js.map +0 -1
- package/src/strand-peer-book-file.ts +0 -39
- package/src/strand-peer-book-protocol.ts +0 -466
- package/src/strand-peer-book-swap.ts +0 -338
- package/src/strand-peer-book.ts +0 -475
- package/src/strand-peer-observer.ts +0 -235
package/dist/cadre-node.js
CHANGED
|
@@ -17,10 +17,11 @@ import { DEFAULT_IDENTITY_KEY_ID } from './key-store.js';
|
|
|
17
17
|
import { loadOrCreateIdentityKey } from './identity-key.js';
|
|
18
18
|
import { MemoryTrustedOwnerStore } from './trusted-owner-store.js';
|
|
19
19
|
import { MemoryBootstrapPeerStore } from './bootstrap-peer-store.js';
|
|
20
|
-
import {
|
|
20
|
+
import { MemoryStrandNetworkStateStore } from './strand-network-state.js';
|
|
21
21
|
import { MemoryEnrolledMachineStore } from './enrolled-machine-store.js';
|
|
22
22
|
import { JoinedStrandSession, KeyStoreJoinedStrandStore, MemoryJoinedStrandStore } from './joined-strand-store.js';
|
|
23
23
|
import { mergePeerAddrs, groupAddrsByPeerId } from './peer-addr-book.js';
|
|
24
|
+
import { strandFretPeerAddrs } from './strand-fret-addrs.js';
|
|
24
25
|
import { verifyCadrePeerVoucher } from './peer-authorization.js';
|
|
25
26
|
import { ed25519PublicKeyB64FromPeerId } from './seed-bootstrap.js';
|
|
26
27
|
import { signPeerRecord, verifyPeerRecordSignature, isPeerRecordFresh, orderSignalingFirst, isSignalingAddr, trailingPeerId, withTrailingPeerId, currentMemberTrustPolicy, DEFAULT_PEER_RECORD_MAX_AGE_MS, DEFAULT_PEER_RECORD_HEARTBEAT_MS } from './peer-record.js';
|
|
@@ -31,12 +32,16 @@ import { PeerJoinBackfill } from './peer-join-backfill.js';
|
|
|
31
32
|
import { deriveCohortMembers } from './strand-cohort.js';
|
|
32
33
|
import { selectControlCohortDials, DEFAULT_CONTROL_COHORT_RECONCILE_MS, DEFAULT_CONTROL_COHORT_TARGET_DEGREE } from './control-cohort.js';
|
|
33
34
|
import { dialPeerAddrs, SelfRelayOnlyError, CONTROL_COHORT_DIAL_ADDRESS_ATTEMPTS } from './peer-dial.js';
|
|
34
|
-
import { ADMISSION_DECISION_TIMEOUT_MS, declaredCohortReadDeadlineMs, peerJoinPushBudget, relayAdmissionReserveDeadlineMs, relayReservationBudgetMs, relayedDialBudgetMs, resolveLinkRoundTripMs } from './link-budget.js';
|
|
35
|
+
import { ADMISSION_DECISION_TIMEOUT_MS, declaredCohortReadDeadlineMs, optimysticDialLimits, peerJoinPushBudget, relayAdmissionReserveDeadlineMs, relayReservationBudgetMs, relayedDialBudgetMs, resolveLinkRoundTripMs } from './link-budget.js';
|
|
35
36
|
import { EnrollmentService } from './enrollment.js';
|
|
36
37
|
import { HibernationManager } from './hibernation-manager.js';
|
|
37
|
-
import { ControlDatabase, isStrandIdConflict } from './control-database.js';
|
|
38
|
+
import { ControlDatabase, isPendingJoinConflict, isStrandIdConflict, pendingJoinId } from './control-database.js';
|
|
39
|
+
import { FormationPostApprovalError } from './strand-formation-rejection.js';
|
|
40
|
+
import { PendingJoinRunner, membershipInvitesToStage, parseStoredDisclosure, requestedPendingJoin } from './pending-join-runner.js';
|
|
38
41
|
import { SeedBootstrapService } from './seed-bootstrap.js';
|
|
39
42
|
import { StrandSolicitationService } from './strand-solicitation.js';
|
|
43
|
+
import { ControlFormationUsageRecorder } from './control-formation-recorder.js';
|
|
44
|
+
import { selectInvitationSiblingAddrs } from './invitation-bootstrap.js';
|
|
40
45
|
import { createMembershipConnectionGater, DEFAULT_ENROLLMENT_WINDOW_MS, UnauthorizedReservationBudget } from './membership-connection-gater.js';
|
|
41
46
|
import { StrandWakeService, dialWake } from './strand-wake-protocol.js';
|
|
42
47
|
import { StrandAddrService, collectStrandAddrs } from './strand-addr-protocol.js';
|
|
@@ -98,8 +103,8 @@ function siblingAnswered(outcome) {
|
|
|
98
103
|
* `primary` followed by the entries of `extra` it does not already contain, de-duplicated
|
|
99
104
|
* and order-preserving.
|
|
100
105
|
*
|
|
101
|
-
* Used
|
|
102
|
-
* freshly-RPC'd sibling answers and the
|
|
106
|
+
* Used where a strand's discovery seed unions two sources of address strings — the
|
|
107
|
+
* freshly-RPC'd sibling answers and the addresses the strand's formation carried. The
|
|
103
108
|
* caller passes the FRESHER source as `primary`, so a stale entry can only ever be
|
|
104
109
|
* appended, never promoted ahead of a current one, and each source keeps whatever
|
|
105
110
|
* signaling-first ordering it arrived with.
|
|
@@ -249,6 +254,15 @@ export class CadreNode {
|
|
|
249
254
|
this.strandWatcher = null;
|
|
250
255
|
this.seedBootstrapService = null;
|
|
251
256
|
this.strandSolicitationService = null;
|
|
257
|
+
/**
|
|
258
|
+
* The service whose formation handler is registered on the control node. Trails
|
|
259
|
+
* {@link strandSolicitationService} while {@link initializeStrandSolicitation} swaps a new
|
|
260
|
+
* one in: the field is set at once so concurrent callers reuse it, the handler only when
|
|
261
|
+
* its queued swap runs.
|
|
262
|
+
*/
|
|
263
|
+
this.registeredSolicitation = null;
|
|
264
|
+
/** Tail of the queued formation-handler swaps ({@link swapFormationResponder}). */
|
|
265
|
+
this.solicitationSwaps = Promise.resolve();
|
|
252
266
|
this.strandWakeService = null;
|
|
253
267
|
/**
|
|
254
268
|
* Control-network strand-address responder. Answers a co-cadre sibling's
|
|
@@ -438,36 +452,54 @@ export class CadreNode {
|
|
|
438
452
|
*/
|
|
439
453
|
this.strandAddrAskDueAt = new Map();
|
|
440
454
|
/**
|
|
441
|
-
*
|
|
442
|
-
*
|
|
443
|
-
*
|
|
444
|
-
*
|
|
445
|
-
*
|
|
446
|
-
*
|
|
447
|
-
* store as `StartStrandConfig.strandPeerBook`) files the node's own signed entry
|
|
448
|
-
* plus every signed entry a strand peer sends it — the only writer whose entries
|
|
449
|
-
* carry a signature, and the only one that can refresh a peer's addresses without
|
|
450
|
-
* meeting that peer. Read by {@link resolveCohortSeed} (launch + hibernation resume) and
|
|
451
|
-
* re-merged by every {@link refreshStrandPeerAddrs} pass, which is what keeps the
|
|
452
|
-
* entries alive past the peerStore's one-hour address expiry.
|
|
455
|
+
* The strand-network addresses a formation carried back
|
|
456
|
+
* (`FormationResultMessage.strandAddrs`), by strand id: the responder's live strand
|
|
457
|
+
* node addresses, kept so the strand's first attach has something to dial. Written by
|
|
458
|
+
* {@link recordFormationStrandAddrs}, read by {@link resolveCohortSeed} (launch and
|
|
459
|
+
* hibernation resume) and by every refresh pass for the peers the strand node's FRET
|
|
460
|
+
* table holds no record for yet ({@link remergeUnrecordedFormationAddrs}).
|
|
453
461
|
*
|
|
454
|
-
* This is the
|
|
462
|
+
* This is the only cross-party input a FIRST attach has. The strand-addr RPC that
|
|
455
463
|
* resolves a strand's addresses is membership-gated and answers own-party siblings
|
|
456
|
-
* only, so without
|
|
457
|
-
*
|
|
458
|
-
*
|
|
459
|
-
*
|
|
460
|
-
*
|
|
464
|
+
* only, so without it a joiner's seed for a two-party strand is empty and the mesh
|
|
465
|
+
* never forms. Every later launch has the strand node's saved network state as well
|
|
466
|
+
* ({@link strandNetworkStateStore}), which is what a restart re-finds the other party
|
|
467
|
+
* from (gotchoices/sereus#18).
|
|
468
|
+
*
|
|
469
|
+
* A re-formation replaces the strand's list. An entry survives a {@link stopStrand}
|
|
470
|
+
* and a stop()→start() of this node instance ({@link cleanup} leaves the map alone),
|
|
471
|
+
* and is dropped where the saved network state is: {@link unpublishStrand},
|
|
472
|
+
* {@link forgetJoinedStrand} and self-revocation.
|
|
473
|
+
*
|
|
474
|
+
* NOTE: accepted tradeoff — in memory only, so a process restart between `formStrand`
|
|
475
|
+
* and the strand's first launch loses the carried addresses, and the strand then
|
|
476
|
+
* launches with no cross-party seed until it is re-formed. Persisting the list on the
|
|
477
|
+
* joined-strand record was weighed and declined: it mixes address state into the join
|
|
478
|
+
* record, and needs an "only until the first run" rule to stop dead addresses being
|
|
479
|
+
* dialled on every launch. Every embedder launches straight after forming, and once
|
|
480
|
+
* the strand has run the saved network state covers every later restart. Revisit if an
|
|
481
|
+
* embedder forms and launches in separate sessions.
|
|
482
|
+
*/
|
|
483
|
+
this.formationStrandAddrs = new Map();
|
|
484
|
+
/**
|
|
485
|
+
* Node-local strand network state (see `strand-network-state.ts`): per strand, the
|
|
486
|
+
* state db-p2p saves for the strand's libp2p node — its FRET routing table with each
|
|
487
|
+
* peer's signed address record. Handed to every strand launch as
|
|
488
|
+
* `StartStrandConfig.networkState`; the strand node saves into it on every connection
|
|
489
|
+
* and re-imports it when it is next built, which is what gives a restarted strand
|
|
490
|
+
* node addresses for the peers it was talking to.
|
|
491
|
+
*
|
|
492
|
+
* Constructed (or adopted from `config.strandNetworkState.store`) by
|
|
493
|
+
* {@link initializeStrandNetworkStateStore} during {@link start}, deliberately NOT
|
|
461
494
|
* cleared by {@link cleanup}, so it survives a stop()→start() cycle of the same node
|
|
462
495
|
* instance — same lifecycle as {@link bootstrapPeerStore}.
|
|
463
496
|
*
|
|
464
|
-
*
|
|
465
|
-
* {@link
|
|
466
|
-
*
|
|
467
|
-
*
|
|
468
|
-
* bounded by time and by the strands this node runs rather than by its history.
|
|
497
|
+
* A strand's state survives a {@link stopStrand} and a hibernation quiesce, and is
|
|
498
|
+
* forgotten by {@link unpublishStrand}, {@link forgetJoinedStrand} and
|
|
499
|
+
* self-revocation. A strand detached because the watcher saw its row gone keeps its
|
|
500
|
+
* state (`NOTE:` on `PersistentStrandNetworkStateStore`).
|
|
469
501
|
*/
|
|
470
|
-
this.
|
|
502
|
+
this.strandNetworkStateStore = null;
|
|
471
503
|
/**
|
|
472
504
|
* PENDING strand membership invitations learned at formation, keyed by strandId — the
|
|
473
505
|
* single-use `Strand.Invite` credential a closed-strand formation result carried back
|
|
@@ -495,9 +527,23 @@ export class CadreNode {
|
|
|
495
527
|
* NOTE: entries live for the node's lifetime (one small pair per formed closed
|
|
496
528
|
* strand); the number of keys is bounded only by the strands this node has ever
|
|
497
529
|
* formed, not by time — if a node ever forms strands at scale, evict on
|
|
498
|
-
* `unpublishStrand` / `forgetJoinedStrand` as
|
|
530
|
+
* `unpublishStrand` / `forgetJoinedStrand` as {@link formationStrandAddrs} is.
|
|
499
531
|
*/
|
|
500
532
|
this.pendingMembershipInvites = new Map();
|
|
533
|
+
/**
|
|
534
|
+
* Invite keys of the membership invitations this process has staged, by
|
|
535
|
+
* {@link adoptFormationMembershipInvite} or from a `joined` `PendingJoin` row
|
|
536
|
+
* ({@link stageMembershipInvitesFromPendingJoins}), so one the reconciler has settled is not
|
|
537
|
+
* staged again by the next pending-join pass. Same lifetime as
|
|
538
|
+
* {@link pendingMembershipInvites}: a restarted process stages each live one once more, and the
|
|
539
|
+
* reconciler settles it again (spent, or the party already seated).
|
|
540
|
+
*/
|
|
541
|
+
this.stagedMembershipInviteKeys = new Set();
|
|
542
|
+
/**
|
|
543
|
+
* The pending-join retry loop (`pending-join-runner.ts`): built by {@link start}, stopped by
|
|
544
|
+
* {@link cleanup}. It acts only while this machine is an enrolled owner.
|
|
545
|
+
*/
|
|
546
|
+
this.pendingJoinRunner = null;
|
|
501
547
|
/**
|
|
502
548
|
* Founder launches refused because the strand was founded before the per-party
|
|
503
549
|
* identity split (`PreSplitStrandIdentityError`), keyed by strandId. Read by
|
|
@@ -878,8 +924,9 @@ export class CadreNode {
|
|
|
878
924
|
// beside the control rows. Before network bring-up, so a mis-scoped injected store
|
|
879
925
|
// fails closed like the two above.
|
|
880
926
|
this.initializeJoinedStrandStore();
|
|
881
|
-
//
|
|
882
|
-
|
|
927
|
+
// Each strand node's saved network state, which those joins are dialed from after
|
|
928
|
+
// a restart. Same placement, same reason.
|
|
929
|
+
this.initializeStrandNetworkStateStore();
|
|
883
930
|
// Read the party's enrolled-machine count out of its node-local record and
|
|
884
931
|
// capture it for buildControlNodeOptions below. This MUST precede
|
|
885
932
|
// createControlNode: Optimystic freezes the cluster policy when the node is
|
|
@@ -990,6 +1037,11 @@ export class CadreNode {
|
|
|
990
1037
|
onDelegateAnnounce: (announcer, strandId, delegate) => this.grantDelegateAdmission(announcer, strandId, delegate),
|
|
991
1038
|
});
|
|
992
1039
|
await this.strandAddrService.initialize(this.controlNode);
|
|
1040
|
+
// Answer strand formation for this party from now on. Every machine of a party
|
|
1041
|
+
// holds the replicated FormationInvite/FormationUsage rows, so an always-on one can
|
|
1042
|
+
// check a token for an inviter that is offline. After the control database is up,
|
|
1043
|
+
// so the responder never reads a database that cannot answer yet.
|
|
1044
|
+
await this.installDefaultFormationResponder();
|
|
993
1045
|
// Server-side push-wake fan-out: only when push is configured.
|
|
994
1046
|
if (this.config.push) {
|
|
995
1047
|
this.pushFanoutService = this.buildPushFanout(this.config.push);
|
|
@@ -1023,6 +1075,8 @@ export class CadreNode {
|
|
|
1023
1075
|
await this.driveControlRelayReservation();
|
|
1024
1076
|
// Schedule self-registration in background
|
|
1025
1077
|
this.scheduleSelfRegistration();
|
|
1078
|
+
// Last: its first pass may dial another party's machines.
|
|
1079
|
+
this.startPendingJoinRunner();
|
|
1026
1080
|
}
|
|
1027
1081
|
catch (error) {
|
|
1028
1082
|
log('Failed to start CadreNode: %o', error);
|
|
@@ -1249,25 +1303,25 @@ export class CadreNode {
|
|
|
1249
1303
|
this.joinedStrands = new JoinedStrandSession(store, this.createPartyJoinedStrandLedger());
|
|
1250
1304
|
}
|
|
1251
1305
|
/**
|
|
1252
|
-
* Construct (or adopt) the node-local strand
|
|
1253
|
-
* {@link
|
|
1306
|
+
* Construct (or adopt) the node-local strand network state (see
|
|
1307
|
+
* {@link strandNetworkStateStore}). Synchronous for the reason
|
|
1254
1308
|
* {@link initializeBootstrapPeerStore} is: an injected store has already loaded its
|
|
1255
1309
|
* persisted entries by the time it is handed in, and the in-memory fallback has
|
|
1256
1310
|
* nothing to load. The instance is kept across stop()→start(). A store scoped to a
|
|
1257
|
-
* different party is a configuration error, fail closed: another party's
|
|
1258
|
-
*
|
|
1311
|
+
* different party is a configuration error, fail closed: another party's routing
|
|
1312
|
+
* tables must never be imported into this node's strands.
|
|
1259
1313
|
*/
|
|
1260
|
-
|
|
1314
|
+
initializeStrandNetworkStateStore() {
|
|
1261
1315
|
const partyId = this.config.controlNetwork.partyId;
|
|
1262
|
-
if (this.
|
|
1316
|
+
if (this.strandNetworkStateStore) {
|
|
1263
1317
|
return;
|
|
1264
1318
|
}
|
|
1265
|
-
const store = this.config.
|
|
1319
|
+
const store = this.config.strandNetworkState?.store ?? new MemoryStrandNetworkStateStore(partyId);
|
|
1266
1320
|
if (store.partyId !== partyId) {
|
|
1267
|
-
throw new Error(`CadreNodeConfig:
|
|
1268
|
-
`but this node serves party ${partyId} — refusing to mix strand
|
|
1321
|
+
throw new Error(`CadreNodeConfig: strandNetworkState.store is scoped to party ${store.partyId}, ` +
|
|
1322
|
+
`but this node serves party ${partyId} — refusing to mix strand network state`);
|
|
1269
1323
|
}
|
|
1270
|
-
this.
|
|
1324
|
+
this.strandNetworkStateStore = store;
|
|
1271
1325
|
}
|
|
1272
1326
|
/**
|
|
1273
1327
|
* The node-local enrolled-machine store (null before {@link start}) — what this
|
|
@@ -1287,13 +1341,13 @@ export class CadreNode {
|
|
|
1287
1341
|
return this.bootstrapPeerStore;
|
|
1288
1342
|
}
|
|
1289
1343
|
/**
|
|
1290
|
-
* The node-local strand
|
|
1291
|
-
*
|
|
1292
|
-
* diagnostics and for a host that wants to show "
|
|
1293
|
-
*
|
|
1344
|
+
* The node-local strand network state (null before {@link start}) — per strand, the
|
|
1345
|
+
* state its strand node last saved and will re-import when next built. Exposed for
|
|
1346
|
+
* diagnostics and for a host that wants to show "what routing table would this
|
|
1347
|
+
* strand restart with?".
|
|
1294
1348
|
*/
|
|
1295
|
-
|
|
1296
|
-
return this.
|
|
1349
|
+
getStrandNetworkStateStore() {
|
|
1350
|
+
return this.strandNetworkStateStore;
|
|
1297
1351
|
}
|
|
1298
1352
|
/**
|
|
1299
1353
|
* The node-local trusted-owner anchor (null before {@link start}). This is
|
|
@@ -1573,11 +1627,15 @@ export class CadreNode {
|
|
|
1573
1627
|
// replaces the default whole (see DEFAULT_CONNECTION_MONITOR).
|
|
1574
1628
|
connectionMonitor: network?.connectionMonitor ?? DEFAULT_CONNECTION_MONITOR,
|
|
1575
1629
|
// The declared link every cadre budget is derived from, stated to Optimystic too so it
|
|
1576
|
-
// derives its own deadlines from the same number: its request
|
|
1577
|
-
//
|
|
1578
|
-
//
|
|
1579
|
-
//
|
|
1630
|
+
// derives its own deadlines from the same number: its request responses, its block
|
|
1631
|
+
// pushes, and libp2p's `inboundUpgradeTimeout`. Always stated, default included:
|
|
1632
|
+
// undeclared, Optimystic keeps LAN deadlines that cannot open a relayed connection at the
|
|
1633
|
+
// link sereus supports (`link-budget.ts`).
|
|
1580
1634
|
linkRoundTripMs: resolveLinkRoundTripMs(network?.linkRoundTripMs),
|
|
1635
|
+
// The three limits on opening a connection (libp2p's per-address and whole-dial limits,
|
|
1636
|
+
// and Optimystic's request dial): Optimystic's derivation from the same link plus the
|
|
1637
|
+
// relay's and the called machine's admission decisions, which run on this dial's clock.
|
|
1638
|
+
...optimysticDialLimits(network?.linkRoundTripMs),
|
|
1581
1639
|
// `{ wsPort }` when a listen entry names WebSocket, otherwise `{}` — and always
|
|
1582
1640
|
// `{}` when `network.transports` is set, since the embedder owns transport policy
|
|
1583
1641
|
// then. Spread NEXT to `transports` because the two answer the same question.
|
|
@@ -1650,11 +1708,9 @@ export class CadreNode {
|
|
|
1650
1708
|
* `StrandSolicitationService.hasOutstandingInvitation`). A formation
|
|
1651
1709
|
* initiator is another party's peer by design and its token is only
|
|
1652
1710
|
* checkable inside the protocol, so the gate asks the coarser question
|
|
1653
|
-
* "does this node expect a stranger at all?".
|
|
1654
|
-
*
|
|
1655
|
-
*
|
|
1656
|
-
* bring-up) and `formStrand`'s lazy initialization both leave the gate
|
|
1657
|
-
* armed, because neither mints an invitation.
|
|
1711
|
+
* "does this node expect a stranger at all?". REGISTERING the responder
|
|
1712
|
+
* does not suspend stranger denial: every node registers one at
|
|
1713
|
+
* {@link start}, and registering mints no invitation.
|
|
1658
1714
|
*
|
|
1659
1715
|
* Ordering is semantically free (the checks are OR'd) but decides who pays:
|
|
1660
1716
|
* checks 1-3 are in-memory, 4/5 share one control-DB read, and only a peer
|
|
@@ -1686,6 +1742,11 @@ export class CadreNode {
|
|
|
1686
1742
|
* layer is fail-open behind `ADMISSION_DECISION_TIMEOUT_MS`, so the live
|
|
1687
1743
|
* read is safe here — unlike the per-stream gate, which must consult the
|
|
1688
1744
|
* materialized {@link authorizedControlPeers} snapshot instead.
|
|
1745
|
+
* NOTE: check 6 adds a second control read (`hasOutstandingFormationInvite`) for a
|
|
1746
|
+
* stranger with no locally minted invitation in play, on every node now that every
|
|
1747
|
+
* node runs the responder — a relay-enabled storage node included. If stranger
|
|
1748
|
+
* connections to such a node ever arrive fast enough for that read to show, cache
|
|
1749
|
+
* the answer for a few seconds.
|
|
1689
1750
|
* NOTE: on a relay-DISABLED node, a sibling whose membership row has not yet
|
|
1690
1751
|
* replicated here is denied until the row converges (typically via the
|
|
1691
1752
|
* owner); either side's next outbound reconcile dial (outbound is never
|
|
@@ -2529,19 +2590,27 @@ export class CadreNode {
|
|
|
2529
2590
|
* handling, is what keeps a mixed list from taking a whole peer offline.
|
|
2530
2591
|
*/
|
|
2531
2592
|
async resolvePeerAddrs(peerId, opts = {}) {
|
|
2593
|
+
return (await this.resolvePeerRecord(peerId, opts))?.addrs ?? [];
|
|
2594
|
+
}
|
|
2595
|
+
/**
|
|
2596
|
+
* {@link resolvePeerAddrs} with the record's `UpdatedAt` kept beside the addresses, for a
|
|
2597
|
+
* caller that ranks peers by how recently they published. `null` wherever that returns `[]`
|
|
2598
|
+
* for a failed gate.
|
|
2599
|
+
*/
|
|
2600
|
+
async resolvePeerRecord(peerId, opts) {
|
|
2532
2601
|
if (!this.controlDatabase) {
|
|
2533
2602
|
throw new Error('CadreNode must be started before resolving peer addrs');
|
|
2534
2603
|
}
|
|
2535
2604
|
const record = await this.controlDatabase.queryPeerRecord(peerId);
|
|
2536
2605
|
if (!record) {
|
|
2537
2606
|
log('resolvePeerAddrs: no record for %s', peerId);
|
|
2538
|
-
return
|
|
2607
|
+
return null;
|
|
2539
2608
|
}
|
|
2540
2609
|
// publicKey <-> peerId binding: the stored key must be the one embedded in
|
|
2541
2610
|
// the requested Ed25519 peer id (also rejects a non-Ed25519 / missing key).
|
|
2542
2611
|
if (!record.publicKey || ed25519PublicKeyB64FromPeerId(peerId) !== record.publicKey) {
|
|
2543
2612
|
log('resolvePeerAddrs: publicKey does not match peerId for %s', peerId);
|
|
2544
|
-
return
|
|
2613
|
+
return null;
|
|
2545
2614
|
}
|
|
2546
2615
|
// Self-signature over (peerId, addrs, updatedAt).
|
|
2547
2616
|
if (!verifyPeerRecordSignature(record)) {
|
|
@@ -2549,13 +2618,13 @@ export class CadreNode {
|
|
|
2549
2618
|
// no addrs — a not-yet-self-published row)" from a genuinely corrupt or
|
|
2550
2619
|
// mixed-revision row. Sig/addrs are public replicated data.
|
|
2551
2620
|
log('resolvePeerAddrs: signature verification failed for %s (updatedAt=%d, addrs=%o, sig=%s)', peerId, record.updatedAt, record.addrs, record.sig ? `${record.sig.slice(0, 16)}…` : '(empty)');
|
|
2552
|
-
return
|
|
2621
|
+
return null;
|
|
2553
2622
|
}
|
|
2554
2623
|
// Freshness: never hand back a dead relay reservation.
|
|
2555
2624
|
const maxAgeMs = opts.maxAgeMs ?? DEFAULT_PEER_RECORD_MAX_AGE_MS;
|
|
2556
2625
|
if (!isPeerRecordFresh(record.updatedAt, maxAgeMs, Date.now())) {
|
|
2557
2626
|
log('resolvePeerAddrs: record for %s is stale (updatedAt=%d, maxAgeMs=%d)', peerId, record.updatedAt, maxAgeMs);
|
|
2558
|
-
return
|
|
2627
|
+
return null;
|
|
2559
2628
|
}
|
|
2560
2629
|
// Pluggable trust gate (defaults to current-member).
|
|
2561
2630
|
const trustPolicy = opts.trustPolicy ?? currentMemberTrustPolicy();
|
|
@@ -2567,7 +2636,7 @@ export class CadreNode {
|
|
|
2567
2636
|
});
|
|
2568
2637
|
if (!trusted) {
|
|
2569
2638
|
log('resolvePeerAddrs: trust policy rejected %s', peerId);
|
|
2570
|
-
return
|
|
2639
|
+
return null;
|
|
2571
2640
|
}
|
|
2572
2641
|
// Order signaling-first (the on-record order was what we verified above),
|
|
2573
2642
|
// optionally restrict to signaling addrs, then parse — dropping any addr
|
|
@@ -2577,7 +2646,7 @@ export class CadreNode {
|
|
|
2577
2646
|
if (opts.signalingOnly) {
|
|
2578
2647
|
addrs = addrs.filter(isSignalingAddr);
|
|
2579
2648
|
}
|
|
2580
|
-
return this.normalizeDialAddrs(this.parseMultiaddrs(addrs), peerId);
|
|
2649
|
+
return { addrs: this.normalizeDialAddrs(this.parseMultiaddrs(addrs), peerId), updatedAt: record.updatedAt };
|
|
2581
2650
|
}
|
|
2582
2651
|
/**
|
|
2583
2652
|
* Guarantee every address handed to a control-network dial terminates in
|
|
@@ -2737,10 +2806,10 @@ export class CadreNode {
|
|
|
2737
2806
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
2738
2807
|
return { dialed: [] };
|
|
2739
2808
|
}
|
|
2740
|
-
// Then re-warm each running strand's own address book from its
|
|
2741
|
-
// from whichever siblings are due an ask. Same reasoning as
|
|
2742
|
-
// below, one layer down: replication runs on the STRAND
|
|
2743
|
-
// layer under cadre-core dials strand peers by bare peer id.
|
|
2809
|
+
// Then re-warm each running strand's own address book from its FRET address
|
|
2810
|
+
// records and from whichever siblings are due an ask. Same reasoning as
|
|
2811
|
+
// warmSiblingAddrBook below, one layer down: replication runs on the STRAND
|
|
2812
|
+
// network, and every layer under cadre-core dials strand peers by bare peer id.
|
|
2744
2813
|
await this.refreshStrandPeerAddrs();
|
|
2745
2814
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
2746
2815
|
return { dialed: [] };
|
|
@@ -3111,15 +3180,18 @@ export class CadreNode {
|
|
|
3111
3180
|
*
|
|
3112
3181
|
* The list may name transports this node cannot dial (a lent node reports TCP
|
|
3113
3182
|
* and `/ws` addresses to a phone that dials WebSockets only). That needs no
|
|
3114
|
-
* filtering here: libp2p's dial queue (`calculateMultiaddrs`, libp2p 3.
|
|
3183
|
+
* filtering here: libp2p's dial queue (`calculateMultiaddrs`, libp2p 3.3.11)
|
|
3115
3184
|
* rejects an address no transport can dial before touching the network, so
|
|
3116
3185
|
* such an address costs a log line.
|
|
3117
3186
|
*
|
|
3118
3187
|
* The list is NOT handed to one `dial()`. libp2p tries a multi-address dial's
|
|
3119
|
-
* addresses one after another under a single deadline,
|
|
3120
|
-
*
|
|
3121
|
-
*
|
|
3122
|
-
*
|
|
3188
|
+
* addresses one after another under a single deadline, and sorts loopback
|
|
3189
|
+
* addresses last — so one or two addresses that never answer use up most of
|
|
3190
|
+
* the deadline before the address that works is tried. libp2p 3.3's
|
|
3191
|
+
* per-address `addressDialTimeout` does not prevent that: cadre sizes it for a
|
|
3192
|
+
* cold relayed open through two admission decisions (`optimysticDialLimits`,
|
|
3193
|
+
* 39 s at the default declared link).
|
|
3194
|
+
* {@link dialPeerAddrs} dials each address on its own time limit instead.
|
|
3123
3195
|
*/
|
|
3124
3196
|
async resolveControlDialAddrs(peerId, resolved) {
|
|
3125
3197
|
if (resolved.length > 0) {
|
|
@@ -3484,6 +3556,8 @@ export class CadreNode {
|
|
|
3484
3556
|
}
|
|
3485
3557
|
// 4. In-session pending owner authorize writes.
|
|
3486
3558
|
await this.drainPendingPeerWrites();
|
|
3559
|
+
// 5. `PendingJoin` rows this process wrote while alone. Their tombstones went out in step 1.
|
|
3560
|
+
await this.pendingJoinRunner?.reissueWritesMadeAlone();
|
|
3487
3561
|
}
|
|
3488
3562
|
/**
|
|
3489
3563
|
* Re-touch this node's own `DeviceToken` row (re-sign + bump `UpdatedAt` via
|
|
@@ -4127,6 +4201,10 @@ export class CadreNode {
|
|
|
4127
4201
|
// it is stopped here.
|
|
4128
4202
|
this.relayReserveSupervisor?.stop();
|
|
4129
4203
|
this.relayReserveSupervisor = null;
|
|
4204
|
+
// Before anything it reads is torn down. An attempt in flight is not awaited (NOTE at
|
|
4205
|
+
// `PendingJoinRunner.stop`).
|
|
4206
|
+
this.pendingJoinRunner?.stop();
|
|
4207
|
+
this.pendingJoinRunner = null;
|
|
4130
4208
|
// Drop the rest of the relay-reservation posture: a torn-down node holds no
|
|
4131
4209
|
// reservation, so neither a restarted instance nor a caller inspecting a failed
|
|
4132
4210
|
// start reports the previous attempt's `reserved`.
|
|
@@ -4165,11 +4243,14 @@ export class CadreNode {
|
|
|
4165
4243
|
await this.pushFanoutService.close().catch((err) => log('Push fan-out close failed: %o', err));
|
|
4166
4244
|
this.pushFanoutService = null;
|
|
4167
4245
|
}
|
|
4168
|
-
// Unregister
|
|
4169
|
-
|
|
4170
|
-
|
|
4171
|
-
|
|
4246
|
+
// Unregister the formation responder once any queued swap has run, so no swap
|
|
4247
|
+
// registers a handler after this. A restarted node installs a fresh one.
|
|
4248
|
+
await this.solicitationSwaps;
|
|
4249
|
+
if (this.registeredSolicitation && this.controlNode) {
|
|
4250
|
+
await this.registeredSolicitation.unregisterResponder(this.controlNode);
|
|
4172
4251
|
}
|
|
4252
|
+
this.registeredSolicitation = null;
|
|
4253
|
+
this.strandSolicitationService = null;
|
|
4173
4254
|
// Stop strand watcher
|
|
4174
4255
|
if (this.strandWatcher) {
|
|
4175
4256
|
await this.strandWatcher.stop();
|
|
@@ -5275,9 +5356,9 @@ export class CadreNode {
|
|
|
5275
5356
|
// `privateKey` — tests, and an embedder that opted out of a stable identity) the
|
|
5276
5357
|
// strand node still runs under a key CADRE-CORE HOLDS: a fresh random Ed25519 key,
|
|
5277
5358
|
// exactly what libp2p would generate internally if handed none, except that the
|
|
5278
|
-
//
|
|
5279
|
-
// the retained launch config keeps
|
|
5280
|
-
//
|
|
5359
|
+
// delegate announcement below can name the strand peer id before the node exists and
|
|
5360
|
+
// the retained launch config keeps that id across a hibernation resume. Stability
|
|
5361
|
+
// across RESTARTS still needs an identity key.
|
|
5281
5362
|
//
|
|
5282
5363
|
// NOTE: derivation requires an Ed25519 identity key, so a node configured
|
|
5283
5364
|
// with some other key type now fails strand launch outright (surfaced as
|
|
@@ -5329,15 +5410,12 @@ export class CadreNode {
|
|
|
5329
5410
|
this.forgetRevokedJoin(revokedStrandId);
|
|
5330
5411
|
this.emit('strand:revoked', { strandId: revokedStrandId });
|
|
5331
5412
|
},
|
|
5332
|
-
//
|
|
5333
|
-
//
|
|
5334
|
-
//
|
|
5335
|
-
|
|
5336
|
-
//
|
|
5337
|
-
|
|
5338
|
-
// the launch config too. `?? undefined`: the field is `null` before start(), and a
|
|
5339
|
-
// strand cannot launch before start(), so this is belt and braces.
|
|
5340
|
-
strandPeerBook: this.strandPeerBookStore ?? undefined,
|
|
5413
|
+
// Where the strand node saves its network state and what it re-imports when it is
|
|
5414
|
+
// built. Retained with the launch config, so a hibernation wake rebuilds the node
|
|
5415
|
+
// over the table the quiesced node last saved. `?? undefined`: the field is `null`
|
|
5416
|
+
// before start(), and a strand cannot launch before start(), so this is belt and
|
|
5417
|
+
// braces.
|
|
5418
|
+
networkState: this.strandNetworkStateStore ?? undefined,
|
|
5341
5419
|
onRejoinBlocked: (blockedStrandId) => this.emit('strand:rejoin-blocked', { strandId: blockedStrandId }),
|
|
5342
5420
|
// The joiner's first-sync write gate (strand-first-sync-gate.ts): a launch that
|
|
5343
5421
|
// comes up `'syncing'` announces the moment its database is published.
|
|
@@ -5457,31 +5535,11 @@ export class CadreNode {
|
|
|
5457
5535
|
*/
|
|
5458
5536
|
async resolveCohortSeed(strandId, delegatePeerId) {
|
|
5459
5537
|
const siblings = await this.resolveSiblingSeed(strandId, delegatePeerId);
|
|
5460
|
-
// Sibling answers FIRST: they were resolved just now, while
|
|
5461
|
-
//
|
|
5538
|
+
// Sibling answers FIRST: they were resolved just now, while the formation-carried
|
|
5539
|
+
// addresses are as old as the formation. `unionAddrs` appends only what the siblings
|
|
5462
5540
|
// did not already name, so a fresher answer is never displaced and each source keeps
|
|
5463
|
-
// its own
|
|
5464
|
-
return unionAddrs(siblings, this.
|
|
5465
|
-
}
|
|
5466
|
-
/**
|
|
5467
|
-
* The strand peer book's contribution to a strand's seed: every live entry's
|
|
5468
|
-
* addresses, freshest peer first, minus this node's own entry when `selfPeerId` is
|
|
5469
|
-
* known (the book swap files one under the strand transport id; a node must never
|
|
5470
|
-
* dial itself). Empty before {@link start} builds the store.
|
|
5471
|
-
*/
|
|
5472
|
-
strandPeerBookAddrs(strandId, selfPeerId) {
|
|
5473
|
-
const store = this.strandPeerBookStore;
|
|
5474
|
-
if (!store) {
|
|
5475
|
-
return [];
|
|
5476
|
-
}
|
|
5477
|
-
const addrs = [];
|
|
5478
|
-
for (const entry of store.entries(strandId)) {
|
|
5479
|
-
if (entry.peerId === selfPeerId) {
|
|
5480
|
-
continue;
|
|
5481
|
-
}
|
|
5482
|
-
addrs.push(...entry.addrs);
|
|
5483
|
-
}
|
|
5484
|
-
return addrs;
|
|
5541
|
+
// its own ordering.
|
|
5542
|
+
return unionAddrs(siblings, this.formationStrandAddrs.get(strandId) ?? []);
|
|
5485
5543
|
}
|
|
5486
5544
|
/**
|
|
5487
5545
|
* The own-party half of {@link resolveCohortSeed}: strand-network addresses resolved
|
|
@@ -5661,9 +5719,9 @@ export class CadreNode {
|
|
|
5661
5719
|
* caller runs the drive regardless).
|
|
5662
5720
|
*
|
|
5663
5721
|
* NOTE: against a relay that is DOWN this hook costs up to two strand-addr
|
|
5664
|
-
* timeouts (dial by peer id, then by addr;
|
|
5722
|
+
* timeouts (dial by peer id, then by addr; 28.5 s each at the default declared link round trip)
|
|
5665
5723
|
* before the reservation drive even starts, so one failed re-drive holds the supervisor
|
|
5666
|
-
* `driving` for those
|
|
5724
|
+
* `driving` for those 57 s plus the drive's own deadline — 75 s at the default, and longer on
|
|
5667
5725
|
* a host that declared a slower one (`link-budget.ts`).
|
|
5668
5726
|
* Bounded and harmless while the relay is unreachable anyway; if recovery
|
|
5669
5727
|
* latency after a relay comes back ever matters, skip the announce when the
|
|
@@ -5697,18 +5755,19 @@ export class CadreNode {
|
|
|
5697
5755
|
this.recordDelegateAnnounces(due.map((relay) => relay.relayPeerId), strandId, now);
|
|
5698
5756
|
}
|
|
5699
5757
|
/**
|
|
5700
|
-
* Keep each running strand's own libp2p address book warm: re-merge
|
|
5701
|
-
*
|
|
5702
|
-
*
|
|
5758
|
+
* Keep each running strand's own libp2p address book warm: re-merge the address
|
|
5759
|
+
* records its FRET routing table holds on every pass
|
|
5760
|
+
* ({@link remergeStrandFretRecords}), and re-ask each connected SIBLING for its
|
|
5761
|
+
* strand addresses over the control mesh when that (sibling, strand) is due
|
|
5703
5762
|
* ({@link strandAddrAskDueAt}).
|
|
5704
5763
|
*
|
|
5705
|
-
* Without this, a strand's address book
|
|
5764
|
+
* Without this, cadre-core writes a strand's address book exactly once — the
|
|
5706
5765
|
* launch/resume seed — and everything below cadre-core that dials a strand peer
|
|
5707
5766
|
* by bare peer id (Optimystic's cluster and repo clients, FRET ping/announce)
|
|
5708
|
-
* loses its address for any
|
|
5767
|
+
* loses its address for any peer it is not currently connected to: a sibling
|
|
5709
5768
|
* that restarted its strand node or rotated its relay reservation is never
|
|
5710
|
-
* re-resolved, and
|
|
5711
|
-
* one-hour expiry (see `peer-addr-book.ts`).
|
|
5769
|
+
* re-resolved, and every address, another party's included, falls off at the
|
|
5770
|
+
* peerStore's one-hour expiry (see `peer-addr-book.ts`).
|
|
5712
5771
|
*
|
|
5713
5772
|
* Due times are per (sibling, strand) and set from each sibling's own outcome, so
|
|
5714
5773
|
* a sibling that connects late — a phone joining after the party's always-on
|
|
@@ -5789,15 +5848,15 @@ export class CadreNode {
|
|
|
5789
5848
|
}
|
|
5790
5849
|
/**
|
|
5791
5850
|
* One strand's share of {@link refreshStrandPeerAddrs}: RPC the siblings that are
|
|
5792
|
-
* due
|
|
5793
|
-
*
|
|
5794
|
-
* strand's failure never costs the others their refresh.
|
|
5851
|
+
* due and merge their answers into the strand's address book, then re-merge the
|
|
5852
|
+
* address records the strand node's FRET table holds. Errors are logged and
|
|
5853
|
+
* swallowed so one strand's failure never costs the others their refresh.
|
|
5795
5854
|
*
|
|
5796
|
-
* The
|
|
5797
|
-
*
|
|
5798
|
-
*
|
|
5799
|
-
*
|
|
5800
|
-
*
|
|
5855
|
+
* The sibling half runs first, so a failure reading the FRET table cannot cost a
|
|
5856
|
+
* sibling answer its merge. The FRET half, and the formation-carried addresses of
|
|
5857
|
+
* the peers it holds no record for, run on EVERY pass, whether or not any sibling is
|
|
5858
|
+
* due or connected — see {@link remergeStrandFretRecords} and
|
|
5859
|
+
* {@link remergeUnrecordedFormationAddrs} for why.
|
|
5801
5860
|
*/
|
|
5802
5861
|
async refreshOneStrandPeerAddrs(strandId, targets, now) {
|
|
5803
5862
|
const controlNode = this.controlNode;
|
|
@@ -5809,33 +5868,93 @@ export class CadreNode {
|
|
|
5809
5868
|
// Inside the try: a strand node torn down mid-pass may throw on any read, and one
|
|
5810
5869
|
// strand's failure must never cost the others their refresh.
|
|
5811
5870
|
const strandPeerId = strandNode.peerId.toString();
|
|
5812
|
-
const contactAddrs = this.strandPeerBookAddrs(strandId, strandPeerId);
|
|
5813
5871
|
const due = targets.filter((target) => now >= (this.strandAddrAskDueAt.get(peerStrandKey(target.peerId, strandId)) ?? 0));
|
|
5814
5872
|
// The running strand node's own peerId is the delegate to announce — see
|
|
5815
5873
|
// the relationship note on refreshStrandPeerAddrs.
|
|
5816
5874
|
const siblingAddrs = due.length === 0
|
|
5817
5875
|
? []
|
|
5818
5876
|
: await this.askSiblingsForStrandAddrs(controlNode, due, strandId, strandPeerId, now);
|
|
5819
|
-
|
|
5820
|
-
// already restarted onto a new node) must never have its store written to.
|
|
5821
|
-
if (!this._running || this.strandManager.getInstance(strandId)?.libp2pNode !== strandNode) {
|
|
5877
|
+
if (!this.isRunningStrandNode(strandId, strandNode)) {
|
|
5822
5878
|
return;
|
|
5823
5879
|
}
|
|
5824
|
-
|
|
5825
|
-
|
|
5826
|
-
return;
|
|
5880
|
+
if (siblingAddrs.length > 0) {
|
|
5881
|
+
await this.mergeStrandPeerAddrs(strandNode, siblingAddrs, strandId);
|
|
5827
5882
|
}
|
|
5828
|
-
|
|
5829
|
-
|
|
5830
|
-
// unchanged address set writes nothing in libp2p's persistent peer store, so it
|
|
5831
|
-
// is cheap today; if many strands run at once, merge only book entries changed
|
|
5832
|
-
// since the last tick plus those nearing the one-hour expiry.
|
|
5833
|
-
await this.mergeStrandPeerAddrs(strandNode, addrs, strandId);
|
|
5883
|
+
const recorded = await this.remergeStrandFretRecords(strandNode, strandId);
|
|
5884
|
+
await this.remergeUnrecordedFormationAddrs(strandNode, strandId, recorded);
|
|
5834
5885
|
}
|
|
5835
5886
|
catch (error) {
|
|
5836
5887
|
log('refreshStrandPeerAddrs: strand %s refresh failed (continuing): %o', strandId, error);
|
|
5837
5888
|
}
|
|
5838
5889
|
}
|
|
5890
|
+
/**
|
|
5891
|
+
* Is `strandNode` still the node `strandId` runs on? Asked again after every await
|
|
5892
|
+
* in a refresh pass: a strand stopped mid-pass, or one already restarted onto a new
|
|
5893
|
+
* node, must never have its store written to.
|
|
5894
|
+
*/
|
|
5895
|
+
isRunningStrandNode(strandId, strandNode) {
|
|
5896
|
+
return this._running && this.strandManager.getInstance(strandId)?.libp2pNode === strandNode;
|
|
5897
|
+
}
|
|
5898
|
+
/**
|
|
5899
|
+
* Re-merge, into a running strand node's address book, the addresses of every peer
|
|
5900
|
+
* its FRET routing table holds a signed address record for (`strand-fret-addrs.ts`).
|
|
5901
|
+
*
|
|
5902
|
+
* This is what keeps ANOTHER PARTY's strand nodes dialable from a node that stays
|
|
5903
|
+
* up. Nothing re-resolves their addresses — the strand-addr RPC answers own-party
|
|
5904
|
+
* siblings only — and the peerStore hides an address one hour after it was first
|
|
5905
|
+
* observed. FRET keeps each peer's record but hands it to the peerStore only when it
|
|
5906
|
+
* arrives or is imported, so without this pass a connection to another party that
|
|
5907
|
+
* drops after the first hour could not be redialed by either side.
|
|
5908
|
+
*
|
|
5909
|
+
* A record is held while FRET holds the peer's table entry, and at most 14 days after
|
|
5910
|
+
* it was last confirmed, so that is also how long a wrong address can keep being
|
|
5911
|
+
* re-merged. Each record is signed by the peer it names and verified before use, so
|
|
5912
|
+
* the exposure is failed dials to an address that peer itself published.
|
|
5913
|
+
*
|
|
5914
|
+
* NOTE: every pass serialises the whole FRET table (`exportTable()`), verifies one
|
|
5915
|
+
* signature per record and makes one `peerStore.merge` per peer, per running strand
|
|
5916
|
+
* per 15 s tick. Fine at strand table sizes, and an unchanged address set writes
|
|
5917
|
+
* nothing in libp2p's persistent peer store. If many strands or large tables make it
|
|
5918
|
+
* show up, read only the entries whose address stamp is near the one-hour mark.
|
|
5919
|
+
*
|
|
5920
|
+
* @returns the ids of the peers the table holds a usable record for.
|
|
5921
|
+
*/
|
|
5922
|
+
async remergeStrandFretRecords(strandNode, strandId) {
|
|
5923
|
+
const { peers, rejected } = await strandFretPeerAddrs(strandNode);
|
|
5924
|
+
const recorded = new Set(peers.keys());
|
|
5925
|
+
if ((peers.size === 0 && rejected === 0) || !this.isRunningStrandNode(strandId, strandNode)) {
|
|
5926
|
+
return recorded;
|
|
5927
|
+
}
|
|
5928
|
+
const counts = { merged: 0, restamped: 0, skipped: 0, failed: 0 };
|
|
5929
|
+
for (const [peerId, addrs] of peers) {
|
|
5930
|
+
counts[await mergePeerAddrs(strandNode, peerId, addrs)]++;
|
|
5931
|
+
}
|
|
5932
|
+
log('strand %s FRET address records re-merged (peers=%d, merged=%d, restamped=%d, failed=%d, rejected=%d)', strandId, peers.size, counts.merged, counts.restamped, counts.failed, rejected);
|
|
5933
|
+
return recorded;
|
|
5934
|
+
}
|
|
5935
|
+
/**
|
|
5936
|
+
* Re-merge the addresses this strand's formation carried ({@link formationStrandAddrs})
|
|
5937
|
+
* for every peer the strand node's FRET table holds no record for (`recorded`).
|
|
5938
|
+
*
|
|
5939
|
+
* Until the first connection to the responder, the carried addresses are the only
|
|
5940
|
+
* thing naming it, and the launch seed writes them once. FRET's `bootstraps` keep only
|
|
5941
|
+
* the peer id, so if the responder stays unreachable past the peerStore's one-hour
|
|
5942
|
+
* expiry, nothing could dial it when it returns. Once FRET holds the peer's own signed
|
|
5943
|
+
* record, that record supersedes what the formation carried and this stops.
|
|
5944
|
+
*/
|
|
5945
|
+
async remergeUnrecordedFormationAddrs(strandNode, strandId, recorded) {
|
|
5946
|
+
const carried = this.formationStrandAddrs.get(strandId);
|
|
5947
|
+
if (!carried) {
|
|
5948
|
+
return;
|
|
5949
|
+
}
|
|
5950
|
+
const unrecorded = [...groupAddrsByPeerId(carried)]
|
|
5951
|
+
.filter(([peerId]) => !recorded.has(peerId))
|
|
5952
|
+
.flatMap(([, addrs]) => addrs.map((addr) => addr.toString()));
|
|
5953
|
+
if (unrecorded.length === 0 || !this.isRunningStrandNode(strandId, strandNode)) {
|
|
5954
|
+
return;
|
|
5955
|
+
}
|
|
5956
|
+
await this.mergeStrandPeerAddrs(strandNode, unrecorded, strandId);
|
|
5957
|
+
}
|
|
5839
5958
|
/**
|
|
5840
5959
|
* RPC `due` siblings for `strandId` and set each one's next due time from its own
|
|
5841
5960
|
* outcome: an answer, even an empty one, waits the full refresh interval; anything
|
|
@@ -5861,11 +5980,11 @@ export class CadreNode {
|
|
|
5861
5980
|
* attributed per peer. Best-effort throughout: an address-book write must never
|
|
5862
5981
|
* fail a launch, a resume, or a reconcile pass.
|
|
5863
5982
|
*
|
|
5864
|
-
* `addrs` is
|
|
5865
|
-
* {@link groupAddrsByPeerId} attributes each
|
|
5866
|
-
* peerId in its final `/p2p/` component — never
|
|
5867
|
-
* which names a different libp2p node entirely.
|
|
5868
|
-
* dropped there and counted here, once per pass.
|
|
5983
|
+
* `addrs` is a peer-agnostic list — the union the strand-addr RPC returns, or the
|
|
5984
|
+
* addresses a formation carried — so {@link groupAddrsByPeerId} attributes each
|
|
5985
|
+
* entry to the **strand transport** peerId in its final `/p2p/` component — never
|
|
5986
|
+
* the sibling's control peerId, which names a different libp2p node entirely.
|
|
5987
|
+
* Entries that name no peer are dropped there and counted here, once per pass.
|
|
5869
5988
|
*
|
|
5870
5989
|
* NOTE: a member could answer with arbitrary multiaddrs bound to arbitrary peer
|
|
5871
5990
|
* ids and poison this address book. That is the same exposure the launch-time
|
|
@@ -5875,14 +5994,13 @@ export class CadreNode {
|
|
|
5875
5994
|
* peerStore's one-hour expiry. No new gating here — cross-party strand trust is
|
|
5876
5995
|
* `backlog/strand-network-nat-relay-reachability`.
|
|
5877
5996
|
*
|
|
5878
|
-
* NOTE: one input
|
|
5879
|
-
* ({@link
|
|
5880
|
-
*
|
|
5881
|
-
*
|
|
5882
|
-
*
|
|
5883
|
-
*
|
|
5884
|
-
*
|
|
5885
|
-
* exposure is a handful of failed dials, not a trust hole.
|
|
5997
|
+
* NOTE: one input outlives that hour — a formation's carried addresses
|
|
5998
|
+
* ({@link formationStrandAddrs}) are in every launch and resume seed for as long as
|
|
5999
|
+
* this process remembers the formation, and re-merged on every refresh pass until the
|
|
6000
|
+
* strand node's FRET table holds the peer's own record, so a junk address from a
|
|
6001
|
+
* responder that is never met keeps being re-merged. Bounded by `sanitizeStrandAddrs`,
|
|
6002
|
+
* replaced by a re-formation, dropped on `unpublishStrand` / `forgetJoinedStrand`, and
|
|
6003
|
+
* still authority-free — so the exposure is a handful of failed dials, not a trust hole.
|
|
5886
6004
|
*/
|
|
5887
6005
|
async mergeStrandPeerAddrs(strandNode, addrs, strandId) {
|
|
5888
6006
|
const counts = { merged: 0, restamped: 0, skipped: 0, failed: 0 };
|
|
@@ -5931,7 +6049,7 @@ export class CadreNode {
|
|
|
5931
6049
|
// is normally a no-op; it is the guard for a bare listen addr and for a relay hop
|
|
5932
6050
|
// whose trailing `/p2p/` names the RELAY. An addr terminating in a DIFFERENT id is
|
|
5933
6051
|
// dropped rather than announced — it does not reach this node, and announcing it
|
|
5934
|
-
// would file our address under someone else's id in the receiver's book.
|
|
6052
|
+
// would file our address under someone else's id in the receiver's address book.
|
|
5935
6053
|
//
|
|
5936
6054
|
// Re-parsed through this package's own `multiaddr` rather than passed straight in:
|
|
5937
6055
|
// libp2p hands back its nested `@multiformats/multiaddr` copy, a structurally
|
|
@@ -5951,7 +6069,8 @@ export class CadreNode {
|
|
|
5951
6069
|
/**
|
|
5952
6070
|
* Leave a strand joined from another party, for the whole party: remove its party-wide
|
|
5953
6071
|
* `JoinedStrand` row (owner-signed, with a `Revocation` tombstone), then this machine's
|
|
5954
|
-
* unpublished record, its
|
|
6072
|
+
* unpublished record, its formation-carried addresses, its saved network state and any
|
|
6073
|
+
* session-kept entry, then
|
|
5955
6074
|
* {@link stopStrand} it here. Every other machine's watcher then sees the row gone and
|
|
5956
6075
|
* detaches the strand, a storage replica included; a machine offline at the time reaps the
|
|
5957
6076
|
* row once the tombstone reaches it. The joiner's counterpart of {@link unpublishStrand},
|
|
@@ -6008,8 +6127,9 @@ export class CadreNode {
|
|
|
6008
6127
|
* session here — the `strand:revoked` contract is that nothing is torn down for the app —
|
|
6009
6128
|
* and its party-wide row is queued for removal by the next connected owner reconcile pass
|
|
6010
6129
|
* (`JoinedStrandSession.forgetAfterThisSession`). A no-op for this party's own strands,
|
|
6011
|
-
* which have neither record; the
|
|
6012
|
-
* removed party must not keep dialing the
|
|
6130
|
+
* which have neither record; the formation-carried addresses and the saved network
|
|
6131
|
+
* state are forgotten either way, since a removed party must not keep dialing the
|
|
6132
|
+
* strand's peers.
|
|
6013
6133
|
*
|
|
6014
6134
|
* NOTE: accepted tradeoff — a sibling machine that has not raised `strand:revoked` itself
|
|
6015
6135
|
* detaches the strand (`strand:stopped`) when the party-wide removal reaches it, instead
|
|
@@ -6033,45 +6153,16 @@ export class CadreNode {
|
|
|
6033
6153
|
this.forgetStrandPeers(strandId, 'self-revocation');
|
|
6034
6154
|
}
|
|
6035
6155
|
/**
|
|
6036
|
-
* Forget a strand's
|
|
6037
|
-
*
|
|
6038
|
-
*
|
|
6156
|
+
* Forget what this node holds about a strand's peers: the addresses its formation
|
|
6157
|
+
* carried (see {@link formationStrandAddrs}) and its saved network state (see
|
|
6158
|
+
* {@link strandNetworkStateStore}). The store write is fire-and-log — the removal is
|
|
6159
|
+
* visible synchronously by the store's contract, and a failed persist only leaves the
|
|
6160
|
+
* entry on disk for the next start.
|
|
6039
6161
|
*/
|
|
6040
6162
|
forgetStrandPeers(strandId, caller) {
|
|
6041
|
-
|
|
6042
|
-
|
|
6043
|
-
|
|
6044
|
-
}
|
|
6045
|
-
/**
|
|
6046
|
-
* The strand peer book's observation writer (`StartStrandConfig.onStrandPeerIdentified`):
|
|
6047
|
-
* a strand node identified a strand peer at `observation.addrs`, so remember it as an
|
|
6048
|
-
* unsigned entry seen just now. The observer never reports self, and the store binds
|
|
6049
|
-
* every address to the peer id before filing it.
|
|
6050
|
-
*/
|
|
6051
|
-
observeStrandPeer(strandId, observation) {
|
|
6052
|
-
this.rememberStrandPeer(strandId, {
|
|
6053
|
-
peerId: observation.peerId,
|
|
6054
|
-
addrs: observation.addrs,
|
|
6055
|
-
issuedAt: 0,
|
|
6056
|
-
lastSeenAt: Date.now()
|
|
6057
|
-
}, 'observeStrandPeer');
|
|
6058
|
-
}
|
|
6059
|
-
/**
|
|
6060
|
-
* Merge one entry into the peer book. Fire-and-log: the entry is visible
|
|
6061
|
-
* synchronously by the store's contract, and the promise tracks durability only. A
|
|
6062
|
-
* persist failure costs restart survival, never this session's seed — the same trade
|
|
6063
|
-
* {@link retainDialTarget} makes.
|
|
6064
|
-
*/
|
|
6065
|
-
rememberStrandPeer(strandId, entry, caller) {
|
|
6066
|
-
const store = this.strandPeerBookStore;
|
|
6067
|
-
if (!store) {
|
|
6068
|
-
// Unreachable in production: start() builds the store before any strand can
|
|
6069
|
-
// launch or form. Logged rather than thrown, for the same reason as retainDialTarget.
|
|
6070
|
-
log('%s: no strand peer book yet — not remembering %s for strand %s', caller, entry.peerId, strandId);
|
|
6071
|
-
return;
|
|
6072
|
-
}
|
|
6073
|
-
void store.merge(strandId, entry).catch((error) => {
|
|
6074
|
-
log('%s: persisting strand %s peer %s failed (continuing): %o', caller, strandId, entry.peerId, error);
|
|
6163
|
+
this.formationStrandAddrs.delete(strandId);
|
|
6164
|
+
void this.strandNetworkStateStore?.forget(strandId).catch((error) => {
|
|
6165
|
+
log('%s: forgetting strand %s network state failed (continuing): %o', caller, strandId, error);
|
|
6075
6166
|
});
|
|
6076
6167
|
}
|
|
6077
6168
|
/**
|
|
@@ -6551,6 +6642,9 @@ export class CadreNode {
|
|
|
6551
6642
|
...(this.trustedOwnerStore ? { trustedOwners: this.trustedOwnerStore } : {}),
|
|
6552
6643
|
}), this.controlNode, this.controlDatabase);
|
|
6553
6644
|
log('Seed bootstrap service initialized');
|
|
6645
|
+
// Embedders wire the owner key after start(), so the runner's first pass usually found
|
|
6646
|
+
// this machine not yet an owner; without this it would wait a full poll interval.
|
|
6647
|
+
this.pendingJoinRunner?.kick();
|
|
6554
6648
|
}
|
|
6555
6649
|
/**
|
|
6556
6650
|
* Push the multiaddrs that future invites should advertise. Pass `null` to
|
|
@@ -7230,19 +7324,32 @@ export class CadreNode {
|
|
|
7230
7324
|
// Strand Solicitation API (native cadre-core formation transport)
|
|
7231
7325
|
// ============================================================================
|
|
7232
7326
|
/**
|
|
7233
|
-
*
|
|
7234
|
-
*
|
|
7327
|
+
* Replace this node's strand solicitation service — the formation responder every node
|
|
7328
|
+
* installs at {@link start}, and the initiator side of {@link formStrand}. Optional: call it
|
|
7329
|
+
* only to customize the responder (an approver, a provisioner, formation deadlines). The
|
|
7330
|
+
* new service's handler replaces the previous one's on the control node, and invitations
|
|
7331
|
+
* the previous service minted stay outstanding.
|
|
7332
|
+
*
|
|
7333
|
+
* Without `options.formationUsageRecorder`, tokens are checked against this party's
|
|
7334
|
+
* replicated `FormationInvite`/`FormationUsage` rows ({@link ControlFormationUsageRecorder}).
|
|
7335
|
+
* A responder that accepts every token is reachable only by constructing
|
|
7336
|
+
* {@link StrandSolicitationService} directly.
|
|
7235
7337
|
*
|
|
7236
7338
|
* @param options - Configuration for the solicitation service
|
|
7237
7339
|
*/
|
|
7238
7340
|
async initializeStrandSolicitation(options) {
|
|
7239
|
-
|
|
7341
|
+
const controlNode = this.controlNode;
|
|
7342
|
+
const controlDatabase = this.controlDatabase;
|
|
7343
|
+
if (!controlNode || !controlDatabase) {
|
|
7240
7344
|
throw new Error('CadreNode must be started before initializing strand solicitation');
|
|
7241
7345
|
}
|
|
7242
7346
|
const service = new StrandSolicitationService({
|
|
7243
7347
|
...options,
|
|
7348
|
+
formationUsageRecorder: options?.formationUsageRecorder ?? new ControlFormationUsageRecorder(controlDatabase),
|
|
7244
7349
|
partyId: this.config.controlNetwork.partyId,
|
|
7245
|
-
|
|
7350
|
+
// Read per formation: the responder is installed during start, before a relay
|
|
7351
|
+
// reservation gives a relay-only node any address, and addresses change after.
|
|
7352
|
+
cadrePeerAddrs: () => this.getMultiaddrs(),
|
|
7246
7353
|
// Every formation deadline derives from this node's declared link, like every other
|
|
7247
7354
|
// dial budget here; a caller's own declaration on the formation config still wins.
|
|
7248
7355
|
formationConfig: {
|
|
@@ -7258,25 +7365,60 @@ export class CadreNode {
|
|
|
7258
7365
|
// issuer here would admit joiners under someone else's authority.
|
|
7259
7366
|
issueMembershipInvite: (strandId, signal) => this.issueStrandMembershipInvite(strandId, signal)
|
|
7260
7367
|
});
|
|
7261
|
-
// Set before the registration is awaited, so a concurrent createOpenInvitation /
|
|
7262
|
-
// formStrand uses this service instead of building a second one whose handler
|
|
7263
|
-
// would collide; a failed registration puts the previous service back.
|
|
7264
|
-
// NOTE: same stray-handler caveat as installSeedBootstrapService — a failed
|
|
7265
|
-
// peer-store merge leaves the formation handler registered, so a retry is a duplicate.
|
|
7266
7368
|
const previous = this.strandSolicitationService;
|
|
7369
|
+
if (previous) {
|
|
7370
|
+
service.adoptMintedInvitations(previous);
|
|
7371
|
+
}
|
|
7372
|
+
// Set before the swap is awaited, so a concurrent createOpenInvitation / formStrand
|
|
7373
|
+
// uses this service instead of building another.
|
|
7267
7374
|
this.strandSolicitationService = service;
|
|
7375
|
+
const swap = this.solicitationSwaps.then(() => this.swapFormationResponder(service, controlNode));
|
|
7376
|
+
// The queue only orders swaps; a failed one rejects this call's own await below.
|
|
7377
|
+
this.solicitationSwaps = swap.catch(() => undefined);
|
|
7378
|
+
await swap;
|
|
7379
|
+
log('Strand solicitation service initialized');
|
|
7380
|
+
}
|
|
7381
|
+
/**
|
|
7382
|
+
* Answer formation from start. Log-and-continue: a node that cannot is still useful, and
|
|
7383
|
+
* {@link createOpenInvitation} / {@link formStrand} install the responder when it is missing.
|
|
7384
|
+
*/
|
|
7385
|
+
async installDefaultFormationResponder() {
|
|
7386
|
+
try {
|
|
7387
|
+
await this.initializeStrandSolicitation();
|
|
7388
|
+
}
|
|
7389
|
+
catch (error) {
|
|
7390
|
+
log('Formation responder not installed at start; createOpenInvitation/formStrand will retry: %o', error);
|
|
7391
|
+
}
|
|
7392
|
+
}
|
|
7393
|
+
/**
|
|
7394
|
+
* Move the control node's formation handler from the registered service to `service`.
|
|
7395
|
+
* Queued by {@link initializeStrandSolicitation} so swaps run one at a time: libp2p allows
|
|
7396
|
+
* one handler per protocol id, and two overlapping swaps would each find the other's
|
|
7397
|
+
* handler in the way. A failed swap registers the previous service again, points
|
|
7398
|
+
* {@link strandSolicitationService} back at it unless a later call has replaced it, and
|
|
7399
|
+
* rethrows.
|
|
7400
|
+
*
|
|
7401
|
+
* NOTE: same stray-handler caveat as {@link installSeedBootstrapService} — a failed
|
|
7402
|
+
* peer-store merge leaves the formation handler registered, so the restore below (and a
|
|
7403
|
+
* retry) rejects as a duplicate.
|
|
7404
|
+
*/
|
|
7405
|
+
async swapFormationResponder(service, controlNode) {
|
|
7406
|
+
const previous = this.registeredSolicitation;
|
|
7268
7407
|
try {
|
|
7269
|
-
await
|
|
7408
|
+
await previous?.unregisterResponder(controlNode);
|
|
7409
|
+
await service.registerResponder(controlNode);
|
|
7410
|
+
this.registeredSolicitation = service;
|
|
7270
7411
|
}
|
|
7271
7412
|
catch (error) {
|
|
7272
7413
|
if (this.strandSolicitationService === service)
|
|
7273
7414
|
this.strandSolicitationService = previous;
|
|
7415
|
+
await previous?.registerResponder(controlNode).catch((restoreError) => log('Re-registering the previous formation responder failed; none answers until the next initializeStrandSolicitation: %o', restoreError));
|
|
7274
7416
|
throw error;
|
|
7275
7417
|
}
|
|
7276
|
-
log('Strand solicitation service initialized');
|
|
7277
7418
|
}
|
|
7278
7419
|
/**
|
|
7279
|
-
* Get the strand solicitation service (for advanced use)
|
|
7420
|
+
* Get the strand solicitation service (for advanced use). Non-null on a started node unless
|
|
7421
|
+
* the install at {@link start} failed.
|
|
7280
7422
|
*/
|
|
7281
7423
|
getStrandSolicitationService() {
|
|
7282
7424
|
return this.strandSolicitationService;
|
|
@@ -7284,28 +7426,66 @@ export class CadreNode {
|
|
|
7284
7426
|
/**
|
|
7285
7427
|
* Create an open invitation for others to form strands with this party.
|
|
7286
7428
|
*
|
|
7429
|
+
* The invitation's bootstrap list names this machine first (it is the one most likely to
|
|
7430
|
+
* run the host strand), then a few of the party's other machines
|
|
7431
|
+
* ({@link siblingInvitationAddrs}), so a joiner can still form while this one is offline.
|
|
7432
|
+
*
|
|
7287
7433
|
* @param sAppId - The sApp to use for formed strands
|
|
7288
7434
|
* @param expirationMs - How long the invitation is valid (ms from now)
|
|
7289
7435
|
* @returns The open invitation to share out-of-band
|
|
7436
|
+
* @throws when neither this machine nor any other machine of the party has an address
|
|
7290
7437
|
*/
|
|
7291
7438
|
async createOpenInvitation(sAppId, expirationMs = 24 * 60 * 60 * 1000 // 24 hours default
|
|
7292
7439
|
) {
|
|
7293
7440
|
if (!this.strandSolicitationService) {
|
|
7294
|
-
//
|
|
7441
|
+
// Only when the install at start failed.
|
|
7295
7442
|
await this.initializeStrandSolicitation();
|
|
7296
7443
|
}
|
|
7297
|
-
const bootstrap = this.getMultiaddrs();
|
|
7444
|
+
const bootstrap = [...this.getMultiaddrs(), ...await this.siblingInvitationAddrs()];
|
|
7298
7445
|
if (bootstrap.length === 0) {
|
|
7299
7446
|
throw new Error('No multiaddrs available for invitation');
|
|
7300
7447
|
}
|
|
7301
7448
|
return await this.strandSolicitationService.createOpenInvitation(sAppId, expirationMs, bootstrap);
|
|
7302
7449
|
}
|
|
7303
7450
|
/**
|
|
7304
|
-
*
|
|
7451
|
+
* Addresses of the party's other machines for an invitation's bootstrap list
|
|
7452
|
+
* ({@link selectInvitationSiblingAddrs} picks which). The source is each authorized
|
|
7453
|
+
* member's signed `CadrePeer` record ({@link resolvePeerRecord}), so a machine whose record
|
|
7454
|
+
* is missing, stale or untrusted is left out; live connections only order them.
|
|
7455
|
+
*
|
|
7456
|
+
* Best-effort: a failed read leaves the invitation naming this machine alone, which is what
|
|
7457
|
+
* it named before siblings were added.
|
|
7458
|
+
*/
|
|
7459
|
+
async siblingInvitationAddrs() {
|
|
7460
|
+
const controlNode = this.controlNode;
|
|
7461
|
+
if (!controlNode || !this.controlDatabase) {
|
|
7462
|
+
return [];
|
|
7463
|
+
}
|
|
7464
|
+
try {
|
|
7465
|
+
const siblings = [];
|
|
7466
|
+
for (const { peerId } of await this.listAuthorizedMembers()) {
|
|
7467
|
+
const resolved = await this.resolvePeerRecord(peerId, {});
|
|
7468
|
+
if (resolved && resolved.addrs.length > 0) {
|
|
7469
|
+
siblings.push({ peerId, updatedAt: resolved.updatedAt, addrs: resolved.addrs.map(String) });
|
|
7470
|
+
}
|
|
7471
|
+
}
|
|
7472
|
+
const connected = new Set(controlNode.getConnections().map((c) => c.remotePeer.toString()));
|
|
7473
|
+
return selectInvitationSiblingAddrs(siblings, connected);
|
|
7474
|
+
}
|
|
7475
|
+
catch (error) {
|
|
7476
|
+
log('createOpenInvitation: resolving the other machines failed; the invitation names only this one: %o', error);
|
|
7477
|
+
return [];
|
|
7478
|
+
}
|
|
7479
|
+
}
|
|
7480
|
+
/**
|
|
7481
|
+
* Form a strand with a responder via an open invitation: one attempt, from this machine, with
|
|
7482
|
+
* nothing recorded party-wide. {@link requestJoin} is the path that keeps trying.
|
|
7305
7483
|
*
|
|
7306
7484
|
* @param invitation - The open invitation received out-of-band
|
|
7307
7485
|
* @param disclosure - Identity/context information to share with the responder
|
|
7308
7486
|
* @returns The member key and strand info if successful
|
|
7487
|
+
* @throws `FormationRejectedError` or `FormationUnreachableError` when the formation fails, and
|
|
7488
|
+
* `FormationPostApprovalError` when it was approved and a step on this machine then failed
|
|
7309
7489
|
*/
|
|
7310
7490
|
async formStrand(invitation, disclosure = {}) {
|
|
7311
7491
|
if (!this.controlNode) {
|
|
@@ -7315,7 +7495,7 @@ export class CadreNode {
|
|
|
7315
7495
|
await this.initializeStrandSolicitation();
|
|
7316
7496
|
}
|
|
7317
7497
|
const result = await this.strandSolicitationService.formStrand(invitation, disclosure, this.controlNode);
|
|
7318
|
-
this.
|
|
7498
|
+
await this.recordFormationStrandAddrs(result.strandId, result.strandAddrs);
|
|
7319
7499
|
// A closed host strand's approval carries the joiner's own membership invitation —
|
|
7320
7500
|
// persist this party's identity key and stage the invitation for bring-up to redeem.
|
|
7321
7501
|
// Runs AFTER the addr recording so a persistence failure (which throws — see the
|
|
@@ -7326,6 +7506,171 @@ export class CadreNode {
|
|
|
7326
7506
|
await this.rememberFormedStrand(result);
|
|
7327
7507
|
return result;
|
|
7328
7508
|
}
|
|
7509
|
+
/**
|
|
7510
|
+
* Ask to join through `invitation` and have the party keep trying until the join works, the
|
|
7511
|
+
* invitation is used up or expires, or the request is dismissed: across restarts, and from
|
|
7512
|
+
* any owner machine of the party (`docs/strands.md` → "Joining while the inviter is
|
|
7513
|
+
* offline"). Records a party-wide `PendingJoin` row, runs one attempt on this machine at
|
|
7514
|
+
* once, and returns the status after it: `'joined'` when the inviter answered yes, `'waiting'`
|
|
7515
|
+
* when it was unreachable or not ready, `'failed'` when it refused for good.
|
|
7516
|
+
*
|
|
7517
|
+
* Asking again for an invitation whose request is still pending adopts that request as it
|
|
7518
|
+
* is; asking again after it finished starts a fresh one. Later changes arrive as
|
|
7519
|
+
* `pendingJoin:changed`. A joined strand is offered through `strand:discovered` like any
|
|
7520
|
+
* {@link formStrand} join. Do not also call {@link formStrand} with the same invitation:
|
|
7521
|
+
* nothing coordinates the two, and one of them gets `token-spent`.
|
|
7522
|
+
*
|
|
7523
|
+
* @throws on a machine that is not an enrolled owner (the row is owner-signed), and on an
|
|
7524
|
+
* invitation already past its expiration
|
|
7525
|
+
*/
|
|
7526
|
+
async requestJoin(invitation, disclosure = {}) {
|
|
7527
|
+
const { database, runner } = this.requirePendingJoins();
|
|
7528
|
+
const signer = await this.requireOwnerSigner('requestJoin needs an enrolled owner machine of this party, because the party-wide join request is ' +
|
|
7529
|
+
'owner-signed. formStrand(invitation, disclosure) joins once from this machine, with no retries.');
|
|
7530
|
+
const now = Date.now();
|
|
7531
|
+
const expiresAt = invitation.expiration.getTime();
|
|
7532
|
+
if (!Number.isFinite(expiresAt)) {
|
|
7533
|
+
throw new Error('The invitation carries no valid expiration');
|
|
7534
|
+
}
|
|
7535
|
+
if (now >= expiresAt) {
|
|
7536
|
+
throw new Error(`The invitation expired at ${invitation.expiration.toISOString()}; ask the inviter for a fresh one`);
|
|
7537
|
+
}
|
|
7538
|
+
const requested = requestedPendingJoin(pendingJoinId(invitation.token), this.encodeInvitation(invitation), disclosure, expiresAt, now);
|
|
7539
|
+
const row = await this.recordJoinRequest(database, requested, signer);
|
|
7540
|
+
log('requestJoin: pending join %s recorded; attempting now', row.Id);
|
|
7541
|
+
return runner.attemptNow(row);
|
|
7542
|
+
}
|
|
7543
|
+
/**
|
|
7544
|
+
* Every join asked for with {@link requestJoin} and not dismissed, pending or finished, as this
|
|
7545
|
+
* machine sees it (see `PendingJoinStatus` for which states are machine-local).
|
|
7546
|
+
*/
|
|
7547
|
+
async listPendingJoins() {
|
|
7548
|
+
const { database, runner } = this.requirePendingJoins();
|
|
7549
|
+
return (await database.queryPendingJoins()).map((row) => runner.statusOf(row));
|
|
7550
|
+
}
|
|
7551
|
+
/**
|
|
7552
|
+
* Remove a join request party-wide. On a pending request this cancels it: every owner machine
|
|
7553
|
+
* stops at its next read, and an attempt already running does not record its outcome (a join
|
|
7554
|
+
* it made stays on that machine, as any {@link formStrand} join does). Owner machines only.
|
|
7555
|
+
* Resolves `false` when there was no such request.
|
|
7556
|
+
*/
|
|
7557
|
+
async dismissPendingJoin(id) {
|
|
7558
|
+
const { database, runner } = this.requirePendingJoins();
|
|
7559
|
+
const { ownerKey, signMessage } = await this.requireOwnerSigner('dismissPendingJoin needs an enrolled owner machine of this party: the removal is owner-signed');
|
|
7560
|
+
const removed = await database.deletePendingJoin(id, ownerKey, signMessage);
|
|
7561
|
+
runner.forgetRow(id);
|
|
7562
|
+
return removed;
|
|
7563
|
+
}
|
|
7564
|
+
requirePendingJoins() {
|
|
7565
|
+
if (!this._running || !this.controlDatabase || !this.pendingJoinRunner) {
|
|
7566
|
+
throw new Error('CadreNode must be started before using pending joins');
|
|
7567
|
+
}
|
|
7568
|
+
return { database: this.controlDatabase, runner: this.pendingJoinRunner };
|
|
7569
|
+
}
|
|
7570
|
+
/** This machine's owner key and signer for an owner-signed control write, or a throw carrying `refusal`. */
|
|
7571
|
+
async requireOwnerSigner(refusal) {
|
|
7572
|
+
const signingKey = await this.enrolledOwnerSigningKey();
|
|
7573
|
+
if (!signingKey) {
|
|
7574
|
+
throw new Error(refusal);
|
|
7575
|
+
}
|
|
7576
|
+
return { ownerKey: signingKey.publicKeyB64, signMessage: signMessageWith(signingKey.privateKeyB64) };
|
|
7577
|
+
}
|
|
7578
|
+
/**
|
|
7579
|
+
* Write a join request's row, or adopt the party's existing row for the same invitation: a
|
|
7580
|
+
* pending one as it is, a finished one replaced by `requested` (the user asked again).
|
|
7581
|
+
*/
|
|
7582
|
+
async recordJoinRequest(database, requested, { ownerKey, signMessage }) {
|
|
7583
|
+
let written;
|
|
7584
|
+
try {
|
|
7585
|
+
written = await database.insertPendingJoin(requested, ownerKey, signMessage);
|
|
7586
|
+
}
|
|
7587
|
+
catch (error) {
|
|
7588
|
+
if (!isPendingJoinConflict(error)) {
|
|
7589
|
+
throw error;
|
|
7590
|
+
}
|
|
7591
|
+
const existing = await database.queryPendingJoin(requested.Id);
|
|
7592
|
+
if (!existing) {
|
|
7593
|
+
throw new Error(`A removed join request for this invitation (${requested.Id}) is still held on this machine until its ` +
|
|
7594
|
+
'clean-up runs, which needs a connection to the party. Ask again once connected.', { cause: error });
|
|
7595
|
+
}
|
|
7596
|
+
if (existing.Outcome === null) {
|
|
7597
|
+
return existing;
|
|
7598
|
+
}
|
|
7599
|
+
written = await database.replacePendingJoin(existing.StampId, requested, ownerKey, signMessage);
|
|
7600
|
+
}
|
|
7601
|
+
this.pendingJoinRunner?.noteWritten(written);
|
|
7602
|
+
return written;
|
|
7603
|
+
}
|
|
7604
|
+
/** Build and start the pending-join retry loop over this node (see {@link pendingJoinRunner}). */
|
|
7605
|
+
startPendingJoinRunner() {
|
|
7606
|
+
const database = () => {
|
|
7607
|
+
if (!this.controlDatabase) {
|
|
7608
|
+
throw new Error('CadreNode is stopped: no control database for pending joins');
|
|
7609
|
+
}
|
|
7610
|
+
return this.controlDatabase;
|
|
7611
|
+
};
|
|
7612
|
+
const outcomeSigner = () => this.requireOwnerSigner('Cannot record a pending join outcome: this machine is not an enrolled owner of the party');
|
|
7613
|
+
this.pendingJoinRunner = new PendingJoinRunner({
|
|
7614
|
+
selfId: this.controlNode.peerId.toString(),
|
|
7615
|
+
linkRoundTripMs: this.config.network?.linkRoundTripMs,
|
|
7616
|
+
isOwner: async () => (await this.enrolledOwnerSigningKey()) !== null,
|
|
7617
|
+
// NOTE: one owner-key read and one `PendingJoin` read per pass (30 s); for a party that never
|
|
7618
|
+
// asked for a join the second reads a never-written block, which consults the cohort. If it
|
|
7619
|
+
// shows up in a device profile, poll more slowly while the table reads empty.
|
|
7620
|
+
readRows: () => database().queryPendingJoins(),
|
|
7621
|
+
readRow: (id) => database().queryPendingJoin(id),
|
|
7622
|
+
attempt: (row) => this.formStrand(this.pendingJoinInvitation(row), parseStoredDisclosure(row)),
|
|
7623
|
+
replace: async (expectedStampId, next) => {
|
|
7624
|
+
const { ownerKey, signMessage } = await outcomeSigner();
|
|
7625
|
+
return database().replacePendingJoin(expectedStampId, next, ownerKey, signMessage);
|
|
7626
|
+
},
|
|
7627
|
+
remove: async (id) => {
|
|
7628
|
+
const { ownerKey, signMessage } = await outcomeSigner();
|
|
7629
|
+
return database().deletePendingJoin(id, ownerKey, signMessage);
|
|
7630
|
+
},
|
|
7631
|
+
isAlone: () => this.committedAlone(),
|
|
7632
|
+
sAppIdOf: (row) => this.pendingJoinSAppId(row),
|
|
7633
|
+
observeRows: (rows) => this.stageMembershipInvitesFromPendingJoins(rows),
|
|
7634
|
+
emit: (status) => this.emit('pendingJoin:changed', status),
|
|
7635
|
+
});
|
|
7636
|
+
this.pendingJoinRunner.start();
|
|
7637
|
+
}
|
|
7638
|
+
/** The invitation a `PendingJoin` row stores. A failure names the row, never the invitation, which is a bearer credential. */
|
|
7639
|
+
pendingJoinInvitation(row) {
|
|
7640
|
+
try {
|
|
7641
|
+
return this.decodeInvitation(row.Invitation);
|
|
7642
|
+
}
|
|
7643
|
+
catch {
|
|
7644
|
+
throw new Error(`PendingJoin ${row.Id}: the stored invitation does not decode`);
|
|
7645
|
+
}
|
|
7646
|
+
}
|
|
7647
|
+
pendingJoinSAppId(row) {
|
|
7648
|
+
try {
|
|
7649
|
+
return this.decodeInvitation(row.Invitation).sAppId;
|
|
7650
|
+
}
|
|
7651
|
+
catch {
|
|
7652
|
+
log('PendingJoin %s: the stored invitation does not decode; reporting no sApp', row.Id);
|
|
7653
|
+
return '';
|
|
7654
|
+
}
|
|
7655
|
+
}
|
|
7656
|
+
/**
|
|
7657
|
+
* Stage the membership invitation of each `joined` row this process has not staged yet, so a
|
|
7658
|
+
* closed strand joined on another owner machine is seated by whichever machine launches it
|
|
7659
|
+
* first; the party key it admits is the replicated `StrandPartyKey` row the finishing machine
|
|
7660
|
+
* seated. A strand that already has an invitation staged keeps it, for the reconciler to
|
|
7661
|
+
* settle first.
|
|
7662
|
+
*/
|
|
7663
|
+
stageMembershipInvitesFromPendingJoins(rows) {
|
|
7664
|
+
for (const { rowId, strandId, invite } of membershipInvitesToStage(rows, this.stagedMembershipInviteKeys, Date.now())) {
|
|
7665
|
+
this.stagedMembershipInviteKeys.add(invite.inviteKey);
|
|
7666
|
+
if (this.pendingMembershipInvites.has(strandId)) {
|
|
7667
|
+
continue;
|
|
7668
|
+
}
|
|
7669
|
+
this.pendingMembershipInvites.set(strandId, invite);
|
|
7670
|
+
log('pending join %s: staged its membership invitation for strand %s', rowId, strandId);
|
|
7671
|
+
this.strandManager.notifyMembershipInviteStaged(strandId);
|
|
7672
|
+
}
|
|
7673
|
+
}
|
|
7329
7674
|
/**
|
|
7330
7675
|
* Remember the strand a formation just joined (see {@link joinedStrandStore}), so it
|
|
7331
7676
|
* is re-offered after a restart even when the app is killed before its `addStrand`, and
|
|
@@ -7347,7 +7692,7 @@ export class CadreNode {
|
|
|
7347
7692
|
});
|
|
7348
7693
|
}
|
|
7349
7694
|
catch (error) {
|
|
7350
|
-
throw new
|
|
7695
|
+
throw new FormationPostApprovalError(result.strandId, `Formation for strand ${result.strandId} was approved (its one-time token is spent), but ` +
|
|
7351
7696
|
'remembering the join in this node\'s joined-strand store failed, so the strand would not ' +
|
|
7352
7697
|
'come back after a restart. Fix the store (usually the configured keyStore), then redeem a ' +
|
|
7353
7698
|
'fresh invitation.', { cause: error });
|
|
@@ -7380,12 +7725,13 @@ export class CadreNode {
|
|
|
7380
7725
|
await this.ensureStrandPartyKey(strandId);
|
|
7381
7726
|
}
|
|
7382
7727
|
catch (error) {
|
|
7383
|
-
throw new
|
|
7728
|
+
throw new FormationPostApprovalError(strandId, `Formation for strand ${strandId} was approved (its one-time token is spent), but ` +
|
|
7384
7729
|
'persisting this party\'s membership identity (StrandPartyKey) failed — the joiner ' +
|
|
7385
7730
|
'cannot become a member without it. Fix the underlying cause, then redeem a fresh ' +
|
|
7386
7731
|
'invitation (the delivered one dies with this error).', { cause: error });
|
|
7387
7732
|
}
|
|
7388
7733
|
this.pendingMembershipInvites.set(strandId, invite);
|
|
7734
|
+
this.stagedMembershipInviteKeys.add(invite.inviteKey);
|
|
7389
7735
|
log('formStrand: staged membership invitation for strand %s (party key persisted)', strandId);
|
|
7390
7736
|
this.strandManager.notifyMembershipInviteStaged(strandId);
|
|
7391
7737
|
}
|
|
@@ -7550,41 +7896,39 @@ export class CadreNode {
|
|
|
7550
7896
|
'runtime did not wake within the formation provisioning budget (still waking)');
|
|
7551
7897
|
}
|
|
7552
7898
|
/**
|
|
7553
|
-
*
|
|
7554
|
-
*
|
|
7555
|
-
*
|
|
7556
|
-
* after every restart.
|
|
7899
|
+
* Keep the responder's strand-network addresses for a strand this node just formed
|
|
7900
|
+
* (see {@link formationStrandAddrs}), so the strand's discovery seed has something to
|
|
7901
|
+
* dial when the app launches it.
|
|
7557
7902
|
*
|
|
7558
7903
|
* The carried list is peer-agnostic (`sanitizeStrandAddrs` bounds and parses it,
|
|
7559
7904
|
* nothing more), so it is attributed per peer here (`groupAddrsByPeerId`, the same
|
|
7560
|
-
* rule the address-book merge applies) and
|
|
7561
|
-
*
|
|
7562
|
-
*
|
|
7563
|
-
*
|
|
7564
|
-
*
|
|
7565
|
-
*
|
|
7566
|
-
|
|
7567
|
-
|
|
7568
|
-
|
|
7569
|
-
|
|
7905
|
+
* rule the address-book merge applies) and kept grouped by peer; an entry naming no
|
|
7906
|
+
* destination peer is dropped there. Scoped strictly to `strandId`: these addresses
|
|
7907
|
+
* reach ONE strand node of ONE other party and must never seed another strand's mesh
|
|
7908
|
+
* or the control peerStore.
|
|
7909
|
+
*
|
|
7910
|
+
* A list that names a peer REPLACES the strand's earlier one: the responder
|
|
7911
|
+
* disclosed its current addresses, so the older list is stale. An empty list, or one
|
|
7912
|
+
* naming no peer, records nothing, so it cannot wipe an earlier disclosure.
|
|
7913
|
+
*
|
|
7914
|
+
* A strand already running when this lands (a re-formation, which is the recovery
|
|
7915
|
+
* path for a dead address) gets the new addresses merged into its address book now.
|
|
7916
|
+
* Nothing else would deliver them: the seed is read only at launch and resume, and the
|
|
7917
|
+
* refresh pass skips a peer FRET already holds a record for — the stale one, here.
|
|
7918
|
+
*/
|
|
7919
|
+
async recordFormationStrandAddrs(strandId, strandAddrs) {
|
|
7920
|
+
const attributed = [...groupAddrsByPeerId([...strandAddrs]).values()]
|
|
7921
|
+
.flatMap((peerAddrs) => peerAddrs.map((addr) => addr.toString()));
|
|
7922
|
+
if (attributed.length === 0) {
|
|
7923
|
+
log('formStrand: responder disclosed no usable strand addrs for %s (%d carried) — cross-party seed unchanged', strandId, strandAddrs.length);
|
|
7570
7924
|
return;
|
|
7571
7925
|
}
|
|
7572
|
-
|
|
7573
|
-
|
|
7574
|
-
|
|
7575
|
-
|
|
7576
|
-
|
|
7577
|
-
|
|
7578
|
-
issuedAt: 0,
|
|
7579
|
-
lastSeenAt: now
|
|
7580
|
-
}, 'formStrand');
|
|
7581
|
-
peers++;
|
|
7582
|
-
}
|
|
7583
|
-
// A strand already running when this lands (a re-formation) needs nothing more: the
|
|
7584
|
-
// refresh pass merges the book into its address book on every reconcile tick, so the
|
|
7585
|
-
// new addresses land within 15 s — which matters because re-forming is the recovery
|
|
7586
|
-
// path for a dead entry.
|
|
7587
|
-
log('formStrand: recorded %d cross-party strand addr(s) for %s under %d peer(s)', strandAddrs.length, strandId, peers);
|
|
7926
|
+
this.formationStrandAddrs.set(strandId, attributed);
|
|
7927
|
+
log('formStrand: kept %d of %d cross-party strand addr(s) for %s', attributed.length, strandAddrs.length, strandId);
|
|
7928
|
+
const runningNode = this.strandManager.getInstance(strandId)?.libp2pNode;
|
|
7929
|
+
if (runningNode) {
|
|
7930
|
+
await this.mergeStrandPeerAddrs(runningNode, attributed, strandId);
|
|
7931
|
+
}
|
|
7588
7932
|
}
|
|
7589
7933
|
/**
|
|
7590
7934
|
* Encode an open invitation for out-of-band delivery (QR, link, etc.).
|