@serfab/cadre-core 1.17.0 → 1.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/dist/cadre-invite-protocol.d.ts +35 -13
- package/dist/cadre-invite-protocol.js +90 -25
- package/dist/cadre-invite-protocol.js.map +1 -1
- package/dist/cadre-node.d.ts +586 -255
- package/dist/cadre-node.js +1176 -564
- package/dist/cadre-node.js.map +1 -1
- package/dist/control-cohort.d.ts +66 -9
- package/dist/control-cohort.js +67 -9
- package/dist/control-cohort.js.map +1 -1
- package/dist/control-database.d.ts +451 -193
- package/dist/control-database.js +701 -359
- package/dist/control-database.js.map +1 -1
- package/dist/control-formation-recorder.d.ts +46 -43
- package/dist/control-formation-recorder.js +77 -78
- package/dist/control-formation-recorder.js.map +1 -1
- package/dist/control-protocol-guard.d.ts +6 -0
- package/dist/control-protocol-guard.js +14 -2
- package/dist/control-protocol-guard.js.map +1 -1
- package/dist/control-schema-version.d.ts +48 -31
- package/dist/control-schema-version.js +67 -45
- package/dist/control-schema-version.js.map +1 -1
- package/dist/control-schema.d.ts +8 -8
- package/dist/control-schema.js +16 -16
- package/dist/control-schema.js.map +1 -1
- package/dist/control-write-retry.d.ts +42 -7
- package/dist/control-write-retry.js +87 -19
- package/dist/control-write-retry.js.map +1 -1
- package/dist/enrolled-machine-store.d.ts +3 -3
- package/dist/enrolled-machine-store.js +3 -3
- package/dist/formation-token.d.ts +48 -0
- package/dist/formation-token.js +26 -0
- package/dist/formation-token.js.map +1 -0
- package/dist/index.d.ts +14 -13
- package/dist/index.js +16 -13
- package/dist/index.js.map +1 -1
- package/dist/joined-strand-store.d.ts +26 -9
- package/dist/joined-strand-store.js +63 -24
- package/dist/joined-strand-store.js.map +1 -1
- package/dist/link-budget.d.ts +9 -7
- package/dist/link-budget.js +9 -7
- package/dist/link-budget.js.map +1 -1
- package/dist/member-redemption.d.ts +15 -9
- package/dist/member-redemption.js +13 -8
- package/dist/member-redemption.js.map +1 -1
- package/dist/membership-view.d.ts +91 -0
- package/dist/membership-view.js +110 -0
- package/dist/membership-view.js.map +1 -0
- package/dist/owner-grant-protocol.d.ts +11 -7
- package/dist/owner-grant-protocol.js +31 -17
- package/dist/owner-grant-protocol.js.map +1 -1
- package/dist/peer-addr-book.d.ts +10 -1
- package/dist/peer-addr-book.js +4 -3
- package/dist/peer-addr-book.js.map +1 -1
- package/dist/peer-join-backfill.d.ts +89 -20
- package/dist/peer-join-backfill.js +265 -108
- package/dist/peer-join-backfill.js.map +1 -1
- package/dist/pending-join-runner.d.ts +48 -18
- package/dist/pending-join-runner.js +74 -56
- package/dist/pending-join-runner.js.map +1 -1
- package/dist/seed-bootstrap.d.ts +47 -40
- package/dist/seed-bootstrap.js +66 -61
- package/dist/seed-bootstrap.js.map +1 -1
- package/dist/strand-database.d.ts +23 -2
- package/dist/strand-database.js +26 -6
- package/dist/strand-database.js.map +1 -1
- package/dist/strand-first-sync-gate.d.ts +176 -89
- package/dist/strand-first-sync-gate.js +288 -122
- package/dist/strand-first-sync-gate.js.map +1 -1
- package/dist/strand-formation-deadlines.d.ts +9 -9
- package/dist/strand-formation-deadlines.js +9 -9
- package/dist/strand-formation-manager.d.ts +23 -17
- package/dist/strand-formation-manager.js +46 -44
- package/dist/strand-formation-manager.js.map +1 -1
- package/dist/strand-formation-protocol.d.ts +5 -8
- package/dist/strand-formation-protocol.js +101 -16
- package/dist/strand-formation-protocol.js.map +1 -1
- package/dist/strand-instance-manager.d.ts +26 -17
- package/dist/strand-instance-manager.js +119 -50
- package/dist/strand-instance-manager.js.map +1 -1
- package/dist/strand-membership-reconciler.d.ts +16 -13
- package/dist/strand-membership-reconciler.js +30 -22
- package/dist/strand-membership-reconciler.js.map +1 -1
- package/dist/strand-membership-writer.d.ts +40 -28
- package/dist/strand-membership-writer.js +212 -153
- package/dist/strand-membership-writer.js.map +1 -1
- package/dist/strand-move.js +5 -2
- package/dist/strand-move.js.map +1 -1
- package/dist/strand-revocation-enforcer.d.ts +134 -60
- package/dist/strand-revocation-enforcer.js +144 -71
- package/dist/strand-revocation-enforcer.js.map +1 -1
- package/dist/strand-solicitation.d.ts +50 -39
- package/dist/strand-solicitation.js +10 -3
- package/dist/strand-solicitation.js.map +1 -1
- package/dist/strand-watcher.d.ts +80 -13
- package/dist/strand-watcher.js +152 -48
- package/dist/strand-watcher.js.map +1 -1
- package/dist/table-change-watch.d.ts +63 -0
- package/dist/table-change-watch.js +209 -0
- package/dist/table-change-watch.js.map +1 -0
- package/dist/timeout-scheduler.d.ts +4 -3
- package/dist/timeout-scheduler.js +4 -3
- package/dist/timeout-scheduler.js.map +1 -1
- package/dist/types.d.ts +90 -49
- package/dist/types.js +4 -4
- package/dist/types.js.map +1 -1
- package/package.json +8 -8
- package/src/cadre-invite-protocol.ts +119 -31
- package/src/cadre-node.ts +1282 -584
- package/src/control-cohort.ts +86 -9
- package/src/control-database.ts +848 -403
- package/src/control-formation-recorder.ts +359 -353
- package/src/control-protocol-guard.ts +16 -2
- package/src/control-schema-version.ts +101 -51
- package/src/control-schema.ts +16 -16
- package/src/control-write-retry.ts +91 -19
- package/src/delegate-admission.ts +351 -351
- package/src/enrolled-machine-store.ts +3 -3
- package/src/formation-token.ts +63 -0
- package/src/index.ts +30 -17
- package/src/joined-strand-store.ts +69 -24
- package/src/link-budget.ts +9 -7
- package/src/member-redemption.ts +20 -13
- package/src/membership-view.ts +144 -0
- package/src/owner-grant-protocol.ts +33 -18
- package/src/peer-addr-book.ts +4 -3
- package/src/peer-join-backfill.ts +971 -761
- package/src/pending-join-runner.ts +76 -53
- package/src/seed-bootstrap.ts +72 -67
- package/src/strand-database.ts +352 -322
- package/src/strand-first-sync-gate.ts +334 -134
- package/src/strand-formation-deadlines.ts +227 -227
- package/src/strand-formation-manager.ts +731 -725
- package/src/strand-formation-protocol.ts +122 -24
- package/src/strand-instance-manager.ts +2092 -2026
- package/src/strand-membership-reconciler.ts +33 -25
- package/src/strand-membership-writer.ts +2093 -2019
- package/src/strand-move.ts +5 -2
- package/src/strand-revocation-enforcer.ts +740 -654
- package/src/strand-solicitation.ts +61 -34
- package/src/strand-watcher.ts +175 -56
- package/src/table-change-watch.ts +285 -0
- package/src/timeout-scheduler.ts +29 -28
- package/src/types.ts +89 -51
package/dist/cadre-node.js
CHANGED
|
@@ -21,7 +21,7 @@ import { MemoryBootstrapPeerStore } from './bootstrap-peer-store.js';
|
|
|
21
21
|
import { MemoryStrandNetworkStateStore } from './strand-network-state.js';
|
|
22
22
|
import { MemoryEnrolledMachineStore } from './enrolled-machine-store.js';
|
|
23
23
|
import { JoinedStrandSession, KeyStoreJoinedStrandStore, MemoryJoinedStrandStore } from './joined-strand-store.js';
|
|
24
|
-
import { mergePeerAddrs, groupAddrsByPeerId } from './peer-addr-book.js';
|
|
24
|
+
import { addrKey, mergePeerAddrs, groupAddrsByPeerId } from './peer-addr-book.js';
|
|
25
25
|
import { strandFretPeerAddrs } from './strand-fret-addrs.js';
|
|
26
26
|
import { verifyCadrePeerVoucher, verifyInvitationAdmission, verifyRevocationSigner } from './peer-authorization.js';
|
|
27
27
|
import { deriveOwnerAnchor } from './owner-anchor-sync.js';
|
|
@@ -30,28 +30,30 @@ import { describeOwnerKeys, namedOwnerKey, planOwnerKeyRemoval, revouchWithinPla
|
|
|
30
30
|
import { ed25519PublicKeyB64FromPeerId } from './ed25519-key.js';
|
|
31
31
|
import { signPeerRecord, verifyPeerRecordSignature, isPeerRecordFresh, orderSignalingFirst, isSignalingAddr, currentMemberTrustPolicy, DEFAULT_PEER_RECORD_MAX_AGE_MS, DEFAULT_PEER_RECORD_HEARTBEAT_MS } from './peer-record.js';
|
|
32
32
|
import { signDeviceTokenRecord, verifyDeviceTokenSignature, isPushPlatform } from './device-token.js';
|
|
33
|
-
import { StrandWatcher } from './strand-watcher.js';
|
|
33
|
+
import { StrandWatcher, DEFAULT_STRAND_WATCH_BACKSTOP_MS } from './strand-watcher.js';
|
|
34
34
|
import { StrandInstanceManager, liveStrandStatus } from './strand-instance-manager.js';
|
|
35
35
|
import { PeerJoinBackfill } from './peer-join-backfill.js';
|
|
36
36
|
import { deriveCohortMembers } from './strand-cohort.js';
|
|
37
|
-
import { selectControlCohortDials, DEFAULT_CONTROL_COHORT_RECONCILE_MS, DEFAULT_CONTROL_COHORT_TARGET_DEGREE } from './control-cohort.js';
|
|
37
|
+
import { selectControlCohortDials, nextReconcileWakeMs, CLAIM_RESTART_REDIAL_STEP_MS, CONTROL_COHORT_REDIAL_MS, DEFAULT_CONTROL_COHORT_RECONCILE_MS, DEFAULT_CONTROL_COHORT_TARGET_DEGREE } from './control-cohort.js';
|
|
38
|
+
import { scheduleRuns } from './table-change-watch.js';
|
|
39
|
+
import { MEMBERSHIP_VIEW_RECONNECT_RELOAD_MS } from './membership-view.js';
|
|
38
40
|
import { dialPeerAddrs, SelfRelayOnlyError, CONTROL_COHORT_DIAL_ADDRESS_ATTEMPTS, trailingPeerId, withTrailingPeerId } from './peer-dial.js';
|
|
39
41
|
import { ADMISSION_DECISION_TIMEOUT_MS, declaredCohortReadDeadlineMs, optimysticDialLimits, peerJoinPushBudget, relayReservationBudgetMs, relayedDialBudgetMs, relayedRequestBudgetMs, resolveLinkRoundTripMs } from './link-budget.js';
|
|
40
42
|
import { EnrollmentService } from './enrollment.js';
|
|
41
43
|
import { HibernationManager } from './hibernation-manager.js';
|
|
42
44
|
import { ControlDatabase, generateStampId, isPendingJoinConflict, isStrandIdConflict, IssuedRowNotHereYetError, pendingJoinId } from './control-database.js';
|
|
43
|
-
import { CadreInviteHandler, encodeCadreInvitation, redeemInPasses, signRedeemRequest } from './cadre-invite-protocol.js';
|
|
45
|
+
import { CadreInviteHandler, encodeCadreInvitation, redeemInPasses, sanitizeAddrs, signRedeemRequest } from './cadre-invite-protocol.js';
|
|
44
46
|
import { OwnerGrantHandler, OwnerGrantRejectedError, encodeOwnerGrant, redeemOwnerGrantInPasses, signOwnerGrantRequest } from './owner-grant-protocol.js';
|
|
45
47
|
import { FormationPostApprovalError } from './strand-formation-rejection.js';
|
|
46
48
|
import { PendingJoinRunner, membershipInvitesToStage, parseStoredDisclosure, requestedPendingJoin } from './pending-join-runner.js';
|
|
47
|
-
import { SeedBootstrapService, mergeSeedPeers, seedDigest } from './seed-bootstrap.js';
|
|
49
|
+
import { SeedBootstrapService, mergeSeedPeers, seedDialTargets, seedDigest } from './seed-bootstrap.js';
|
|
48
50
|
import { claimSecretTrustPolicy } from './seed-trust-policy.js';
|
|
49
51
|
import { ClaimRefusedError, claimProof, parseClaimSecret } from './claim-proof.js';
|
|
50
52
|
import { StrandSolicitationService } from './strand-solicitation.js';
|
|
51
53
|
import { ControlFormationUsageRecorder } from './control-formation-recorder.js';
|
|
52
54
|
import { cadreInvitationReach, selectCadreInvitationMembers, selectInvitationSiblingAddrs } from './invitation-bootstrap.js';
|
|
53
55
|
import { createMembershipConnectionGater, UnauthorizedReservationBudget } from './membership-connection-gater.js';
|
|
54
|
-
import { controlNetworkName, installControlProtocolGuard } from './control-protocol-guard.js';
|
|
56
|
+
import { controlNetworkName, installControlProtocolGuard, servesControlDatabase } from './control-protocol-guard.js';
|
|
55
57
|
import { StrandWakeService, dialWake } from './strand-wake-protocol.js';
|
|
56
58
|
import { StrandAddrService, collectStrandAddrs } from './strand-addr-protocol.js';
|
|
57
59
|
import { DelegateAdmissionStore, extractCircuitRelayTargets, dueRelayAnnounces, prunePeerStrandKeys, peerStrandKey, recordRelayRefusals, relaysNotRefusing } from './delegate-admission.js';
|
|
@@ -73,8 +75,8 @@ const timing = debug('sereus:cadre:timing');
|
|
|
73
75
|
*
|
|
74
76
|
* Ten minutes sits comfortably inside the peerStore's one-hour address expiry
|
|
75
77
|
* (`MAX_ADDRESS_AGE`, see `peer-addr-book.ts`) with headroom for several missed
|
|
76
|
-
* passes, and far above the
|
|
77
|
-
* strand-addr RPC fan-out stays cheap.
|
|
78
|
+
* passes, and far above the reconcile pass's 2-minute backstop the refresh rides
|
|
79
|
+
* on, so the strand-addr RPC fan-out stays cheap.
|
|
78
80
|
*/
|
|
79
81
|
export const STRAND_PEER_ADDR_REFRESH_MS = 10 * 60 * 1000;
|
|
80
82
|
/**
|
|
@@ -83,8 +85,9 @@ export const STRAND_PEER_ADDR_REFRESH_MS = 10 * 60 * 1000;
|
|
|
83
85
|
* phone asking before its `CadrePeer` row has replicated to the sibling is refused
|
|
84
86
|
* and its delegate grant goes unrecorded, and waiting the full
|
|
85
87
|
* {@link STRAND_PEER_ADDR_REFRESH_MS} for the next try is the delay this bounds.
|
|
86
|
-
*
|
|
87
|
-
*
|
|
88
|
+
* A pass arms its own wake for the earliest due time it leaves behind
|
|
89
|
+
* (`nextReconcileWakeMs`), so a sibling that keeps failing costs one timed-out
|
|
90
|
+
* RPC a minute, with no pass in between.
|
|
88
91
|
*/
|
|
89
92
|
export const STRAND_PEER_ADDR_RETRY_MS = 60 * 1000;
|
|
90
93
|
/**
|
|
@@ -451,8 +454,8 @@ export class CadreNode {
|
|
|
451
454
|
*
|
|
452
455
|
* Cleared by {@link cleanup} only, deliberately NOT by {@link detachStrand}: the watcher's
|
|
453
456
|
* filter rejection stops a replica through that very method, and forgetting the id there
|
|
454
|
-
* would have the next
|
|
455
|
-
* relaunch the replica —
|
|
457
|
+
* would have the next watcher run find it unknown again, re-admit the strand provisionally
|
|
458
|
+
* and relaunch the replica — on every other run, forever. A strand id's sApp never changes,
|
|
456
459
|
* and the map is bounded by the strands this party has published, like
|
|
457
460
|
* {@link discoveredStrands}.
|
|
458
461
|
*/
|
|
@@ -480,19 +483,24 @@ export class CadreNode {
|
|
|
480
483
|
* wait that ends in mutual denial (the upstream gate is fail-closed on
|
|
481
484
|
* timeout, unlike the fail-open connection gater). Optimystic's predicate is
|
|
482
485
|
* synchronous by contract. So the set is refreshed OUT OF BAND instead — after
|
|
483
|
-
* start,
|
|
484
|
-
*
|
|
485
|
-
*
|
|
486
|
-
*
|
|
486
|
+
* start, at the head of every {@link reconcileControlCohort} pass, whenever a
|
|
487
|
+
* membership table changes (the gate watch {@link start} takes through
|
|
488
|
+
* `ControlDatabase.watchMembershipTables`, which hears a local commit and a change
|
|
489
|
+
* replicated into this node's storage alike, within a 50 ms window), and immediately
|
|
490
|
+
* after every LOCAL membership mutation so a just-vouched peer is admitted
|
|
491
|
+
* before the writer's call returns. That last refresh is AUTOMATIC: the
|
|
492
|
+
* control database notifies {@link refreshMembershipGate} after every
|
|
487
493
|
* committed `CadrePeer` write (see `ControlDatabase.mutateCadrePeer`), so no
|
|
488
|
-
* writer has to remember. A change
|
|
489
|
-
*
|
|
490
|
-
*
|
|
491
|
-
*
|
|
492
|
-
*
|
|
493
|
-
*
|
|
494
|
-
*
|
|
495
|
-
*
|
|
494
|
+
* writer has to remember. A change this node's storage never hears of — a
|
|
495
|
+
* machine outside the block's cohort on a party larger than the replication
|
|
496
|
+
* breadth — is picked up at the pass's 2-minute backstop: bounded staleness,
|
|
497
|
+
* acceptable because this gate is defense in depth: rows an unadmitted peer
|
|
498
|
+
* manages to write are still disbelieved at read time by the voucher-anchored
|
|
499
|
+
* predicate. For cohort choice the same window means a member whose row
|
|
500
|
+
* arrived by replication is left out of this node's control-block cohorts
|
|
501
|
+
* until that refresh. Optimystic reads the predicate live on every lookup, so
|
|
502
|
+
* replacing this set is the whole re-evaluation; there is no per-peer
|
|
503
|
+
* reclassify call to make.
|
|
496
504
|
*/
|
|
497
505
|
this.authorizedControlPeers = new Set();
|
|
498
506
|
/**
|
|
@@ -530,9 +538,9 @@ export class CadreNode {
|
|
|
530
538
|
* When this node last ANNOUNCED a delegate, keyed `<targetPeerId>\n<strandId>`
|
|
531
539
|
* (the peer announced TO, not the delegate). Throttles
|
|
532
540
|
* {@link refreshDelegateGrants} to once per `DELEGATE_GRANT_TTL_MS / 2` per
|
|
533
|
-
* (relay, strand) so
|
|
534
|
-
* chatter; the launch/resume announce passes record here too.
|
|
535
|
-
* strand is no longer running are pruned on each reconcile pass.
|
|
541
|
+
* (relay, strand) so a run of change-driven reconcile passes never becomes
|
|
542
|
+
* per-pass RPC chatter; the launch/resume announce passes record here too.
|
|
543
|
+
* Keys whose strand is no longer running are pruned on each reconcile pass.
|
|
536
544
|
*/
|
|
537
545
|
this.delegateAnnounceAt = new Map();
|
|
538
546
|
/**
|
|
@@ -559,12 +567,13 @@ export class CadreNode {
|
|
|
559
567
|
* strand, keyed `peerStrandKey(siblingControlPeerId, strandId)` → epoch ms; a
|
|
560
568
|
* missing key is due. Bounds {@link refreshStrandPeerAddrs}'s fan-out per
|
|
561
569
|
* (sibling, strand) rather than per strand, so a sibling that connects after a
|
|
562
|
-
* pass is asked on the
|
|
563
|
-
* sibling's answer set. An answer (even an empty one) makes the sibling
|
|
564
|
-
* in {@link STRAND_PEER_ADDR_REFRESH_MS}; no answer, in
|
|
565
|
-
* {@link STRAND_PEER_ADDR_RETRY_MS}
|
|
566
|
-
*
|
|
567
|
-
*
|
|
570
|
+
* pass is asked on the pass its connection starts instead of waiting out a stamp
|
|
571
|
+
* another sibling's answer set. An answer (even an empty one) makes the sibling
|
|
572
|
+
* due again in {@link STRAND_PEER_ADDR_REFRESH_MS}; no answer, in
|
|
573
|
+
* {@link STRAND_PEER_ADDR_RETRY_MS}, and each pass arms its own wake for the
|
|
574
|
+
* earliest due time it leaves behind (`nextReconcileWakeMs`). Keys whose strand
|
|
575
|
+
* stopped running or whose sibling is no longer connected are pruned every pass,
|
|
576
|
+
* so a resumed strand and a reconnected sibling are both asked at once.
|
|
568
577
|
*/
|
|
569
578
|
this.strandAddrAskDueAt = new Map();
|
|
570
579
|
/**
|
|
@@ -646,6 +655,18 @@ export class CadreNode {
|
|
|
646
655
|
* `unpublishStrand` / `forgetJoinedStrand` as {@link formationStrandAddrs} is.
|
|
647
656
|
*/
|
|
648
657
|
this.pendingMembershipInvites = new Map();
|
|
658
|
+
/**
|
|
659
|
+
* The party key {@link adoptFormationMembershipInvite} just seated or read, by strand id,
|
|
660
|
+
* so the launch that follows a join does not read the `StrandPartyKey` row again. Taken
|
|
661
|
+
* (deleted) by the {@link resolveStrandPartyKey} that uses it, so any later launch reads
|
|
662
|
+
* the row; dropped by {@link detachStrand}. The joiner's counterpart of the founder's
|
|
663
|
+
* `StrandConfig.partyMemberPrivateKey`, without the app carrying the secret.
|
|
664
|
+
*
|
|
665
|
+
* NOTE: an entry is not dropped when the row is removed, because no `CadreNode` path removes
|
|
666
|
+
* it; an embedder calling `getControlDatabase().deleteStrandPartyKey` between a join and its
|
|
667
|
+
* launch would get the deleted key. If a node-level rotation is added, drop the entry there.
|
|
668
|
+
*/
|
|
669
|
+
this.formedPartyKeys = new Map();
|
|
649
670
|
/**
|
|
650
671
|
* Invite keys of the membership invitations this process has staged, by
|
|
651
672
|
* {@link adoptFormationMembershipInvite} or from a `JoinSuccess` row
|
|
@@ -739,7 +760,7 @@ export class CadreNode {
|
|
|
739
760
|
this.joinedStrandStore = null;
|
|
740
761
|
/**
|
|
741
762
|
* This session's joined strands: {@link joinedStrandStore} and the party-wide
|
|
742
|
-
* `JoinedStrand` table, unioned with the control rows each watcher
|
|
763
|
+
* `JoinedStrand` table, unioned with the control rows each watcher run, published and
|
|
743
764
|
* removed by the reconcile pass. Per SESSION, unlike the store — rebuilt at every
|
|
744
765
|
* {@link start}, which is what ends a revoked join's "keep offering until this session
|
|
745
766
|
* ends" (see `JoinedStrandSession.forgetAfterThisSession`).
|
|
@@ -752,17 +773,41 @@ export class CadreNode {
|
|
|
752
773
|
/** Listener that re-publishes the self record when reachable addresses change. */
|
|
753
774
|
this.selfPeerUpdateHandler = null;
|
|
754
775
|
/**
|
|
755
|
-
*
|
|
756
|
-
*
|
|
757
|
-
*
|
|
758
|
-
* {@link
|
|
776
|
+
* The reconcile pass's schedule (see {@link reconcileControlCohort}): single-flight
|
|
777
|
+
* runs on `table-change-watch.ts`'s rule, with no table subscription of its own — the
|
|
778
|
+
* gate watch ({@link membershipGateWatch}), the control connection listeners,
|
|
779
|
+
* `self:peer:update`, and the pass's own wake ({@link armReconcileWake}) `trigger` it,
|
|
780
|
+
* and its backstop (`network.controlCohort.reconcileMs`, default
|
|
781
|
+
* `DEFAULT_CONTROL_COHORT_RECONCILE_MS`) runs it when nothing has. Built alongside
|
|
782
|
+
* {@link recordRefreshTimer} in {@link startRecordRefresh}, about 1 s after start, and
|
|
783
|
+
* disposed in {@link stopRecordRefresh}; its timers never keep the loop alive.
|
|
784
|
+
*/
|
|
785
|
+
this.controlReconcileRuns = null;
|
|
786
|
+
/**
|
|
787
|
+
* The subscription on the membership tables (`ControlDatabase.watchMembershipTables`) that
|
|
788
|
+
* re-materializes the membership gate and triggers {@link controlReconcileRuns} on every
|
|
789
|
+
* change ({@link onMembershipGateTablesChanged}).
|
|
790
|
+
* Taken in {@link start} right after the membership listener is wired, disposed in
|
|
791
|
+
* {@link cleanup} before the control database closes.
|
|
792
|
+
*/
|
|
793
|
+
this.membershipGateWatch = null;
|
|
794
|
+
/** The second membership-view drop after a sibling connects ({@link armMembershipViewReload}); cleared in {@link stopRecordRefresh}. */
|
|
795
|
+
this.membershipViewReloadTimer = null;
|
|
796
|
+
/**
|
|
797
|
+
* Consecutive passes whose dial of a selected sibling or a cold-start bootstrap peer
|
|
798
|
+
* failed (`ControlCohortPassOutcome.unconnected`), for the redial ladder
|
|
799
|
+
* ({@link armReconcileWake}); 0 after a pass none of whose dials did. Per session:
|
|
800
|
+
* reset in {@link stopRecordRefresh}.
|
|
759
801
|
*/
|
|
760
|
-
this.
|
|
802
|
+
this.reconcileRedialStreak = 0;
|
|
803
|
+
/** Peers a {@link redialClaimedNode} loop is running for, so a repeated claim starts no second one. */
|
|
804
|
+
this.claimRedialPeers = new Set();
|
|
761
805
|
/**
|
|
762
|
-
* Single-flight guard for {@link reconcileControlCohort}.
|
|
763
|
-
*
|
|
764
|
-
*
|
|
765
|
-
*
|
|
806
|
+
* Single-flight guard for {@link reconcileControlCohort}. An explicit call, the
|
|
807
|
+
* `self:peer:update` trigger and a scheduled run can fire close together; collapsing
|
|
808
|
+
* concurrent passes into one in-flight run prevents two passes from double-dialing the
|
|
809
|
+
* same siblings (mirrors {@link registerSelfInFlight}). A scheduled run waits for this
|
|
810
|
+
* pass and then starts its own rather than joining it.
|
|
766
811
|
*/
|
|
767
812
|
this.reconcileControlCohortInFlight = null;
|
|
768
813
|
/**
|
|
@@ -785,19 +830,27 @@ export class CadreNode {
|
|
|
785
830
|
* machine in the party is already discarding this node's address as stale.
|
|
786
831
|
*/
|
|
787
832
|
this.lastSelfRecordPublishAt = null;
|
|
833
|
+
/**
|
|
834
|
+
* The newest record this process signed for its own `CadrePeer` row: written by either path
|
|
835
|
+
* of {@link publishSelfRecord}, or handed to a member to write by {@link redeemCadreInvitation}.
|
|
836
|
+
* What {@link publishSelfRecordUnlessCurrent} compares the row against. Per session, cleared
|
|
837
|
+
* with {@link lastSelfRecordPublishAt}: a row carried over from an earlier process is never
|
|
838
|
+
* taken as current.
|
|
839
|
+
*/
|
|
840
|
+
this.lastSignedSelfRecord = null;
|
|
788
841
|
/** Say-once latch for {@link escalateIfSelfRecordStale}; re-armed by the next publish. */
|
|
789
842
|
this.selfRecordStaleWarned = false;
|
|
790
843
|
// ── Write-while-alone re-replication (control-write-ensure-replicated) ──────
|
|
791
844
|
/**
|
|
792
845
|
* Re-replication queue for owner `authorizePeer` writes that committed while
|
|
793
|
-
* this node was alone
|
|
794
|
-
*
|
|
795
|
-
*
|
|
846
|
+
* this node was alone ({@link committedAlone}) — the Optimystic commit was
|
|
847
|
+
* local-only (the block's cohort was this node) and never broadcast. Maps the
|
|
848
|
+
* affected subject peerId → 'authorize', to
|
|
796
849
|
* re-issue once the cohort grows. Removals are NOT tracked here — a delete
|
|
797
850
|
* leaves a `Revocation` tombstone, tracked by {@link pendingRevocations}; a
|
|
798
851
|
* removePeer also clears any queued authorize for the same subject (the row is
|
|
799
|
-
* gone, so re-issuing the insert would be wrong). Drained
|
|
800
|
-
*
|
|
852
|
+
* gone, so re-issuing the insert would be wrong). Drained when a sibling becomes
|
|
853
|
+
* reachable ({@link updateControlSiblingReachability}) by {@link drainPendingControlReplication}.
|
|
801
854
|
*/
|
|
802
855
|
this.pendingPeerWrites = new Map();
|
|
803
856
|
/**
|
|
@@ -821,14 +874,6 @@ export class CadreNode {
|
|
|
821
874
|
* exactly a "removals from before this lifetime" case).
|
|
822
875
|
*/
|
|
823
876
|
this.reissuedHeldRevocations = false;
|
|
824
|
-
/**
|
|
825
|
-
* Whether this process has seen the singleton `Revocation` ledger marker filed — its own
|
|
826
|
-
* `'opened'` or an `'already-open'` answer, in {@link openRevocationLedgerIfDue}. A
|
|
827
|
-
* record of work done, not a cached authorization answer: the marker can never be
|
|
828
|
-
* deleted (`NoDelete`), so this can only go stale if the database itself is replaced,
|
|
829
|
-
* which is why stop() clears it with the other per-process replication state.
|
|
830
|
-
*/
|
|
831
|
-
this.revocationLedgerOpened = false;
|
|
832
877
|
/** This node's own `CadrePeer` self-write committed local-only (re-touched on growth). */
|
|
833
878
|
this.pendingSelfPeerWrite = false;
|
|
834
879
|
/** This node's own `DeviceToken` self-write committed local-only (re-touched on growth). */
|
|
@@ -843,14 +888,24 @@ export class CadreNode {
|
|
|
843
888
|
/** Single-flight guard for the re-replication drain (mirrors {@link registerSelfInFlight}). */
|
|
844
889
|
this.drainControlReplicationInFlight = null;
|
|
845
890
|
/**
|
|
846
|
-
*
|
|
847
|
-
*
|
|
848
|
-
*
|
|
891
|
+
* Connected control peers whose identify says they serve the control database
|
|
892
|
+
* ({@link onControlPeerIdentified}): the peers Optimystic can count into a control block's
|
|
893
|
+
* cohort, before its membership check. A peer joins on its first `peer:identify` of a
|
|
894
|
+
* connected period, not on `connection:open`: until it identifies, every control block's
|
|
895
|
+
* cohort is this node alone. It leaves when its last connection closes. Read through
|
|
896
|
+
* {@link hasReachableControlSibling}, which applies the membership check.
|
|
849
897
|
*/
|
|
850
|
-
this.
|
|
851
|
-
/**
|
|
852
|
-
|
|
853
|
-
|
|
898
|
+
this.identifiedControlPeers = new Set();
|
|
899
|
+
/**
|
|
900
|
+
* {@link hasReachableControlSibling} at its last evaluation, so the drain fires on the
|
|
901
|
+
* false→true edge only (the earliest point a re-issue reaches the cohort), not on every later
|
|
902
|
+
* identify. Re-armed to false when no reachable sibling remains
|
|
903
|
+
* ({@link updateControlSiblingReachability}).
|
|
904
|
+
*/
|
|
905
|
+
this.controlSiblingReachable = false;
|
|
906
|
+
/** `peer:identify` listener: {@link onControlPeerIdentified} (teardown in {@link stopRecordRefresh}). */
|
|
907
|
+
this.controlPeerIdentifyHandler = null;
|
|
908
|
+
/** `connection:close` listener: {@link onControlConnectionClosed} (teardown in {@link stopRecordRefresh}). */
|
|
854
909
|
this.controlConnectionCloseHandler = null;
|
|
855
910
|
this.config = config;
|
|
856
911
|
this.hostUnclaimedStrands = config.hostUnclaimedStrands ?? config.profile === 'storage';
|
|
@@ -1013,7 +1068,7 @@ export class CadreNode {
|
|
|
1013
1068
|
// closed before any network bring-up, and the retained dial targets are
|
|
1014
1069
|
// loaded before the first reconcile pass could consult them.
|
|
1015
1070
|
this.initializeBootstrapPeerStore();
|
|
1016
|
-
// The remembered cross-party joins, which the strand watcher built below
|
|
1071
|
+
// The remembered cross-party joins, which the strand watcher built below reads
|
|
1017
1072
|
// beside the control rows. Before network bring-up, so a mis-scoped injected store
|
|
1018
1073
|
// fails closed like the two above.
|
|
1019
1074
|
this.initializeJoinedStrandStore();
|
|
@@ -1092,6 +1147,20 @@ export class CadreNode {
|
|
|
1092
1147
|
// rest of start() and the refresh early-returns while it is — a write in that
|
|
1093
1148
|
// window is covered by start's own pass below.
|
|
1094
1149
|
this.controlDatabase.setMembershipChangeListener((reason) => this.refreshMembershipGate(reason));
|
|
1150
|
+
// The gate watch: a change to any membership table, replicated in or committed here,
|
|
1151
|
+
// re-materializes the gate and starts a reconcile pass. Separate from the pass so a
|
|
1152
|
+
// replicated revocation does not wait behind a pass spending its dial budget on an
|
|
1153
|
+
// unreachable sibling. The watch handler only arms a timer, so nothing runs under the
|
|
1154
|
+
// write lock the membership listener is called with; `onChange` runs from that timer.
|
|
1155
|
+
//
|
|
1156
|
+
// NOTE: a table this `Database` has not yet touched hears local commits only
|
|
1157
|
+
// (`table-change-watch.ts`). The start refresh reads `CadrePeer`, `OwnerKey` and
|
|
1158
|
+
// `Revocation`; the four invitation and grant tables are first read when some row is
|
|
1159
|
+
// admitted through one, and a load the committed-read path served is not kept and
|
|
1160
|
+
// arms no bridge, so a chain row that replicates in before then waits for the next
|
|
1161
|
+
// change on a touched table or the backstop. Nothing depends on a chain row until a
|
|
1162
|
+
// `CadrePeer` or `OwnerKey` row refers to it, and that row is itself such a change.
|
|
1163
|
+
this.membershipGateWatch = this.controlDatabase.watchMembershipTables(() => this.onMembershipGateTablesChanged(), { label: 'membership-gate' });
|
|
1095
1164
|
// Committed-delete seam → write-while-alone revocation queue. Same wiring
|
|
1096
1165
|
// window argument as the membership listener above.
|
|
1097
1166
|
this.controlDatabase.setGuardedDeleteListener((revocation) => this.noteGuardedDelete(revocation));
|
|
@@ -1104,7 +1173,7 @@ export class CadreNode {
|
|
|
1104
1173
|
this.strandWatcher = new StrandWatcher(queryable, {
|
|
1105
1174
|
onStrandAdded: async (strand) => this.handleStrandAdded(strand),
|
|
1106
1175
|
onStrandRemoved: async (strandId) => this.handleStrandRemoved(strandId)
|
|
1107
|
-
}, this.config.strandFilter ?? { mode: 'all' }, this.config.strandWatchInterval ??
|
|
1176
|
+
}, this.config.strandFilter ?? { mode: 'all' }, this.config.strandWatchInterval ?? DEFAULT_STRAND_WATCH_BACKSTOP_MS, this // CadreNode implements SAppIdLookup
|
|
1108
1177
|
);
|
|
1109
1178
|
t0 = performance.now();
|
|
1110
1179
|
await this.strandWatcher.start();
|
|
@@ -1174,8 +1243,8 @@ export class CadreNode {
|
|
|
1174
1243
|
timing('[start] total: %dms', Math.round(performance.now() - tTotal));
|
|
1175
1244
|
log('CadreNode started successfully');
|
|
1176
1245
|
// Wire the control-connection growth listeners immediately (not via the
|
|
1177
|
-
// delayed refresh path) so no early `
|
|
1178
|
-
// write-while-alone re-replication drain must fire
|
|
1246
|
+
// delayed refresh path) so no early `peer:identify` is missed — the
|
|
1247
|
+
// write-while-alone re-replication drain must fire when the first sibling is reachable.
|
|
1179
1248
|
this.wireControlConnectionListeners();
|
|
1180
1249
|
// Arm the control network's peer-join block catch-up now that the control
|
|
1181
1250
|
// DB (which the push-time membership gate reads) is up. Armed even when
|
|
@@ -1524,7 +1593,7 @@ export class CadreNode {
|
|
|
1524
1593
|
`but this node serves party ${partyId} — refusing to offer another party's joined strands`);
|
|
1525
1594
|
}
|
|
1526
1595
|
this.joinedStrandStore = store;
|
|
1527
|
-
this.joinedStrands = new JoinedStrandSession(store, this.createPartyJoinedStrandLedger());
|
|
1596
|
+
this.joinedStrands = new JoinedStrandSession(store, this.createPartyJoinedStrandLedger(), () => this.strandWatcher?.refresh());
|
|
1528
1597
|
}
|
|
1529
1598
|
/**
|
|
1530
1599
|
* Construct (or adopt) the node-local strand network state (see
|
|
@@ -1665,8 +1734,8 @@ export class CadreNode {
|
|
|
1665
1734
|
}
|
|
1666
1735
|
/**
|
|
1667
1736
|
* Arm the CONTROL network's peer-join block catch-up: push every block in the
|
|
1668
|
-
* control database's own raw store to each
|
|
1669
|
-
*
|
|
1737
|
+
* control database's own raw store to each AUTHORIZED member on each
|
|
1738
|
+
* (re)connection, skipping what it is known to hold. This is what physically replicates
|
|
1670
1739
|
* control blocks committed while the writer was alone — the named
|
|
1671
1740
|
* collection-header blocks written once at genesis above all, whose revision
|
|
1672
1741
|
* never moves again, so no later commit ever carries them to a member that
|
|
@@ -1675,7 +1744,9 @@ export class CadreNode {
|
|
|
1675
1744
|
* for peers it knew about before the restart).
|
|
1676
1745
|
*
|
|
1677
1746
|
* Unlike the per-strand instances (see `strand-instance-manager.ts`), pushes
|
|
1678
|
-
* are gated on {@link
|
|
1747
|
+
* are gated on the authorized-member snapshot ({@link authorizedControlPeers},
|
|
1748
|
+
* which {@link refreshAuthorizedControlPeers} materializes from the same rows and
|
|
1749
|
+
* rule {@link isAuthorizedMember} queries), judged at push time: the control
|
|
1679
1750
|
* network's inbound connection gate deliberately admits non-members in
|
|
1680
1751
|
* several states (seed delivery to an un-enrolled node, an open enrollment
|
|
1681
1752
|
* window, an outstanding invitation, configured bootstrap/relay peers), and
|
|
@@ -1691,22 +1762,18 @@ export class CadreNode {
|
|
|
1691
1762
|
* The one non-member the gate admits is a device whose invitation
|
|
1692
1763
|
* redemption this member is admitting ({@link admitRedeemingDevice}).
|
|
1693
1764
|
*
|
|
1694
|
-
* No-ops (logged) when
|
|
1695
|
-
*
|
|
1696
|
-
* no way to dial.
|
|
1765
|
+
* No-ops (logged) when {@link controlBackfillStorage} answers nothing (disabled,
|
|
1766
|
+
* or no control storage configured) or the node exposes no key network — the
|
|
1767
|
+
* backfill would have nothing to read or no way to dial.
|
|
1697
1768
|
*/
|
|
1698
1769
|
startControlBackfill() {
|
|
1699
|
-
if (
|
|
1770
|
+
if (!this.controlNode) {
|
|
1700
1771
|
return;
|
|
1701
1772
|
}
|
|
1702
|
-
|
|
1703
|
-
// idempotent (it returns the same wrapped store `buildControlNodeOptions` already
|
|
1704
|
-
// resolved), so the catch-up cannot be silently disarmed by a future reordering that
|
|
1705
|
-
// moves this call ahead of the one that populated the field.
|
|
1706
|
-
const storage = this.resolveControlStorage();
|
|
1773
|
+
const storage = this.controlBackfillStorage();
|
|
1707
1774
|
const keyNetwork = this.controlNode.keyNetwork;
|
|
1708
1775
|
if (!storage || !keyNetwork) {
|
|
1709
|
-
log('Control peer-join block catch-up is inert: %s', !storage ? 'no control storage configured' : 'control node exposes no keyNetwork');
|
|
1776
|
+
log('Control peer-join block catch-up is inert: %s', !storage ? 'disabled, or no control storage configured' : 'control node exposes no keyNetwork');
|
|
1710
1777
|
return;
|
|
1711
1778
|
}
|
|
1712
1779
|
const networkName = this.controlNetworkName();
|
|
@@ -1719,7 +1786,11 @@ export class CadreNode {
|
|
|
1719
1786
|
// under — derived from the same networkName binding the node options
|
|
1720
1787
|
// used, never re-spelled here.
|
|
1721
1788
|
protocolPrefix: `/optimystic/${networkName}`,
|
|
1722
|
-
|
|
1789
|
+
reportBlockHeld: this.controlNode.reportBlockHeld,
|
|
1790
|
+
// The membership snapshot, not {@link isControlNetworkMember}: that predicate admits
|
|
1791
|
+
// everyone while the snapshot is empty and admits bootstrap/relay peers outright, and
|
|
1792
|
+
// neither may receive the control store.
|
|
1793
|
+
authorizePeer: async (peerId) => this.admittingDevices.has(peerId) || this.authorizedControlPeers.has(peerId)
|
|
1723
1794
|
}, {
|
|
1724
1795
|
// A shorter settle window than the strand default (1000 ms): the control
|
|
1725
1796
|
// store is small (a party's membership — dozens of blocks), a re-push is
|
|
@@ -1774,7 +1845,7 @@ export class CadreNode {
|
|
|
1774
1845
|
const release = this.countAdmittingDevice(peerId);
|
|
1775
1846
|
try {
|
|
1776
1847
|
const result = await backfill.forceCatchUpPeer(peerIdFromString(peerId));
|
|
1777
|
-
log('Device %s caught up before its admission: offered=%d accepted=%d rejected=%d', peerId, result.offered, result.accepted, result.rejected.length);
|
|
1848
|
+
log('Device %s caught up before its admission: offered=%d alreadyHeld=%d accepted=%d rejected=%d', peerId, result.offered, result.alreadyHeld, result.accepted, result.rejected.length);
|
|
1778
1849
|
return release;
|
|
1779
1850
|
}
|
|
1780
1851
|
catch (error) {
|
|
@@ -1838,6 +1909,19 @@ export class CadreNode {
|
|
|
1838
1909
|
this.controlStorage = wrapStorageWithCache(resolved, scope);
|
|
1839
1910
|
return this.controlStorage;
|
|
1840
1911
|
}
|
|
1912
|
+
/**
|
|
1913
|
+
* The store the control network's peer-join block catch-up reads, or `undefined` when this
|
|
1914
|
+
* runtime does not arm it: `controlBackfill.enabled: false`, or no control storage. Both of
|
|
1915
|
+
* its readers ask here so they cannot disagree: {@link startControlBackfill} arms on it, and
|
|
1916
|
+
* {@link buildControlNodeOptions} turns Optimystic's rebalance reaction off on it.
|
|
1917
|
+
*
|
|
1918
|
+
* Through the resolver, not the {@link controlStorage} memo field: the resolver is
|
|
1919
|
+
* idempotent, so the catch-up cannot be silently disarmed by a future reordering that moves
|
|
1920
|
+
* a read ahead of the one that populated the field.
|
|
1921
|
+
*/
|
|
1922
|
+
controlBackfillStorage() {
|
|
1923
|
+
return this.config.controlBackfill?.enabled === false ? undefined : this.resolveControlStorage();
|
|
1924
|
+
}
|
|
1841
1925
|
/**
|
|
1842
1926
|
* Warn the operator when `network.announceAddrs` will silently cost this node the
|
|
1843
1927
|
* relay reachability it also configured.
|
|
@@ -1940,6 +2024,19 @@ export class CadreNode {
|
|
|
1940
2024
|
cohortQueryTimeoutMs: declaredCohortReadDeadlineMs(network)
|
|
1941
2025
|
}),
|
|
1942
2026
|
arachnode: { enableRingZulu: profile === 'storage' },
|
|
2027
|
+
// One join catch-up per node, for the strand node's reason (`strand-instance-manager.ts`,
|
|
2028
|
+
// at its `rebalance` option): Optimystic's rebalance growth arm would push a joining
|
|
2029
|
+
// member each control block again, one request per block. Every peer the growth arm
|
|
2030
|
+
// pushes to, the control catch-up pushes to as well, except the ones it deliberately
|
|
2031
|
+
// withholds the control store from: the cohort predicate (`isNetworkMember` below) admits
|
|
2032
|
+
// everyone while the membership snapshot is empty, and bootstrap/relay peers outright.
|
|
2033
|
+
// With the catch-up disabled the reaction stays on.
|
|
2034
|
+
//
|
|
2035
|
+
// NOTE: no ring shifts and no reclaimable shed blocks on this node either, unused today.
|
|
2036
|
+
// If storage nodes ever need them, turn the reaction back on: the catch-up already reports
|
|
2037
|
+
// its accepted pushes to the rebalance monitor (`reportBlockHeld`); measure whether the
|
|
2038
|
+
// growth arm still re-sends.
|
|
2039
|
+
...(this.controlBackfillStorage() && { rebalance: { enabled: false } }),
|
|
1943
2040
|
...(identityKey && { privateKey: identityKey }),
|
|
1944
2041
|
// With `network.relayAddrs` set, built with libp2p's relay discovery off (given a
|
|
1945
2042
|
// transport list to wrap — `node-transports.ts`), so the supervisor
|
|
@@ -2002,10 +2099,8 @@ export class CadreNode {
|
|
|
2002
2099
|
// Who this node SENDS control writes to (cohort and coordinator choice), where
|
|
2003
2100
|
// `authorizeInboundStream` is who may send to it. Without it Optimystic picks cohort
|
|
2004
2101
|
// members from the protocols a peer's peer-store record lists, which any connected
|
|
2005
|
-
// peer can fill.
|
|
2006
|
-
|
|
2007
|
-
// whose verified redemption is being written, which must hold the rows that admit it.
|
|
2008
|
-
isNetworkMember: (peerId) => this.isControlNetworkMember(peerId) || this.admittingDevices.has(peerId)
|
|
2102
|
+
// peer can fill.
|
|
2103
|
+
isNetworkMember: (peerId) => this.isControlCohortMember(peerId)
|
|
2009
2104
|
};
|
|
2010
2105
|
return nodeOptions;
|
|
2011
2106
|
}
|
|
@@ -2085,8 +2180,8 @@ export class CadreNode {
|
|
|
2085
2180
|
* (it redeems here by seating the row from its bundle, after which check 7
|
|
2086
2181
|
* admits) or from a sibling whose membership row is still replicating (check 5
|
|
2087
2182
|
* admits once it lands; if it has not landed by the deadline, the connection
|
|
2088
|
-
* is closed and
|
|
2089
|
-
* dials are never gated). The protocol guard refuses all of them every
|
|
2183
|
+
* is closed and the reconcile pass the close starts on either side re-forms it,
|
|
2184
|
+
* since outbound dials are never gated). The protocol guard refuses all of them every
|
|
2090
2185
|
* members-only protocol meanwhile. A circuit-relay reservation admitted by
|
|
2091
2186
|
* {@link admitControlRelayReservation} disarms the deadline (see
|
|
2092
2187
|
* `membership-connection-gater.ts` → "The relay-reservation seam").
|
|
@@ -2310,6 +2405,17 @@ export class CadreNode {
|
|
|
2310
2405
|
}
|
|
2311
2406
|
return this.authorizedControlPeers.size === 0 || this.authorizedControlPeers.has(remotePeerId);
|
|
2312
2407
|
}
|
|
2408
|
+
/**
|
|
2409
|
+
* Whether this node sends control writes to `remotePeerId`: Optimystic's `isNetworkMember`
|
|
2410
|
+
* for the control node (cohort and coordinator choice), and the membership half of
|
|
2411
|
+
* {@link hasReachableControlSibling}. The stream gate's rule ({@link isControlNetworkMember}),
|
|
2412
|
+
* unlogged, because Optimystic asks several times per peer per cohort assembly and counts
|
|
2413
|
+
* refusals on its own debug line; plus a device whose verified redemption is being written
|
|
2414
|
+
* ({@link admittingDevices}), which must hold the rows that admit it.
|
|
2415
|
+
*/
|
|
2416
|
+
isControlCohortMember(remotePeerId) {
|
|
2417
|
+
return this.isControlNetworkMember(remotePeerId) || this.admittingDevices.has(remotePeerId);
|
|
2418
|
+
}
|
|
2313
2419
|
/**
|
|
2314
2420
|
* Per-stream authorization for the protocol guard's `members-snapshot` class
|
|
2315
2421
|
* (`control-protocol-guard.ts`): the four Optimystic control-DB protocols
|
|
@@ -2349,7 +2455,8 @@ export class CadreNode {
|
|
|
2349
2455
|
* this listener with its write lock HELD (`notifyMembershipChanged`), so a retrying
|
|
2350
2456
|
* read here would sleep its backoff holding the lock and stall every other local
|
|
2351
2457
|
* writer. Nothing is lost — this refresh already keeps the previous snapshot on
|
|
2352
|
-
* failure and is re-driven by the next membership
|
|
2458
|
+
* failure and is re-driven by the next membership change (local or replicated, through
|
|
2459
|
+
* the gate watch) and by the reconcile pass's backstop.
|
|
2353
2460
|
*
|
|
2354
2461
|
* The trusted-owner anchor sync ({@link syncOwnerAnchor}) runs FIRST, so the snapshot
|
|
2355
2462
|
* built here judges `CadrePeer` rows against the anchor as the `OwnerKey` table now
|
|
@@ -2424,15 +2531,20 @@ export class CadreNode {
|
|
|
2424
2531
|
// commit is what would authorize it), and the gate re-judges at push time.
|
|
2425
2532
|
//
|
|
2426
2533
|
// NOTE: every refresh re-arms every connected peer that is not yet caught
|
|
2427
|
-
// up
|
|
2428
|
-
// connection that is admitted but never authorized — a configured
|
|
2534
|
+
// up. A connection that is admitted but never authorized — a configured
|
|
2429
2535
|
// bootstrap or relay peer, the steady state for a NAT'd node — is
|
|
2430
|
-
// therefore re-judged on every
|
|
2431
|
-
//
|
|
2432
|
-
//
|
|
2433
|
-
//
|
|
2434
|
-
//
|
|
2536
|
+
// therefore re-judged on every refresh, forever: every local membership
|
|
2537
|
+
// write, every replicated membership change (the gate watch), and every
|
|
2538
|
+
// reconcile pass, down to the 2-minute backstop on an idle party. Each
|
|
2539
|
+
// such pass is one timer and a lookup in the snapshot just set (the gate
|
|
2540
|
+
// reads no control database). If a node ever holds many such connections,
|
|
2541
|
+
// or refreshes get frequent, skip re-arming peers whose last pass was
|
|
2542
|
+
// denied under the SAME membership snapshot instead of re-arming
|
|
2543
|
+
// unconditionally.
|
|
2435
2544
|
void this.controlBackfill?.scheduleConnectedPeers();
|
|
2545
|
+
// Same join order for the growth edge: a sibling that identified before this commit
|
|
2546
|
+
// authorized it becomes reachable here.
|
|
2547
|
+
this.updateControlSiblingReachability();
|
|
2436
2548
|
await this.hangUpRevokedControlPeers(rows, loadTombstones);
|
|
2437
2549
|
await this.noteOwnRemoval(rows, loadTombstones);
|
|
2438
2550
|
}
|
|
@@ -2619,9 +2731,10 @@ export class CadreNode {
|
|
|
2619
2731
|
return this.bootstrapPeerIds;
|
|
2620
2732
|
}
|
|
2621
2733
|
/**
|
|
2622
|
-
* What the strand watcher
|
|
2734
|
+
* What the strand watcher reads: this party's control rows plus a row per cross-party
|
|
2623
2735
|
* join (see {@link joinedStrands}), so a joined strand is offered and relaunched on
|
|
2624
|
-
* the same path as the party's own.
|
|
2736
|
+
* the same path as the party's own. It runs on a change to either table; the
|
|
2737
|
+
* machine-local joins wake it through {@link JoinedStrandSession}'s change callback.
|
|
2625
2738
|
*/
|
|
2626
2739
|
createStrandQueryable() {
|
|
2627
2740
|
return {
|
|
@@ -2633,7 +2746,14 @@ export class CadreNode {
|
|
|
2633
2746
|
log('Querying strands from control database');
|
|
2634
2747
|
const control = await this.controlDatabase.queryStrands();
|
|
2635
2748
|
return this.joinedStrands ? await this.joinedStrands.withControlRows(control) : control;
|
|
2636
|
-
}
|
|
2749
|
+
},
|
|
2750
|
+
watchChanges: (onChange, options) => {
|
|
2751
|
+
if (!this.controlDatabase) {
|
|
2752
|
+
throw new Error('CadreNode must initialize its control database before watching its strand tables');
|
|
2753
|
+
}
|
|
2754
|
+
return this.controlDatabase.watchTables(['CadreControl.Strand', 'CadreControl.JoinedStrand'], onChange, options);
|
|
2755
|
+
},
|
|
2756
|
+
lastReadPartial: () => this.joinedStrands?.lastReadPartial() ?? false
|
|
2637
2757
|
};
|
|
2638
2758
|
}
|
|
2639
2759
|
/**
|
|
@@ -2648,10 +2768,6 @@ export class CadreNode {
|
|
|
2648
2768
|
return this.controlDatabase;
|
|
2649
2769
|
};
|
|
2650
2770
|
return {
|
|
2651
|
-
// NOTE: one more control read per watcher poll (5 s), and for a party that never
|
|
2652
|
-
// joined anything it reads a never-written block, which consults the cohort. If poll
|
|
2653
|
-
// cost shows up in a device profile, cache the list and re-read it on the reconcile
|
|
2654
|
-
// cadence.
|
|
2655
2771
|
list: () => database().queryJoinedStrands(),
|
|
2656
2772
|
names: async (strandId) => (await database().queryStrand(strandId)) !== null
|
|
2657
2773
|
|| (await database().queryJoinedStrand(strandId)) !== null,
|
|
@@ -2729,26 +2845,30 @@ export class CadreNode {
|
|
|
2729
2845
|
* after start (non-blocking). {@link registerSelf} is idempotent and safely
|
|
2730
2846
|
* no-ops when it cannot yet sign/insert (e.g. owner key not installed),
|
|
2731
2847
|
* so the timer is harmless even when registration only becomes possible later.
|
|
2848
|
+
* A record this process already put in place that is still current (a founding owner's
|
|
2849
|
+
* explicit publish, a redeemed invitation's admission, a restart's first-growth re-touch) is
|
|
2850
|
+
* not written again ({@link publishSelfRecordUnlessCurrent}). Whether the node is connected
|
|
2851
|
+
* yet does not matter: a device that redeems within a second of starting is.
|
|
2732
2852
|
*/
|
|
2733
2853
|
scheduleSelfRegistration() {
|
|
2734
2854
|
this.selfRegistrationTimer = setTimeout(() => {
|
|
2735
2855
|
void (async () => {
|
|
2736
2856
|
try {
|
|
2737
|
-
await this.
|
|
2857
|
+
await this.publishSelfRecordUnlessCurrent();
|
|
2738
2858
|
}
|
|
2739
2859
|
catch (error) {
|
|
2740
|
-
// Background task — a failed publish must not crash the node. A
|
|
2741
|
-
//
|
|
2742
|
-
//
|
|
2743
|
-
//
|
|
2860
|
+
// Background task — a failed publish must not crash the node. A publish that loses
|
|
2861
|
+
// its row to another writer — the owner's re-touch, or a restarted machine's own row
|
|
2862
|
+
// replicating in while the update runs — is re-run by the write funnel over the row
|
|
2863
|
+
// it finds (`ControlDatabase.updateSelfPeerRecord`); one that lost past the retry
|
|
2864
|
+
// budget fails here, and the next connected reconcile pass is the fallback
|
|
2744
2865
|
// (publishFirstSelfRecord).
|
|
2745
2866
|
log('Self-registration failed: %o', error);
|
|
2746
2867
|
}
|
|
2747
|
-
// Wire the ongoing refresh
|
|
2748
|
-
// an eager reconcile pass
|
|
2749
|
-
// interval was just armed inside startRecordRefresh).
|
|
2868
|
+
// Wire the ongoing refresh and the control-cohort reconcile schedule, then
|
|
2869
|
+
// run an eager reconcile pass now that the node has settled.
|
|
2750
2870
|
this.startRecordRefresh();
|
|
2751
|
-
|
|
2871
|
+
this.controlReconcileRuns?.trigger(0);
|
|
2752
2872
|
})();
|
|
2753
2873
|
}, 1000); // Small delay to ensure node is fully started
|
|
2754
2874
|
// Node-only: don't keep the event loop alive solely for this timer.
|
|
@@ -2820,16 +2940,15 @@ export class CadreNode {
|
|
|
2820
2940
|
return 'skipped';
|
|
2821
2941
|
}
|
|
2822
2942
|
// Collected only past that skip, which a node waiting for its row replicated in takes on
|
|
2823
|
-
//
|
|
2943
|
+
// reconcile pass made while a sibling is reachable ({@link publishFirstSelfRecord}).
|
|
2824
2944
|
const addrs = await this.collectSelfAddrs();
|
|
2825
2945
|
if (!existing && ownerService) {
|
|
2826
2946
|
// First-time row: requires an owner signature (the node is its own
|
|
2827
2947
|
// owner). insertSelfPeerRecord throws if no owner key is present.
|
|
2828
2948
|
const record = this.signSelfRecord(peerId, signingKey, addrs, null);
|
|
2949
|
+
const aloneBefore = this.committedAlone();
|
|
2829
2950
|
if (await ownerService.insertSelfPeerRecord(record)) {
|
|
2830
|
-
|
|
2831
|
-
this.pendingSelfPeerWrite = true;
|
|
2832
|
-
this.noteSelfRecordPublished();
|
|
2951
|
+
this.noteSelfPeerWritten(record, aloneBefore);
|
|
2833
2952
|
log('registerSelf: inserted own CadrePeer record (owner-signed, updatedAt=%d, %d addrs, sig=%s…)', record.updatedAt, addrs.length, record.sig.slice(0, 16));
|
|
2834
2953
|
return 'inserted';
|
|
2835
2954
|
}
|
|
@@ -2843,43 +2962,97 @@ export class CadreNode {
|
|
|
2843
2962
|
// first publish", add a fourth SelfRegistrationOutcome rather than re-labelling.
|
|
2844
2963
|
log('registerSelf: own CadrePeer row appeared mid-publish (concurrent authorize); self-updating to carry the signature');
|
|
2845
2964
|
}
|
|
2846
|
-
//
|
|
2847
|
-
//
|
|
2848
|
-
// the
|
|
2849
|
-
//
|
|
2850
|
-
//
|
|
2851
|
-
//
|
|
2852
|
-
|
|
2853
|
-
|
|
2854
|
-
|
|
2855
|
-
const current = existing ?? await this.controlDatabase.queryPeerRecord(peerId);
|
|
2856
|
-
if (!current) {
|
|
2965
|
+
// The update body reads the row under the write lock and signs over what it finds, on
|
|
2966
|
+
// every attempt — so neither `existing` (a read that predates this publish) nor a first
|
|
2967
|
+
// attempt that lost the row to another writer decides the `UpdatedAt` the strictly-greater
|
|
2968
|
+
// self-update rule is measured against, and a removal landing mid-publish re-reads as
|
|
2969
|
+
// absent instead of matching no rows under a 'refreshed'. The record it returns is the
|
|
2970
|
+
// one in the row.
|
|
2971
|
+
const aloneBefore = this.committedAlone();
|
|
2972
|
+
const record = await this.controlDatabase.updateSelfPeerRecord(peerId, (current) => this.signSelfRecord(peerId, signingKey, addrs, current));
|
|
2973
|
+
if (record === null) {
|
|
2857
2974
|
// Two causes, and the second is a steady state rather than a race: the row really
|
|
2858
2975
|
// vanished mid-publish (next refresh re-inserts), or this node was REVOKED — its row
|
|
2859
2976
|
// is physically present, so insertCadrePeer's in-lock existence check no-op'd above,
|
|
2860
|
-
// but
|
|
2861
|
-
//
|
|
2862
|
-
//
|
|
2977
|
+
// but the body reads it as absent because its stamp is retired. The revoked case
|
|
2978
|
+
// repeats every heartbeat and never re-inserts; that is intended (a revoked node has
|
|
2979
|
+
// nothing to publish), so this stays a skip, not an error.
|
|
2863
2980
|
log('registerSelf: own CadrePeer row not readable (vanished mid-publish, or this node is revoked and its stamp retired); skipping');
|
|
2864
2981
|
return 'skipped';
|
|
2865
2982
|
}
|
|
2866
|
-
|
|
2867
|
-
await this.controlDatabase.updateSelfPeerRecord(record);
|
|
2868
|
-
if (this.committedAlone())
|
|
2869
|
-
this.pendingSelfPeerWrite = true;
|
|
2870
|
-
this.noteSelfRecordPublished();
|
|
2983
|
+
this.noteSelfPeerWritten(record, aloneBefore);
|
|
2871
2984
|
log('registerSelf: refreshed own CadrePeer record (updatedAt=%d, %d addrs, sig=%s…)', record.updatedAt, addrs.length, record.sig.slice(0, 16));
|
|
2872
2985
|
return 'refreshed';
|
|
2873
2986
|
}
|
|
2874
2987
|
/**
|
|
2875
|
-
*
|
|
2876
|
-
*
|
|
2877
|
-
*
|
|
2988
|
+
* Bookkeeping for a committed write of this node's own `CadrePeer` row by {@link publishSelfRecord}:
|
|
2989
|
+
* `record` is the one the row holds (for the update path, the one its body signed on the
|
|
2990
|
+
* attempt that committed, not the first it signed). Queued for the drain's re-touch
|
|
2991
|
+
* ({@link pendingSelfPeerWrite}) when no sibling was reachable when the write started or when
|
|
2992
|
+
* it ended: a sibling that identifies while the write is out may do so after the write's cohort
|
|
2993
|
+
* was chosen, and the first-growth drain skips a row holding {@link lastSignedSelfRecord} unless
|
|
2994
|
+
* this flag is set. Both fields are set here together so no write can leave the record without
|
|
2995
|
+
* the flag.
|
|
2996
|
+
*/
|
|
2997
|
+
noteSelfPeerWritten(record, aloneBefore) {
|
|
2998
|
+
if (aloneBefore || this.committedAlone())
|
|
2999
|
+
this.pendingSelfPeerWrite = true;
|
|
3000
|
+
this.lastSignedSelfRecord = record;
|
|
3001
|
+
this.noteSelfRecordPublished();
|
|
3002
|
+
}
|
|
3003
|
+
/**
|
|
3004
|
+
* Record that this node's own address record is in place as of `at`, and re-arm the
|
|
3005
|
+
* stale-record warning. Called for BOTH publishing paths of {@link publishSelfRecord}
|
|
3006
|
+
* ({@link noteSelfPeerWritten}), and by {@link publishSelfRecordUnlessCurrent} with the stamp
|
|
3007
|
+
* of a record already in place — a `skipped` publish wrote nothing and found nothing, so it
|
|
3008
|
+
* must not refresh the stamp.
|
|
2878
3009
|
*/
|
|
2879
|
-
noteSelfRecordPublished() {
|
|
2880
|
-
this.lastSelfRecordPublishAt =
|
|
3010
|
+
noteSelfRecordPublished(at = Date.now()) {
|
|
3011
|
+
this.lastSelfRecordPublishAt = at;
|
|
2881
3012
|
this.selfRecordStaleWarned = false;
|
|
2882
3013
|
}
|
|
3014
|
+
/**
|
|
3015
|
+
* {@link registerSelf}, unless the own row already holds the record this process last signed
|
|
3016
|
+
* ({@link lastSignedSelfRecord}) and that record is current ({@link isCurrentSelfRecord});
|
|
3017
|
+
* then nothing is written and the record counts as published at its own stamp. For a publish
|
|
3018
|
+
* that puts a record in place (the boot timer, the reconcile pass's first publish, the
|
|
3019
|
+
* first-growth drain), never the heartbeat, which exists to re-stamp freshness. `'current'`
|
|
3020
|
+
* is this method's own answer; {@link registerSelf} always writes.
|
|
3021
|
+
*/
|
|
3022
|
+
async publishSelfRecordUnlessCurrent() {
|
|
3023
|
+
if (this.registerSelfInFlight) {
|
|
3024
|
+
return this.registerSelfInFlight;
|
|
3025
|
+
}
|
|
3026
|
+
const record = this.lastSignedSelfRecord;
|
|
3027
|
+
if (record !== null && await this.isCurrentSelfRecord(record)) {
|
|
3028
|
+
this.noteSelfRecordPublished(record.updatedAt);
|
|
3029
|
+
log('registerSelf: own CadrePeer record already current (updatedAt=%d); not written again', record.updatedAt);
|
|
3030
|
+
return 'current';
|
|
3031
|
+
}
|
|
3032
|
+
return this.registerSelf();
|
|
3033
|
+
}
|
|
3034
|
+
/**
|
|
3035
|
+
* Is `record` still the one to publish: signed less than half a heartbeat ago, with this
|
|
3036
|
+
* node's profile, over the addresses it would publish now, and the signature the own row
|
|
3037
|
+
* holds? The age bound keeps a record that is not re-written fresh for resolvers: at most half
|
|
3038
|
+
* a heartbeat old here and re-stamped by the heartbeat within one more, so never older than
|
|
3039
|
+
* 1.5 heartbeats (11.25 min) against {@link DEFAULT_PEER_RECORD_MAX_AGE_MS}. Cheap checks
|
|
3040
|
+
* first: the row comes from the membership view, the addresses from the app's resolver.
|
|
3041
|
+
*/
|
|
3042
|
+
async isCurrentSelfRecord(record) {
|
|
3043
|
+
if (Date.now() - record.updatedAt >= DEFAULT_PEER_RECORD_HEARTBEAT_MS / 2 || record.profile !== this.config.profile) {
|
|
3044
|
+
return false;
|
|
3045
|
+
}
|
|
3046
|
+
const row = await this.controlDatabase?.queryPeerRecord(record.peerId);
|
|
3047
|
+
if (row?.sig !== record.sig) {
|
|
3048
|
+
return false;
|
|
3049
|
+
}
|
|
3050
|
+
// Sanitized, as the record a redemption carries is (`signRedeemRequest`); a published
|
|
3051
|
+
// record is not, so one whose list sanitizing would change (over 16 addresses) is never
|
|
3052
|
+
// current and is written as before.
|
|
3053
|
+
const addrs = sanitizeAddrs(await this.collectSelfAddrs());
|
|
3054
|
+
return addrs.length === record.addrs.length && addrs.every((addr, i) => addr === record.addrs[i]);
|
|
3055
|
+
}
|
|
2883
3056
|
/**
|
|
2884
3057
|
* Report an abandoned control write to the embedding app.
|
|
2885
3058
|
*
|
|
@@ -3007,19 +3180,24 @@ export class CadreNode {
|
|
|
3007
3180
|
* process does not learn it was removed ({@link noteOwnRemoval}), and pays that read on every
|
|
3008
3181
|
* pass.
|
|
3009
3182
|
*
|
|
3010
|
-
*
|
|
3011
|
-
*
|
|
3012
|
-
*
|
|
3013
|
-
* then never see rows written while it
|
|
3014
|
-
*
|
|
3015
|
-
*
|
|
3183
|
+
* Only while a sibling is reachable ({@link hasReachableControlSibling}): a row written
|
|
3184
|
+
* elsewhere arrives from a sibling, and an owner's publish committed alone is local-only and
|
|
3185
|
+
* forks its own history — a restarted owner whose embedder runs a pass before it reconnects
|
|
3186
|
+
* would write its row on the stale side of the fork, and then never see rows written while it
|
|
3187
|
+
* was down (`cadre-invite-any-member` runs that pass and fails every time without this
|
|
3188
|
+
* condition). A connection is not enough: until the sibling identifies, the write's cohort is
|
|
3189
|
+
* this node alone. The boot-time publish and the first-growth drain already try at the
|
|
3190
|
+
* node's lone and first-sibling moments; this retries on the passes after.
|
|
3191
|
+
*
|
|
3192
|
+
* A machine admitted by invitation finds its row already holding the record its redemption
|
|
3193
|
+
* carried, and writes nothing ({@link publishSelfRecordUnlessCurrent}).
|
|
3016
3194
|
*/
|
|
3017
3195
|
async publishFirstSelfRecord() {
|
|
3018
3196
|
if (this.lastSelfRecordPublishAt !== null || this.isAwaitingClaim() || this.removal !== null
|
|
3019
|
-
|| this.
|
|
3197
|
+
|| !this.hasReachableControlSibling()) {
|
|
3020
3198
|
return;
|
|
3021
3199
|
}
|
|
3022
|
-
const outcome = await this.
|
|
3200
|
+
const outcome = await this.publishSelfRecordUnlessCurrent().catch((error) => {
|
|
3023
3201
|
log('reconcileControlCohort: first self-record publish failed (continuing): %o', error);
|
|
3024
3202
|
return 'skipped';
|
|
3025
3203
|
});
|
|
@@ -3055,15 +3233,20 @@ export class CadreNode {
|
|
|
3055
3233
|
this.escalateIfSelfRecordStale(reason, error);
|
|
3056
3234
|
});
|
|
3057
3235
|
};
|
|
3058
|
-
const
|
|
3059
|
-
|
|
3060
|
-
|
|
3236
|
+
const backstopMs = this.controlReconcileBackstopMs();
|
|
3237
|
+
// The pass's schedule. A run never joins a pass already in flight: that pass may have
|
|
3238
|
+
// read its rows before the change that woke this run, so joining it would drop the
|
|
3239
|
+
// change until the backstop. It waits for the pass, then starts its own.
|
|
3240
|
+
this.controlReconcileRuns = scheduleRuns(async () => {
|
|
3241
|
+
await this.reconcileControlCohortInFlight?.catch(() => undefined);
|
|
3242
|
+
await this.reconcileControlCohort().catch((error) => log('Control-cohort reconcile (scheduled) failed: %o', error));
|
|
3243
|
+
}, { label: 'control-reconcile', backstopMs });
|
|
3061
3244
|
// Address churn re-publishes the self record AND re-checks the control cohort:
|
|
3062
3245
|
// a relay-reservation rotation / NAT change can drop a sibling connection, so
|
|
3063
3246
|
// the next pass should re-observe and re-dial it.
|
|
3064
3247
|
this.selfPeerUpdateHandler = () => {
|
|
3065
3248
|
republish('self:peer:update');
|
|
3066
|
-
|
|
3249
|
+
this.controlReconcileRuns?.trigger(0);
|
|
3067
3250
|
};
|
|
3068
3251
|
this.controlNode.addEventListener('self:peer:update', this.selfPeerUpdateHandler);
|
|
3069
3252
|
// A node AUTHORIZED AFTER IT BOOTED (seeded, claimed, invited) gets its first
|
|
@@ -3083,12 +3266,28 @@ export class CadreNode {
|
|
|
3083
3266
|
// dependable there.
|
|
3084
3267
|
this.recordRefreshTimer = setInterval(() => republish('heartbeat'), DEFAULT_PEER_RECORD_HEARTBEAT_MS);
|
|
3085
3268
|
this.recordRefreshTimer?.unref?.();
|
|
3086
|
-
|
|
3087
|
-
|
|
3088
|
-
|
|
3089
|
-
|
|
3269
|
+
log('Record refresh wired (heartbeat=%dms, cohortReconcileBackstop=%dms)', DEFAULT_PEER_RECORD_HEARTBEAT_MS, backstopMs);
|
|
3270
|
+
}
|
|
3271
|
+
/** The reconcile schedule's backstop: `network.controlCohort.reconcileMs`, default `DEFAULT_CONTROL_COHORT_RECONCILE_MS`. */
|
|
3272
|
+
controlReconcileBackstopMs() {
|
|
3273
|
+
return this.config.network?.controlCohort?.reconcileMs ?? DEFAULT_CONTROL_COHORT_RECONCILE_MS;
|
|
3090
3274
|
}
|
|
3091
|
-
/**
|
|
3275
|
+
/**
|
|
3276
|
+
* The gate watch's run ({@link membershipGateWatch}): re-materialize the gate, so a
|
|
3277
|
+
* replicated revocation hangs up the removed machine now (`hangUpRevokedControlPeers`
|
|
3278
|
+
* ends every refresh), then start a reconcile pass for whatever else the change means
|
|
3279
|
+
* (a new sibling to dial, a tombstoned row to reap). A local membership write refreshes
|
|
3280
|
+
* the gate twice — once from the awaited membership listener, so the writer sees its
|
|
3281
|
+
* change before returning, and once here, 50 ms later. The listener's refresh runs under
|
|
3282
|
+
* the write lock, so its reads take the committed-read path and the view keeps nothing
|
|
3283
|
+
* from them; this refresh is the read that refills the view after the write, which some
|
|
3284
|
+
* later caller would otherwise pay. Accepted: one reload per change, not two.
|
|
3285
|
+
*/
|
|
3286
|
+
async onMembershipGateTablesChanged() {
|
|
3287
|
+
await this.refreshMembershipGate('table-change');
|
|
3288
|
+
this.controlReconcileRuns?.trigger(0);
|
|
3289
|
+
}
|
|
3290
|
+
/** Tear down the self-record refresh timers, the reconcile schedule and the listeners (see {@link cleanup}). */
|
|
3092
3291
|
stopRecordRefresh() {
|
|
3093
3292
|
if (this.selfRegistrationTimer) {
|
|
3094
3293
|
clearTimeout(this.selfRegistrationTimer);
|
|
@@ -3098,24 +3297,30 @@ export class CadreNode {
|
|
|
3098
3297
|
clearInterval(this.recordRefreshTimer);
|
|
3099
3298
|
this.recordRefreshTimer = null;
|
|
3100
3299
|
}
|
|
3101
|
-
if (this.
|
|
3102
|
-
|
|
3103
|
-
this.
|
|
3300
|
+
if (this.controlReconcileRuns) {
|
|
3301
|
+
this.controlReconcileRuns.dispose();
|
|
3302
|
+
this.controlReconcileRuns = null;
|
|
3303
|
+
}
|
|
3304
|
+
if (this.membershipViewReloadTimer) {
|
|
3305
|
+
clearTimeout(this.membershipViewReloadTimer);
|
|
3306
|
+
this.membershipViewReloadTimer = null;
|
|
3104
3307
|
}
|
|
3308
|
+
this.reconcileRedialStreak = 0;
|
|
3105
3309
|
if (this.controlNode) {
|
|
3106
3310
|
if (this.selfPeerUpdateHandler) {
|
|
3107
3311
|
this.controlNode.removeEventListener('self:peer:update', this.selfPeerUpdateHandler);
|
|
3108
3312
|
}
|
|
3109
|
-
if (this.
|
|
3110
|
-
this.controlNode.removeEventListener('
|
|
3313
|
+
if (this.controlPeerIdentifyHandler) {
|
|
3314
|
+
this.controlNode.removeEventListener('peer:identify', this.controlPeerIdentifyHandler);
|
|
3111
3315
|
}
|
|
3112
3316
|
if (this.controlConnectionCloseHandler) {
|
|
3113
3317
|
this.controlNode.removeEventListener('connection:close', this.controlConnectionCloseHandler);
|
|
3114
3318
|
}
|
|
3115
3319
|
}
|
|
3116
3320
|
this.selfPeerUpdateHandler = null;
|
|
3117
|
-
this.
|
|
3321
|
+
this.controlPeerIdentifyHandler = null;
|
|
3118
3322
|
this.controlConnectionCloseHandler = null;
|
|
3323
|
+
this.identifiedControlPeers.clear();
|
|
3119
3324
|
// Reset the write-while-alone re-replication state so a stop()→start() cycle
|
|
3120
3325
|
// re-arms the growth edge and re-runs BOTH one-shot passes. Any queued
|
|
3121
3326
|
// local-only writes are moot once the node is torn down: owner inserts are
|
|
@@ -3123,10 +3328,9 @@ export class CadreNode {
|
|
|
3123
3328
|
// first-growth revocation sweep — which is why the sweep flag must reset too,
|
|
3124
3329
|
// or the second lifetime would skip the sweep and strand exactly the
|
|
3125
3330
|
// committed-alone tombstones it exists to carry.
|
|
3126
|
-
this.
|
|
3331
|
+
this.controlSiblingReachable = false;
|
|
3127
3332
|
this.reconstructedLocalOnlyWrites = false;
|
|
3128
3333
|
this.reissuedHeldRevocations = false;
|
|
3129
|
-
this.revocationLedgerOpened = false;
|
|
3130
3334
|
this.pendingPeerWrites.clear();
|
|
3131
3335
|
this.pendingRevocations.clear();
|
|
3132
3336
|
this.pendingSelfPeerWrite = false;
|
|
@@ -3135,6 +3339,7 @@ export class CadreNode {
|
|
|
3135
3339
|
// from scratch, and carrying the previous lifetime's stamp forward would either
|
|
3136
3340
|
// suppress the stale-record warning or fire it on a node that just came up.
|
|
3137
3341
|
this.lastSelfRecordPublishAt = null;
|
|
3342
|
+
this.lastSignedSelfRecord = null;
|
|
3138
3343
|
this.selfRecordStaleWarned = false;
|
|
3139
3344
|
}
|
|
3140
3345
|
/**
|
|
@@ -3311,26 +3516,37 @@ export class CadreNode {
|
|
|
3311
3516
|
* Run one proactive control-cohort dial pass to keep this node connected to its
|
|
3312
3517
|
* cadre siblings (so the `CadreControl` collections form a replicating cohort).
|
|
3313
3518
|
* Public so the cohort-growth-driven re-replication path
|
|
3314
|
-
* (`control-write-ensure-replicated`)
|
|
3315
|
-
*
|
|
3316
|
-
*
|
|
3519
|
+
* (`control-write-ensure-replicated`), `addDrone`'s callers, `claimNode`,
|
|
3520
|
+
* `redeemCadreInvitation` and tests can run a pass on demand. On its own a pass
|
|
3521
|
+
* runs when something it acts on changes, and otherwise at a backstop
|
|
3522
|
+
* ({@link controlReconcileRuns}): once after start; on `self:peer:update`; when a
|
|
3523
|
+
* membership table changes, local commit or replicated ({@link membershipGateWatch});
|
|
3524
|
+
* when a control connection to an authorized sibling opens or closes
|
|
3525
|
+
* ({@link onControlSiblingConnectionChange}); on the redial and strand-address wakes
|
|
3526
|
+
* a pass arms for itself ({@link armReconcileWake}); and at the backstop,
|
|
3527
|
+
* `network.controlCohort.reconcileMs` (default `DEFAULT_CONTROL_COHORT_RECONCILE_MS`,
|
|
3528
|
+
* 2 minutes) after the last pass settled. A connection to a relay, a bootstrap node or
|
|
3529
|
+
* another party's machine starts no pass: the pass acts on none of them, beyond
|
|
3530
|
+
* counting them as "connected" for its write steps, which therefore run at the next
|
|
3531
|
+
* membership change or backstop on a one-machine party whose only connection is a
|
|
3532
|
+
* relay.
|
|
3317
3533
|
*
|
|
3318
3534
|
* Concurrent triggers collapse into a single in-flight pass
|
|
3319
3535
|
* (see {@link reconcileControlCohortInFlight}) so two passes never double-dial;
|
|
3320
|
-
* a call that joins an in-flight pass resolves to THAT pass's result
|
|
3536
|
+
* a call that joins an in-flight pass resolves to THAT pass's result (a scheduled
|
|
3537
|
+
* run waits for it instead, so the change that woke the run is seen).
|
|
3321
3538
|
* Best-effort throughout: a failure to resolve/dial any one sibling is logged
|
|
3322
3539
|
* and the pass continues; the whole pass is a no-op when the node is alone.
|
|
3323
3540
|
*
|
|
3324
3541
|
* Resolves to the peers the pass dialled ({@link ControlCohortReconcileResult}),
|
|
3325
3542
|
* so a caller can tell a link this pass opened from one something else opened —
|
|
3326
|
-
* the pass skips a peer that is already connected. The
|
|
3327
|
-
* discard it.
|
|
3543
|
+
* the pass skips a peer that is already connected. The scheduled runs discard it.
|
|
3328
3544
|
*/
|
|
3329
3545
|
async reconcileControlCohort() {
|
|
3330
3546
|
if (this.reconcileControlCohortInFlight) {
|
|
3331
3547
|
return this.reconcileControlCohortInFlight;
|
|
3332
3548
|
}
|
|
3333
|
-
const op = this.
|
|
3549
|
+
const op = this.runReconcileControlCohortAndArmWake();
|
|
3334
3550
|
this.reconcileControlCohortInFlight = op;
|
|
3335
3551
|
try {
|
|
3336
3552
|
return await op;
|
|
@@ -3339,38 +3555,75 @@ export class CadreNode {
|
|
|
3339
3555
|
this.reconcileControlCohortInFlight = null;
|
|
3340
3556
|
}
|
|
3341
3557
|
}
|
|
3558
|
+
/** One pass, then the wake it earns ({@link armReconcileWake}); the public result is the pass's `dialed`. */
|
|
3559
|
+
async runReconcileControlCohortAndArmWake() {
|
|
3560
|
+
const startedAt = Date.now();
|
|
3561
|
+
const outcome = await this.runReconcileControlCohort();
|
|
3562
|
+
this.armReconcileWake(outcome, startedAt);
|
|
3563
|
+
return { dialed: outcome.dialed };
|
|
3564
|
+
}
|
|
3565
|
+
/**
|
|
3566
|
+
* After every pass, explicit or scheduled: advance or reset the redial streak and
|
|
3567
|
+
* `trigger` the schedule for the earliest of the pass's own reasons to run again
|
|
3568
|
+
* (`nextReconcileWakeMs`): a redial for a sibling or bootstrap peer it left
|
|
3569
|
+
* unconnected, or a strand-address ask falling due before the backstop. Nothing else
|
|
3570
|
+
* the pass does needs a wake sooner than the backstop: delegate grants re-announce
|
|
3571
|
+
* every 15 minutes inside a 30-minute TTL, the FRET and formation address re-merges
|
|
3572
|
+
* guard a one-hour expiry, and the self record has its own heartbeat. A pass that
|
|
3573
|
+
* runs before the schedule exists (an explicit call during the first second after
|
|
3574
|
+
* start) still counts for the streak.
|
|
3575
|
+
*/
|
|
3576
|
+
armReconcileWake(outcome, startedAt) {
|
|
3577
|
+
this.reconcileRedialStreak = outcome.unconnected ? this.reconcileRedialStreak + 1 : 0;
|
|
3578
|
+
const runs = this.controlReconcileRuns;
|
|
3579
|
+
if (!runs) {
|
|
3580
|
+
return;
|
|
3581
|
+
}
|
|
3582
|
+
const wakeMs = nextReconcileWakeMs({
|
|
3583
|
+
redialStreak: this.reconcileRedialStreak,
|
|
3584
|
+
passStartedAt: startedAt,
|
|
3585
|
+
strandAddrDueAt: this.strandAddrAskDueAt.values(),
|
|
3586
|
+
now: Date.now(),
|
|
3587
|
+
backstopMs: this.controlReconcileBackstopMs(),
|
|
3588
|
+
});
|
|
3589
|
+
if (wakeMs !== undefined) {
|
|
3590
|
+
log('reconcileControlCohort: next pass in %dms (redial streak %d)', wakeMs, this.reconcileRedialStreak);
|
|
3591
|
+
runs.trigger(wakeMs);
|
|
3592
|
+
}
|
|
3593
|
+
}
|
|
3342
3594
|
/** Body of {@link reconcileControlCohort}; serialised by its single-flight guard. */
|
|
3343
3595
|
async runReconcileControlCohort() {
|
|
3344
3596
|
// Shutdown / not-yet-started guard (mirrors publishSelfRecord). A pass that
|
|
3345
3597
|
// fires after stop() began must early-return rather than touch a torn-down node.
|
|
3346
3598
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3347
|
-
return { dialed: [] };
|
|
3348
|
-
}
|
|
3349
|
-
//
|
|
3350
|
-
//
|
|
3351
|
-
//
|
|
3352
|
-
//
|
|
3353
|
-
// pass's length when that is longer (passes are single-flight, and one
|
|
3354
|
-
// unreachable sibling's dial alone can take `controlDialBudget().totalMs`).
|
|
3599
|
+
return { dialed: [], unconnected: false };
|
|
3600
|
+
}
|
|
3601
|
+
// Refresh the per-stream gate's materialized authorized set at the head of every
|
|
3602
|
+
// pass, so the sibling enumeration below judges rows the same way the gate does.
|
|
3603
|
+
// Changes reach the gate on their own through the gate watch; this refresh is the
|
|
3604
|
+
// backstop's, for a change this node's storage never heard of.
|
|
3355
3605
|
// NOTE: this refresh and the sibling enumeration below each run their own
|
|
3356
3606
|
// CadrePeer query (two reads per pass), plus a third from
|
|
3357
3607
|
// `refreshStrandPeerAddrs` on every pass where a strand is running AND this
|
|
3358
3608
|
// node holds a control connection (one read for the whole pass, not one per
|
|
3359
3609
|
// strand; it runs even when no sibling is due, because pruning departed
|
|
3360
3610
|
// siblings needs the current target set). Each also reads Revocation first.
|
|
3361
|
-
//
|
|
3362
|
-
//
|
|
3363
|
-
//
|
|
3364
|
-
//
|
|
3365
|
-
// control-founding-consult-budget.spec.ts).
|
|
3366
|
-
//
|
|
3611
|
+
// All of them are served from the control database's membership views
|
|
3612
|
+
// (`ControlDatabase.readMembershipRows`): an in-memory scan while the view is
|
|
3613
|
+
// current, storage only for the first read after the table changed or after
|
|
3614
|
+
// `MEMBERSHIP_VIEW_MAX_AGE_MS`, and an idle pass makes no cohort consult
|
|
3615
|
+
// (pinned in control-founding-consult-budget.spec.ts). Threading one row set
|
|
3616
|
+
// through the refresh, the reap, the enumeration and the strand-address targets
|
|
3617
|
+
// would save no storage read and was declined
|
|
3618
|
+
// (`control-reconcile-woken-by-changes-reads-once`); rows going stale during a
|
|
3619
|
+
// pass is harmless, since a change during the pass starts another one.
|
|
3367
3620
|
await this.refreshMembershipGate('reconcile');
|
|
3368
3621
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3369
|
-
return { dialed: [] };
|
|
3622
|
+
return { dialed: [], unconnected: false };
|
|
3370
3623
|
}
|
|
3371
3624
|
await this.publishFirstSelfRecord();
|
|
3372
3625
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3373
|
-
return { dialed: [] };
|
|
3626
|
+
return { dialed: [], unconnected: false };
|
|
3374
3627
|
}
|
|
3375
3628
|
// Refresh relay delegate grants BEFORE the sibling enumeration below: a
|
|
3376
3629
|
// solo cadre with a party relay has no siblings to dial but must still keep
|
|
@@ -3378,7 +3631,7 @@ export class CadreNode {
|
|
|
3378
3631
|
// the reservation and faces the connection gate again).
|
|
3379
3632
|
await this.refreshDelegateGrants();
|
|
3380
3633
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3381
|
-
return { dialed: [] };
|
|
3634
|
+
return { dialed: [], unconnected: false };
|
|
3382
3635
|
}
|
|
3383
3636
|
// Then re-warm each running strand's own address book from its FRET address
|
|
3384
3637
|
// records and from whichever siblings are due an ask. Same reasoning as
|
|
@@ -3386,14 +3639,15 @@ export class CadreNode {
|
|
|
3386
3639
|
// network, and every layer under cadre-core dials strand peers by bare peer id.
|
|
3387
3640
|
await this.refreshStrandPeerAddrs();
|
|
3388
3641
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3389
|
-
return { dialed: [] };
|
|
3642
|
+
return { dialed: [], unconnected: false };
|
|
3390
3643
|
}
|
|
3391
3644
|
const selfPeerId = this.controlNode.peerId.toString();
|
|
3392
3645
|
// Reap guarded rows this node still holds for incarnations an already-committed
|
|
3393
3646
|
// Revocation tombstone retires — the half of a removal replication cannot carry
|
|
3394
|
-
// (a delete of an already-absent row replays as nothing).
|
|
3395
|
-
//
|
|
3396
|
-
//
|
|
3647
|
+
// (a delete of an already-absent row replays as nothing). Only while a sibling is
|
|
3648
|
+
// reachable (hasReachableControlSibling, not merely connected): a reap is a write, and a
|
|
3649
|
+
// write committed alone is local-only and forks this node's own history, which is the
|
|
3650
|
+
// condition this work exists to stop creating
|
|
3397
3651
|
// (tickets/blocked/forked-control-collection-sync-livelocks.md). Nothing is urgent
|
|
3398
3652
|
// here — the stale row already reads as absent everywhere — so waiting for
|
|
3399
3653
|
// connectivity costs nothing and keeps the reap fork-CLOSING rather than
|
|
@@ -3415,7 +3669,7 @@ export class CadreNode {
|
|
|
3415
3669
|
// avoid. Bounded by how long a sweep runs, which is bounded by the backlog of
|
|
3416
3670
|
// unreaped rows (0 in steady state). If that backlog can ever be large, move the
|
|
3417
3671
|
// check inside reapRevokedRows' loop via an injected predicate.
|
|
3418
|
-
if (this.
|
|
3672
|
+
if (this.hasReachableControlSibling()) {
|
|
3419
3673
|
try {
|
|
3420
3674
|
const reaped = await this.controlDatabase.reapRevokedRows(selfPeerId);
|
|
3421
3675
|
if (reaped > 0) {
|
|
@@ -3428,32 +3682,14 @@ export class CadreNode {
|
|
|
3428
3682
|
log('reconcileControlCohort: reap pass failed (continuing): %o', error);
|
|
3429
3683
|
}
|
|
3430
3684
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3431
|
-
return { dialed: [] };
|
|
3432
|
-
}
|
|
3433
|
-
// File the Revocation ledger marker once, so that table stops being a never-written
|
|
3434
|
-
// block, which the storage layer re-checks with the cohort on every read (and every
|
|
3435
|
-
// membership lookup and guarded insert reads it). Connected-only for the reap's
|
|
3436
|
-
// reason above: a marker committed alone is local-only, and one filed by a
|
|
3437
|
-
// disconnected owner while another machine creates the same collection is exactly
|
|
3438
|
-
// that fork. Owner-only, and at most once per process — see openRevocationLedgerIfDue.
|
|
3439
|
-
//
|
|
3440
|
-
// NOTE: accepted tradeoff — a solo founder does not file the marker until it holds a
|
|
3441
|
-
// control connection (normally its first sibling; a relay or bootstrap connection also
|
|
3442
|
-
// counts — see getControlConnectionCount — and is the reap's same residual), so until
|
|
3443
|
-
// then it keeps paying one local findCluster per read of
|
|
3444
|
-
// the missing Revocation block (0.009 ms each, measured upstream; no network work on
|
|
3445
|
-
// a cohort of one). Fork safety weighed over that and kept; revisit if findCluster
|
|
3446
|
-
// ever shows up as material in a device profile, or if a solo-founding marker can be
|
|
3447
|
-
// made fork-safe (e.g. filed inside the genesis transaction, before any other machine
|
|
3448
|
-
// can hold the party's collections).
|
|
3449
|
-
await this.openRevocationLedgerIfDue();
|
|
3450
|
-
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3451
|
-
return { dialed: [] };
|
|
3685
|
+
return { dialed: [], unconnected: false };
|
|
3452
3686
|
}
|
|
3453
3687
|
// Publish this machine's unpublished cross-party joins party-wide and remove the rows a
|
|
3454
|
-
// self-revocation queued (JoinedStrandSession.syncWithParty).
|
|
3455
|
-
//
|
|
3456
|
-
// a join
|
|
3688
|
+
// self-revocation queued (JoinedStrandSession.syncWithParty). Behind the reap's gate,
|
|
3689
|
+
// for its reason; owner-only by the session's own signer check. Nothing is urgent:
|
|
3690
|
+
// a join recorded locally starts no pass, so it reaches the party on the next pass
|
|
3691
|
+
// while a sibling is reachable — a membership change, a sibling connecting, or the
|
|
3692
|
+
// backstop, 2 minutes at most.
|
|
3457
3693
|
try {
|
|
3458
3694
|
await this.joinedStrands?.syncWithParty();
|
|
3459
3695
|
}
|
|
@@ -3461,7 +3697,7 @@ export class CadreNode {
|
|
|
3461
3697
|
log('reconcileControlCohort: joined-strand sync failed (retrying next pass): %o', error);
|
|
3462
3698
|
}
|
|
3463
3699
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3464
|
-
return { dialed: [] };
|
|
3700
|
+
return { dialed: [], unconnected: false };
|
|
3465
3701
|
}
|
|
3466
3702
|
}
|
|
3467
3703
|
// 1. Enumerate known siblings. This membership read is itself a pull-on-read
|
|
@@ -3475,11 +3711,11 @@ export class CadreNode {
|
|
|
3475
3711
|
// because filling this table needs a connection and getting a connection
|
|
3476
3712
|
// needs this table. Fall back to the seed's bootstrap addresses; a solo
|
|
3477
3713
|
// cadre has none and this is a no-op. Must not throw or busy-loop.
|
|
3478
|
-
return
|
|
3714
|
+
return this.dialColdStartBootstrap();
|
|
3479
3715
|
}
|
|
3480
3716
|
// Re-guard after the await: a stop() may have raced the membership read.
|
|
3481
3717
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3482
|
-
return { dialed: [] };
|
|
3718
|
+
return { dialed: [], unconnected: false };
|
|
3483
3719
|
}
|
|
3484
3720
|
// 2. Classify backbone (always-on) members and select a bounded dial set.
|
|
3485
3721
|
const targetDegree = this.config.network?.controlCohort?.targetDegree
|
|
@@ -3493,51 +3729,32 @@ export class CadreNode {
|
|
|
3493
3729
|
// keeping what resolved for the dial loop below.
|
|
3494
3730
|
const resolved = await this.warmSiblingAddrBook(siblings);
|
|
3495
3731
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3496
|
-
return { dialed: [] };
|
|
3732
|
+
return { dialed: [], unconnected: false };
|
|
3497
3733
|
}
|
|
3498
3734
|
// 4. Skip already-connected peers (no re-dial / churn for live connections).
|
|
3499
3735
|
const connected = new Set(this.controlNode.getConnections().map((c) => c.remotePeer.toString()));
|
|
3500
3736
|
// 5. Dial each selected, not-yet-connected sibling, best-effort, from the
|
|
3501
|
-
// addresses step 3 already resolved for it.
|
|
3737
|
+
// addresses step 3 already resolved for it. A failed dial earns the pass a
|
|
3738
|
+
// redial wake (armReconcileWake); a sibling with nothing to dial does not.
|
|
3502
3739
|
const dialed = [];
|
|
3740
|
+
let unconnected = false;
|
|
3503
3741
|
for (const sibling of dials) {
|
|
3504
3742
|
if (!this._running || !this.controlNode) {
|
|
3505
|
-
return { dialed };
|
|
3743
|
+
return { dialed, unconnected: false };
|
|
3506
3744
|
}
|
|
3507
3745
|
if (connected.has(sibling.peerId)) {
|
|
3508
3746
|
continue;
|
|
3509
3747
|
}
|
|
3510
|
-
|
|
3748
|
+
const outcome = await this.dialControlSibling(sibling, resolved.get(sibling.peerId) ?? []);
|
|
3749
|
+
if (outcome === 'connected') {
|
|
3511
3750
|
dialed.push(sibling.peerId);
|
|
3512
3751
|
}
|
|
3752
|
+
else if (outcome === 'failed') {
|
|
3753
|
+
unconnected = true;
|
|
3754
|
+
}
|
|
3513
3755
|
}
|
|
3514
|
-
log('reconcileControlCohort: pass complete (siblings=%d, selected=%d, dialed=%d)', siblings.length, dials.length, dialed.length);
|
|
3515
|
-
return { dialed };
|
|
3516
|
-
}
|
|
3517
|
-
/**
|
|
3518
|
-
* The owner-only half of the reconcile pass's ledger-marker step: file the singleton
|
|
3519
|
-
* `Revocation` marker ({@link SeedBootstrapService.openRevocationLedger}) unless this
|
|
3520
|
-
* process has already seen it filed. The connectivity gate is the caller's
|
|
3521
|
-
* ({@link runReconcileControlCohort}); why the marker exists is on
|
|
3522
|
-
* {@link ControlDatabase.openRevocationLedger}.
|
|
3523
|
-
*
|
|
3524
|
-
* Sets {@link revocationLedgerOpened} on `'opened'` or `'already-open'`. A node that
|
|
3525
|
-
* cannot sign as an owner does nothing: it picks the owner's marker up the first time it
|
|
3526
|
-
* reads the table, like any other replicated row. Best-effort like every step of the
|
|
3527
|
-
* pass — a failure is logged and leaves the flag clear, so the next connected pass retries.
|
|
3528
|
-
*/
|
|
3529
|
-
async openRevocationLedgerIfDue() {
|
|
3530
|
-
if (this.revocationLedgerOpened || !this.seedBootstrapService?.canAuthorize()) {
|
|
3531
|
-
return;
|
|
3532
|
-
}
|
|
3533
|
-
try {
|
|
3534
|
-
const outcome = await this.seedBootstrapService.openRevocationLedger();
|
|
3535
|
-
this.revocationLedgerOpened = true;
|
|
3536
|
-
log('reconcileControlCohort: revocation ledger marker %s', outcome);
|
|
3537
|
-
}
|
|
3538
|
-
catch (error) {
|
|
3539
|
-
log('reconcileControlCohort: filing the revocation ledger marker failed (retrying next pass): %o', error);
|
|
3540
|
-
}
|
|
3756
|
+
log('reconcileControlCohort: pass complete (siblings=%d, selected=%d, dialed=%d, unconnected=%s)', siblings.length, dials.length, dialed.length, unconnected);
|
|
3757
|
+
return { dialed, unconnected };
|
|
3541
3758
|
}
|
|
3542
3759
|
/**
|
|
3543
3760
|
* Resolve every sibling's signed address record once and merge what resolves
|
|
@@ -3562,8 +3779,9 @@ export class CadreNode {
|
|
|
3562
3779
|
* current ({@link refreshDialHint}).
|
|
3563
3780
|
*
|
|
3564
3781
|
* NOTE: this resolves EVERY sibling serially before the dial loop below runs,
|
|
3565
|
-
* so it costs one record query per sibling per reconcile pass (
|
|
3566
|
-
*
|
|
3782
|
+
* so it costs one record query per sibling per reconcile pass (served from the
|
|
3783
|
+
* membership rows view while it is current) and each one delays the pass's first
|
|
3784
|
+
* dial. A cadre is a handful of devices, so today
|
|
3567
3785
|
* that is a few extra local reads; if cadres ever grow large, batch the records
|
|
3568
3786
|
* into one query, merge only on change, or move the warm pass after the dials.
|
|
3569
3787
|
*/
|
|
@@ -3688,40 +3906,41 @@ export class CadreNode {
|
|
|
3688
3906
|
});
|
|
3689
3907
|
}
|
|
3690
3908
|
/**
|
|
3691
|
-
* Dial one sibling from its already-resolved addresses, best-effort.
|
|
3692
|
-
* whether the dial resolved (false when no address resolves or the dial fails).
|
|
3909
|
+
* Dial one sibling from its already-resolved addresses, best-effort.
|
|
3693
3910
|
*
|
|
3694
3911
|
* A per-peer failure (NAT, offline, relay down, connection-gater denial, or
|
|
3695
3912
|
* the {@link controlDialBudget} expiring) is logged and swallowed so one
|
|
3696
3913
|
* unreachable sibling never aborts the pass — exactly like
|
|
3697
3914
|
* {@link SeedBootstrapService.applySeed}'s owner-dial loop. A failed dial is
|
|
3698
|
-
*
|
|
3699
|
-
*
|
|
3700
|
-
*
|
|
3915
|
+
* retried on the pass's redial ladder ({@link armReconcileWake}: 15 s, 30 s, 60 s,
|
|
3916
|
+
* then every backstop) and on any pass a change starts sooner. A sibling with no
|
|
3917
|
+
* dialable address, or whose every address relays through this node
|
|
3918
|
+
* ({@link SelfRelayOnlyError}), is not dialed at all and is `undialable`: it gets a
|
|
3919
|
+
* one-line log and no redial, since only a change — its record replicating in, or
|
|
3920
|
+
* the sibling connecting — can make it dialable, and each of those starts a pass.
|
|
3701
3921
|
*/
|
|
3702
3922
|
async dialControlSibling(sibling, resolved) {
|
|
3703
3923
|
const controlNode = this.controlNode;
|
|
3704
3924
|
if (!controlNode) {
|
|
3705
|
-
return
|
|
3925
|
+
return 'undialable';
|
|
3706
3926
|
}
|
|
3707
3927
|
const addrs = await this.resolveControlDialAddrs(sibling.peerId, resolved);
|
|
3708
3928
|
if (addrs.length === 0) {
|
|
3709
3929
|
log('reconcileControlCohort: no dialable control address for sibling %s; skipping', sibling.peerId);
|
|
3710
|
-
return
|
|
3930
|
+
return 'undialable';
|
|
3711
3931
|
}
|
|
3712
3932
|
try {
|
|
3713
3933
|
log('reconcileControlCohort: dialing sibling %s (%d addr(s))', sibling.peerId, addrs.length);
|
|
3714
3934
|
await dialPeerAddrs(controlNode, sibling.peerId, addrs, this.controlDialBudget(), `reconcileControlCohort dial of sibling ${sibling.peerId}`);
|
|
3715
|
-
return
|
|
3935
|
+
return 'connected';
|
|
3716
3936
|
}
|
|
3717
3937
|
catch (error) {
|
|
3718
3938
|
if (error instanceof SelfRelayOnlyError) {
|
|
3719
3939
|
log('reconcileControlCohort: sibling %s is reachable only by relaying through this node; waiting for it to reconnect', sibling.peerId);
|
|
3940
|
+
return 'undialable';
|
|
3720
3941
|
}
|
|
3721
|
-
|
|
3722
|
-
|
|
3723
|
-
}
|
|
3724
|
-
return false;
|
|
3942
|
+
log('reconcileControlCohort: dial of sibling %s failed (continuing): %o', sibling.peerId, error);
|
|
3943
|
+
return 'failed';
|
|
3725
3944
|
}
|
|
3726
3945
|
}
|
|
3727
3946
|
/**
|
|
@@ -3774,7 +3993,7 @@ export class CadreNode {
|
|
|
3774
3993
|
* These addresses are never merged into the libp2p address book: they are
|
|
3775
3994
|
* unverified, and the address book takes only verified ones (see
|
|
3776
3995
|
* `mergePeerAddrs`). Layers that dial by bare peer id use the connection this
|
|
3777
|
-
* dial opens, and after it drops the
|
|
3996
|
+
* dial opens, and after it drops the reconcile pass the close starts dials again.
|
|
3778
3997
|
*
|
|
3779
3998
|
* NOTE: an entry is consulted here only for a current sibling, so a peer revoked
|
|
3780
3999
|
* by another owner is not dialed from it, even though the entry stays (only this
|
|
@@ -3825,8 +4044,7 @@ export class CadreNode {
|
|
|
3825
4044
|
* Called for every seed this node accepts, on BOTH intake paths — the
|
|
3826
4045
|
* {@link applySeed} wrapper (which may run on a throwaway service, so the
|
|
3827
4046
|
* service itself is the wrong place to keep this) and the inbound
|
|
3828
|
-
* `/sereus/seed/1.0.0` handler via `onSeedApplied`.
|
|
3829
|
-
* skipped: there is nothing to dial.
|
|
4047
|
+
* `/sereus/seed/1.0.0` handler via `onSeedApplied`.
|
|
3830
4048
|
*
|
|
3831
4049
|
* `preferred` is the seed's own claim about its peers, exactly as
|
|
3832
4050
|
* `SeedBootstrapService.applySeed` uses it to choose its dial targets. That is
|
|
@@ -3834,14 +4052,15 @@ export class CadreNode {
|
|
|
3834
4052
|
* signature-checked against a trust-anchored signer before this runs, and the
|
|
3835
4053
|
* flag only *selects a dial target*; a dial grants no authority.
|
|
3836
4054
|
*
|
|
3837
|
-
*
|
|
4055
|
+
* The peers retained are `seedDialTargets` (addressless peers and self dropped), the same
|
|
4056
|
+
* ones the service dials; they are returned for the `seed:applied` payload.
|
|
3838
4057
|
*/
|
|
3839
4058
|
recordSeedBootstrapPeers(seed) {
|
|
3840
|
-
|
|
3841
|
-
|
|
3842
|
-
|
|
3843
|
-
}
|
|
4059
|
+
const targets = seedDialTargets(seed, this.controlNode?.peerId.toString());
|
|
4060
|
+
for (const { peerId, multiaddrs } of targets) {
|
|
4061
|
+
this.retainDialTarget(peerId, multiaddrs, 'recordSeedBootstrapPeers');
|
|
3844
4062
|
}
|
|
4063
|
+
return targets;
|
|
3845
4064
|
}
|
|
3846
4065
|
/**
|
|
3847
4066
|
* Retain (or replace) one peer's out-of-band dial target in
|
|
@@ -3855,9 +4074,8 @@ export class CadreNode {
|
|
|
3855
4074
|
* failing a seed or an add that has already been accepted.
|
|
3856
4075
|
*
|
|
3857
4076
|
* Two peers are never retained. One with no address: there is nothing to dial.
|
|
3858
|
-
* And self: `
|
|
3859
|
-
*
|
|
3860
|
-
* could be handed this node's own id. Retaining self would make the cold-start
|
|
4077
|
+
* And self: `addDrone` could be handed this node's own id (seed intake arrives
|
|
4078
|
+
* already filtered by `seedDialTargets`). Retaining self would make the cold-start
|
|
3861
4079
|
* pass dial this node forever (the steady-state pass filters self out of its
|
|
3862
4080
|
* sibling list for the same reason).
|
|
3863
4081
|
*/
|
|
@@ -3891,15 +4109,16 @@ export class CadreNode {
|
|
|
3891
4109
|
* help: it dials only siblings enumerated from that very table. Retrying here
|
|
3892
4110
|
* is the only way back in.
|
|
3893
4111
|
*
|
|
3894
|
-
* Unbounded
|
|
3895
|
-
*
|
|
3896
|
-
*
|
|
3897
|
-
* a
|
|
3898
|
-
*
|
|
3899
|
-
*
|
|
3900
|
-
*
|
|
3901
|
-
*
|
|
3902
|
-
*
|
|
4112
|
+
* Unbounded: the branch is already gated by "the control database has no
|
|
4113
|
+
* siblings", so it stops the moment the node is actually in the party, and a node
|
|
4114
|
+
* that is stranded MUST keep trying — a give-up rule would turn a transient outage
|
|
4115
|
+
* into a permanent one. A failed dial earns the pass a redial wake on the ladder
|
|
4116
|
+
* {@link armReconcileWake} runs (15 s, 30 s, 60 s, then every backstop, 2 minutes
|
|
4117
|
+
* by default), so the steady cost is one dial per bootstrap peer per backstop (a
|
|
4118
|
+
* seed prefers its minter and the party's always-on machines, a few at most), each
|
|
4119
|
+
* failing fast against an unreachable address.
|
|
4120
|
+
* NOTE: the ladder is per pass, not per peer; if seeds ever carry many preferred
|
|
4121
|
+
* peers, add per-peer backoff here.
|
|
3903
4122
|
*
|
|
3904
4123
|
* Best-effort per peer, exactly like {@link dialControlSibling}: one dead
|
|
3905
4124
|
* address never aborts the pass.
|
|
@@ -3910,14 +4129,15 @@ export class CadreNode {
|
|
|
3910
4129
|
// cannot mutate what we are iterating.
|
|
3911
4130
|
const targets = this.bootstrapPeerStore?.all();
|
|
3912
4131
|
if (!controlNode || !targets || targets.size === 0) {
|
|
3913
|
-
return [];
|
|
4132
|
+
return { dialed: [], unconnected: false };
|
|
3914
4133
|
}
|
|
3915
4134
|
// Same skip rule as step 3 of the steady-state pass: no churn on live links.
|
|
3916
4135
|
const connected = new Set(controlNode.getConnections().map((c) => c.remotePeer.toString()));
|
|
3917
4136
|
const dialed = [];
|
|
4137
|
+
let unconnected = false;
|
|
3918
4138
|
for (const [peerId, entry] of targets) {
|
|
3919
4139
|
if (!this._running || !this.controlNode) {
|
|
3920
|
-
return dialed;
|
|
4140
|
+
return { dialed, unconnected: false };
|
|
3921
4141
|
}
|
|
3922
4142
|
// NOTE: a connection this node still holds in `status: 'open'` after the remote
|
|
3923
4143
|
// has already aborted it counts as connected here, so the cold-start retry stays
|
|
@@ -3932,9 +4152,12 @@ export class CadreNode {
|
|
|
3932
4152
|
if (await this.dialBootstrapPeer(peerId, entry.addrs)) {
|
|
3933
4153
|
dialed.push(peerId);
|
|
3934
4154
|
}
|
|
4155
|
+
else {
|
|
4156
|
+
unconnected = true;
|
|
4157
|
+
}
|
|
3935
4158
|
}
|
|
3936
|
-
log('reconcileControlCohort: cold-start pass complete (bootstrap=%d, connected=%d, dialed=%d)', targets.size, connected.size, dialed.length);
|
|
3937
|
-
return dialed;
|
|
4159
|
+
log('reconcileControlCohort: cold-start pass complete (bootstrap=%d, connected=%d, dialed=%d, unconnected=%s)', targets.size, connected.size, dialed.length, unconnected);
|
|
4160
|
+
return { dialed, unconnected };
|
|
3938
4161
|
}
|
|
3939
4162
|
/**
|
|
3940
4163
|
* Bind a retained dial target's addresses (see {@link bootstrapPeerStore}) to
|
|
@@ -3951,28 +4174,32 @@ export class CadreNode {
|
|
|
3951
4174
|
bootstrapDialAddrs(peerId, addrs) {
|
|
3952
4175
|
return this.normalizeDialAddrs(this.parseMultiaddrs(addrs), peerId);
|
|
3953
4176
|
}
|
|
3954
|
-
/**
|
|
3955
|
-
|
|
4177
|
+
/**
|
|
4178
|
+
* Dial one peer from its retained dial target's addresses, best-effort: a cold-start bootstrap
|
|
4179
|
+
* peer, or a just-claimed node ({@link redialClaimedNode}), which `context` names in the logs.
|
|
4180
|
+
* Returns whether it connected.
|
|
4181
|
+
*/
|
|
4182
|
+
async dialBootstrapPeer(peerId, addrs, context = 'reconcileControlCohort(cold-start)') {
|
|
3956
4183
|
const controlNode = this.controlNode;
|
|
3957
4184
|
if (!controlNode) {
|
|
3958
4185
|
return false;
|
|
3959
4186
|
}
|
|
3960
4187
|
const parsed = this.bootstrapDialAddrs(peerId, addrs);
|
|
3961
4188
|
if (parsed.length === 0) {
|
|
3962
|
-
log('
|
|
4189
|
+
log('%s: no dialable address for bootstrap peer %s; skipping', context, peerId);
|
|
3963
4190
|
return false;
|
|
3964
4191
|
}
|
|
3965
4192
|
try {
|
|
3966
|
-
log('
|
|
3967
|
-
await dialPeerAddrs(controlNode, peerId, parsed, this.controlDialBudget(),
|
|
4193
|
+
log('%s: dialing bootstrap peer %s (%d addr(s))', context, peerId, parsed.length);
|
|
4194
|
+
await dialPeerAddrs(controlNode, peerId, parsed, this.controlDialBudget(), `${context} dial of bootstrap peer ${peerId}`);
|
|
3968
4195
|
return true;
|
|
3969
4196
|
}
|
|
3970
4197
|
catch (error) {
|
|
3971
4198
|
if (error instanceof SelfRelayOnlyError) {
|
|
3972
|
-
log('
|
|
4199
|
+
log('%s: bootstrap peer %s is reachable only by relaying through this node; waiting for it to reconnect', context, peerId);
|
|
3973
4200
|
}
|
|
3974
4201
|
else {
|
|
3975
|
-
log('
|
|
4202
|
+
log('%s: dial of bootstrap peer %s failed (continuing): %o', context, peerId, error);
|
|
3976
4203
|
}
|
|
3977
4204
|
return false;
|
|
3978
4205
|
}
|
|
@@ -3982,31 +4209,33 @@ export class CadreNode {
|
|
|
3982
4209
|
//
|
|
3983
4210
|
// Optimystic's coordinator commits a control write WITHOUT broadcasting when the
|
|
3984
4211
|
// block's cluster has ≤1 member. An owner that authorizes/removes a peer or
|
|
3985
|
-
// (re)publishes its own record while no sibling is
|
|
4212
|
+
// (re)publishes its own record while no sibling is reachable therefore writes a
|
|
3986
4213
|
// row that exists ONLY in its local control DB — a sibling that connects later
|
|
3987
4214
|
// never observes it (pull-on-read routes to the block's cluster, which never
|
|
3988
4215
|
// learned of the write). reconcileControlCohort shrinks the alone window but
|
|
3989
|
-
// cannot close it (a write in the instant before any sibling
|
|
3990
|
-
// local-only). The remedy: detect "wrote while alone" (
|
|
3991
|
-
//
|
|
3992
|
-
//
|
|
4216
|
+
// cannot close it (a write in the instant before any sibling identifies is still
|
|
4217
|
+
// local-only). The remedy: detect "wrote while alone" (no identified sibling the
|
|
4218
|
+
// cohort would admit: committedAlone), queue the affected row, and RE-ISSUE the write
|
|
4219
|
+
// once one is reachable, as an idempotent monotonic update that now broadcasts.
|
|
3993
4220
|
// See docs/architecture.md (Control Network Convergence → "Writes made while alone").
|
|
3994
4221
|
// ============================================================================
|
|
3995
4222
|
/**
|
|
3996
|
-
* Whether a control write happening now would commit local-only
|
|
3997
|
-
*
|
|
3998
|
-
*
|
|
3999
|
-
*
|
|
4000
|
-
*
|
|
4001
|
-
*
|
|
4002
|
-
*
|
|
4003
|
-
* is acceptable as the agreed first cut.
|
|
4223
|
+
* Whether a control write happening now would commit local-only: no sibling is reachable
|
|
4224
|
+
* ({@link hasReachableControlSibling}, the public form, so the two cannot drift). A relay, a
|
|
4225
|
+
* bootstrap node or a peer that has not identified yet holds a connection without being in the
|
|
4226
|
+
* write's cohort, so a write made with only those connected is queued too. One made alone at
|
|
4227
|
+
* the revision the cohort reached meanwhile is a fork no later consult sees, because a consult
|
|
4228
|
+
* compares revision numbers only; the re-issue on the growth edge is what conflicts with the
|
|
4229
|
+
* cohort's copy and pulls it in.
|
|
4004
4230
|
*
|
|
4005
|
-
*
|
|
4006
|
-
* the
|
|
4231
|
+
* Still a proxy for the precise signal (the block's own cohort): a reachable sibling outside
|
|
4232
|
+
* this block's cohort leaves the write local-only, caught by the cohort's pull-on-read. On the
|
|
4233
|
+
* control network every block's cohort is the whole party, so that gap is a party larger than
|
|
4234
|
+
* `CONTROL_REPLICATION_BREADTH`. Re-issuing a replicated row is harmless (an idempotent
|
|
4235
|
+
* monotonic bump).
|
|
4007
4236
|
*/
|
|
4008
4237
|
committedAlone() {
|
|
4009
|
-
return this.
|
|
4238
|
+
return !this.hasReachableControlSibling();
|
|
4010
4239
|
}
|
|
4011
4240
|
/**
|
|
4012
4241
|
* Record (or clear) a just-committed owner membership write in the
|
|
@@ -4060,7 +4289,7 @@ export class CadreNode {
|
|
|
4060
4289
|
/**
|
|
4061
4290
|
* Drain the write-while-alone re-replication queue: re-issue the writes that
|
|
4062
4291
|
* committed local-only now that the cohort can broadcast them. Public so the
|
|
4063
|
-
*
|
|
4292
|
+
* growth trigger ({@link updateControlSiblingReachability}) and tests can drive
|
|
4064
4293
|
* it; concurrent calls collapse into one in-flight drain
|
|
4065
4294
|
* ({@link drainControlReplicationInFlight}) so two growth signals never
|
|
4066
4295
|
* double-issue. Best-effort throughout — a per-row failure leaves that entry
|
|
@@ -4098,13 +4327,15 @@ export class CadreNode {
|
|
|
4098
4327
|
// and receiving-node staleness is the pre-existing pull-on-read /
|
|
4099
4328
|
// periodic-refresh property.
|
|
4100
4329
|
await this.drainPendingRevocations();
|
|
4101
|
-
// 2. Self rows.
|
|
4102
|
-
//
|
|
4103
|
-
//
|
|
4330
|
+
// 2. Self rows. A self write made while alone is always re-touched: the re-touch
|
|
4331
|
+
// is what carries it to the cohort. Otherwise the first growth re-touches a row
|
|
4332
|
+
// a prior process may have committed local-only, unless the row already holds a
|
|
4333
|
+
// current record this process signed (a redeemed invitation's admission).
|
|
4104
4334
|
// registerSelf is idempotent + single-flight, so this never races a
|
|
4105
4335
|
// duplicate publish with the heartbeat republish.
|
|
4106
4336
|
if (firstGrowth || this.pendingSelfPeerWrite) {
|
|
4107
|
-
|
|
4337
|
+
const publish = this.pendingSelfPeerWrite ? this.registerSelf() : this.publishSelfRecordUnlessCurrent();
|
|
4338
|
+
await publish.catch((error) => log('drain: registerSelf failed: %o', error));
|
|
4108
4339
|
this.pendingSelfPeerWrite = false;
|
|
4109
4340
|
}
|
|
4110
4341
|
if (firstGrowth || this.pendingSelfDeviceWrite) {
|
|
@@ -4254,8 +4485,11 @@ export class CadreNode {
|
|
|
4254
4485
|
}
|
|
4255
4486
|
/**
|
|
4256
4487
|
* Body of {@link reconstructAuthoredMembership}'s sweep: re-issue each candidate
|
|
4257
|
-
* owner-authored row, returning how many were re-touched. A
|
|
4258
|
-
*
|
|
4488
|
+
* owner-authored row, returning how many were re-touched. A row the sweep already holds as
|
|
4489
|
+
* self-published is skipped without a call; the re-touch itself decides again over the row
|
|
4490
|
+
* it reads under the lock, so a row that vanished or was self-published since the sweep's
|
|
4491
|
+
* read is reported, not written. A per-row failure is logged and skipped; a teardown
|
|
4492
|
+
* mid-sweep stops it.
|
|
4259
4493
|
*/
|
|
4260
4494
|
async reissueAuthoredMembershipRows(selfPeerId, members) {
|
|
4261
4495
|
let touched = 0;
|
|
@@ -4263,17 +4497,15 @@ export class CadreNode {
|
|
|
4263
4497
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
4264
4498
|
return touched;
|
|
4265
4499
|
}
|
|
4266
|
-
|
|
4500
|
+
// Self, queued for the drain (step 4 re-touches it once), or peer-owned (self-Sig
|
|
4501
|
+
// present) → not this sweep's to bump.
|
|
4502
|
+
if (member.peerId === selfPeerId || this.pendingPeerWrites.has(member.peerId) || member.sig) {
|
|
4267
4503
|
continue;
|
|
4268
4504
|
}
|
|
4269
4505
|
try {
|
|
4270
|
-
|
|
4271
|
-
|
|
4272
|
-
if (!record || record.sig) {
|
|
4273
|
-
continue;
|
|
4506
|
+
if (await this.reissuePeerAuthorize(member.peerId) === 'retouched') {
|
|
4507
|
+
touched++;
|
|
4274
4508
|
}
|
|
4275
|
-
await this.reissuePeerAuthorize(member.peerId, record.updatedAt);
|
|
4276
|
-
touched++;
|
|
4277
4509
|
}
|
|
4278
4510
|
catch (error) {
|
|
4279
4511
|
log('drain: reconstruction re-touch of %s failed (continuing): %o', member.peerId, error);
|
|
@@ -4307,9 +4539,10 @@ export class CadreNode {
|
|
|
4307
4539
|
await this.deferMembershipGateRefresh('drain-reissue', () => this.reissuePendingPeerWrites());
|
|
4308
4540
|
}
|
|
4309
4541
|
/**
|
|
4310
|
-
* Body of {@link drainPendingPeerWrites}' loop. An entry is cleared
|
|
4311
|
-
*
|
|
4312
|
-
* the
|
|
4542
|
+
* Body of {@link drainPendingPeerWrites}' loop. An entry is cleared on every outcome the
|
|
4543
|
+
* re-touch reports — re-touched, or nothing left to re-issue because the row was removed
|
|
4544
|
+
* since or the peer self-published; the re-touch decides which over the row it reads under
|
|
4545
|
+
* the lock. A failure is logged and the entry stays queued for the next cohort growth.
|
|
4313
4546
|
*/
|
|
4314
4547
|
async reissuePendingPeerWrites() {
|
|
4315
4548
|
for (const peerId of [...this.pendingPeerWrites.keys()]) {
|
|
@@ -4317,13 +4550,8 @@ export class CadreNode {
|
|
|
4317
4550
|
return;
|
|
4318
4551
|
}
|
|
4319
4552
|
try {
|
|
4320
|
-
const
|
|
4321
|
-
|
|
4322
|
-
// Removed since, or peer self-published — nothing for the owner to re-issue.
|
|
4323
|
-
this.pendingPeerWrites.delete(peerId);
|
|
4324
|
-
continue;
|
|
4325
|
-
}
|
|
4326
|
-
await this.reissuePeerAuthorize(peerId, record.updatedAt);
|
|
4553
|
+
const outcome = await this.reissuePeerAuthorize(peerId);
|
|
4554
|
+
log('drain: re-issue of %s (authorize): %s', peerId, outcome);
|
|
4327
4555
|
this.pendingPeerWrites.delete(peerId);
|
|
4328
4556
|
}
|
|
4329
4557
|
catch (error) {
|
|
@@ -4336,57 +4564,196 @@ export class CadreNode {
|
|
|
4336
4564
|
* `UpdatedAt` strictly above the stored value (and the wall clock) and re-sign via
|
|
4337
4565
|
* the owner branch of `CadrePeer.AuthorizedUpdate`. The row already exists
|
|
4338
4566
|
* locally (it committed there, just local-only), so this is an UPDATE, not the
|
|
4339
|
-
* original INSERT.
|
|
4567
|
+
* original INSERT. The bump, and whether the row is still the owner's to re-issue, are
|
|
4568
|
+
* decided over the row as read inside the write ({@link SeedBootstrapService.reauthorizePeer}'s
|
|
4569
|
+
* re-broadcast mode).
|
|
4340
4570
|
*/
|
|
4341
|
-
async reissuePeerAuthorize(peerId
|
|
4571
|
+
async reissuePeerAuthorize(peerId) {
|
|
4342
4572
|
if (!this.seedBootstrapService) {
|
|
4343
|
-
|
|
4573
|
+
throw new Error('Seed bootstrap service not initialized. Call initializeSeedBootstrap() first.');
|
|
4344
4574
|
}
|
|
4345
|
-
|
|
4346
|
-
await this.seedBootstrapService.reauthorizePeer(peerId, updatedAt);
|
|
4575
|
+
return await this.seedBootstrapService.reauthorizePeer(peerId, 'rebroadcast');
|
|
4347
4576
|
}
|
|
4348
4577
|
/**
|
|
4349
|
-
*
|
|
4350
|
-
* write-while-alone re-replication queue (single-flight;
|
|
4351
|
-
* not on
|
|
4352
|
-
*
|
|
4578
|
+
* Re-evaluate {@link hasReachableControlSibling} and act on its edges: on false→true, drain
|
|
4579
|
+
* the write-while-alone re-replication queue (single-flight; best-effort, a drain failure is
|
|
4580
|
+
* logged, not thrown); on true→false, re-arm, so the next reachable sibling drains again.
|
|
4581
|
+
* Called when a peer identifies, when a connection closes, and after every membership refresh,
|
|
4582
|
+
* where a sibling that connected before this node authorized it (the production join order)
|
|
4583
|
+
* becomes reachable. A device counted in for its admission ({@link admittingDevices}) is not
|
|
4584
|
+
* a trigger: the admission's own writes are in flight, and its row landing refreshes
|
|
4585
|
+
* membership.
|
|
4353
4586
|
*/
|
|
4354
|
-
|
|
4587
|
+
updateControlSiblingReachability() {
|
|
4355
4588
|
if (!this._running || !this.controlNode) {
|
|
4356
4589
|
return;
|
|
4357
4590
|
}
|
|
4358
|
-
const
|
|
4359
|
-
if (
|
|
4591
|
+
const reachable = this.hasReachableControlSibling();
|
|
4592
|
+
if (reachable === this.controlSiblingReachable) {
|
|
4360
4593
|
return;
|
|
4361
4594
|
}
|
|
4362
|
-
this.
|
|
4363
|
-
|
|
4364
|
-
|
|
4365
|
-
|
|
4595
|
+
this.controlSiblingReachable = reachable;
|
|
4596
|
+
if (!reachable) {
|
|
4597
|
+
return;
|
|
4598
|
+
}
|
|
4599
|
+
log('Control cohort grew (a sibling is reachable); draining write-while-alone re-replication queue');
|
|
4600
|
+
void this.drainPendingControlReplication('sibling-reachable')
|
|
4601
|
+
.catch((error) => log('Control re-replication drain (sibling-reachable) failed: %o', error));
|
|
4366
4602
|
}
|
|
4367
|
-
/**
|
|
4368
|
-
|
|
4369
|
-
|
|
4603
|
+
/**
|
|
4604
|
+
* A connected peer identified as serving the control database: count it
|
|
4605
|
+
* ({@link identifiedControlPeers}), re-evaluate the growth edge, and run the sibling-open
|
|
4606
|
+
* reactions, once per connected period. libp2p re-emits `peer:identify` on every
|
|
4607
|
+
* identify-push from a connected peer, and those repeats start nothing.
|
|
4608
|
+
*/
|
|
4609
|
+
onControlPeerIdentified(peerId) {
|
|
4610
|
+
const id = peerId.toString();
|
|
4611
|
+
// An identify that reports after its connection closed counts nobody.
|
|
4612
|
+
if (this.identifiedControlPeers.has(id) || !this.controlNode || this.controlNode.getConnections(peerId).length === 0) {
|
|
4613
|
+
return;
|
|
4614
|
+
}
|
|
4615
|
+
this.identifiedControlPeers.add(id);
|
|
4616
|
+
this.updateControlSiblingReachability();
|
|
4617
|
+
this.onControlSiblingConnectionChange(id, 'open');
|
|
4618
|
+
}
|
|
4619
|
+
/**
|
|
4620
|
+
* A control connection closed: `remotePeer` stops counting once it has none left, and the
|
|
4621
|
+
* growth edge re-arms if no reachable sibling remains. The sibling-close reaction runs on every
|
|
4622
|
+
* close. libp2p's connection manager drops the closed connection from `getConnections` before
|
|
4623
|
+
* this listener runs: libp2p dispatches each event on its internal bus, where the connection
|
|
4624
|
+
* manager listens, before re-dispatching it to the node's own listeners.
|
|
4625
|
+
*/
|
|
4626
|
+
onControlConnectionClosed(remotePeer) {
|
|
4627
|
+
const id = remotePeer.toString();
|
|
4628
|
+
if (!this.controlNode || this.controlNode.getConnections(remotePeer).length === 0) {
|
|
4629
|
+
this.identifiedControlPeers.delete(id);
|
|
4630
|
+
}
|
|
4631
|
+
this.updateControlSiblingReachability();
|
|
4632
|
+
this.onControlSiblingConnectionChange(id, 'close');
|
|
4633
|
+
}
|
|
4634
|
+
/**
|
|
4635
|
+
* Count the peers that identified before {@link wireControlConnectionListeners} ran, whose
|
|
4636
|
+
* `peer:identify` has already fired, from their peer-store records. Updates the growth edge
|
|
4637
|
+
* but runs no sibling-open reaction, as a connection that predates the listeners never did. A
|
|
4638
|
+
* peer not yet in the peer store is still identifying, and its own event counts it.
|
|
4639
|
+
*/
|
|
4640
|
+
async noteIdentifiedControlPeers() {
|
|
4641
|
+
const node = this.controlNode;
|
|
4642
|
+
if (!node) {
|
|
4370
4643
|
return;
|
|
4371
4644
|
}
|
|
4372
|
-
|
|
4373
|
-
|
|
4645
|
+
const connected = new Map();
|
|
4646
|
+
for (const connection of node.getConnections()) {
|
|
4647
|
+
connected.set(connection.remotePeer.toString(), connection.remotePeer);
|
|
4648
|
+
}
|
|
4649
|
+
const partyId = this.config.controlNetwork.partyId;
|
|
4650
|
+
await Promise.all([...connected.values()].map(async (peerId) => {
|
|
4651
|
+
try {
|
|
4652
|
+
const { protocols } = await node.peerStore.get(peerId);
|
|
4653
|
+
// Still wired to this node: a stop while the read was out cleared the set.
|
|
4654
|
+
const wired = this.controlPeerIdentifyHandler !== null && this.controlNode === node;
|
|
4655
|
+
if (wired && servesControlDatabase(partyId, protocols) && node.getConnections(peerId).length > 0) {
|
|
4656
|
+
this.identifiedControlPeers.add(peerId.toString());
|
|
4657
|
+
}
|
|
4658
|
+
}
|
|
4659
|
+
catch (error) {
|
|
4660
|
+
if (error.name !== 'NotFoundError') {
|
|
4661
|
+
log('Peer store read for connected control peer %s failed — left to its identify: %o', peerId.toString(), error);
|
|
4662
|
+
}
|
|
4663
|
+
}
|
|
4664
|
+
}));
|
|
4665
|
+
this.updateControlSiblingReachability();
|
|
4666
|
+
}
|
|
4667
|
+
/**
|
|
4668
|
+
* An AUTHORIZED sibling became reachable (its identify, once per connected period:
|
|
4669
|
+
* {@link onControlPeerIdentified}) or one of its connections closed: start a reconcile
|
|
4670
|
+
* pass. On open the pass asks the new sibling for strand addresses and re-arms the
|
|
4671
|
+
* delegate grants; on close it re-dials. First, on open, drop the membership rows
|
|
4672
|
+
* views: a machine that was offline while a sibling committed learns of the commit
|
|
4673
|
+
* only when it next reads the block from storage, and dropping the views makes the
|
|
4674
|
+
* pass's reads do that now rather than after the views' maximum age. Those reads can
|
|
4675
|
+
* fall inside the read-repair window, so the open also arms a second drop past it
|
|
4676
|
+
* ({@link armMembershipViewReload}). Any other
|
|
4677
|
+
* peer — a relay, a bootstrap node, another party's machine — starts nothing.
|
|
4678
|
+
*
|
|
4679
|
+
* On identify rather than `connection:open` because, until the sibling identifies,
|
|
4680
|
+
* Optimystic's cohort for every control block is this node alone: a pass started at the
|
|
4681
|
+
* open writes (the first self-record publish) and reads on a cohort of one.
|
|
4682
|
+
*/
|
|
4683
|
+
onControlSiblingConnectionChange(remotePeer, change) {
|
|
4684
|
+
if (!this.authorizedControlPeers.has(remotePeer)) {
|
|
4685
|
+
return;
|
|
4374
4686
|
}
|
|
4687
|
+
if (change === 'open') {
|
|
4688
|
+
// NOTE: a machine removed from the cadre that has not learned of it keeps redialing
|
|
4689
|
+
// its siblings, each of which admits it provisionally and hangs up at the deadline
|
|
4690
|
+
// (28.5 s at the default link), and every such open drops the views here and once
|
|
4691
|
+
// more past the read-repair window: two storage reloads, with their cohort consults
|
|
4692
|
+
// refused, per cycle until it learns. If a removed machine's churn ever shows up,
|
|
4693
|
+
// skip both drops for a peer whose previous connection the remote closed inside the
|
|
4694
|
+
// provisional deadline.
|
|
4695
|
+
this.controlDatabase?.invalidateMembershipViews();
|
|
4696
|
+
this.armMembershipViewReload();
|
|
4697
|
+
}
|
|
4698
|
+
// NOTE: a close starts a pass at once, and a sibling in its bring-up quiet period lets the
|
|
4699
|
+
// dial resolve and then drops the connection (`createMembershipConnectionGater`'s deny
|
|
4700
|
+
// timing), so a restarting sibling costs one dial-connect-close pass after another for its
|
|
4701
|
+
// whole bring-up (5-6 passes in ~200 ms against a claimed node restarting locally). If a
|
|
4702
|
+
// slow device's bring-up ever makes that churn show up, delay the pass a close starts when
|
|
4703
|
+
// the previous connection to that sibling lived only a few milliseconds.
|
|
4704
|
+
log('Control connection to sibling %s %s; reconcile pass triggered', remotePeer, change);
|
|
4705
|
+
this.controlReconcileRuns?.trigger(0);
|
|
4706
|
+
}
|
|
4707
|
+
/**
|
|
4708
|
+
* Drop the membership views again {@link MEMBERSHIP_VIEW_RECONNECT_RELOAD_MS} after a
|
|
4709
|
+
* sibling connects, and start a pass whose reads reload them. This node's reads before
|
|
4710
|
+
* the connection stamped Optimystic's read-repair window on the control blocks, so the
|
|
4711
|
+
* reload the open itself starts is served from the local copy with no cohort consult,
|
|
4712
|
+
* and the view would keep rows that predate the sibling's commits until its maximum age.
|
|
4713
|
+
* The reload past the window starts the consults; one that brings a newer revision
|
|
4714
|
+
* raises the table change that drops the view and refreshes the gate. Re-armed by every
|
|
4715
|
+
* open, so it runs one window after the latest.
|
|
4716
|
+
*
|
|
4717
|
+
* NOTE: a consult Optimystic defers (16 already in flight) starts nothing and leaves the block
|
|
4718
|
+
* unstamped, so the view keeps its rows until the first read past its maximum age, which on an
|
|
4719
|
+
* idle node is the reconcile backstop. If `cluster-tx:read-repair-deferred` lines show at a
|
|
4720
|
+
* reconnect, re-arm this reload once more instead of waiting for the backstop.
|
|
4721
|
+
*/
|
|
4722
|
+
armMembershipViewReload() {
|
|
4723
|
+
if (this.membershipViewReloadTimer) {
|
|
4724
|
+
clearTimeout(this.membershipViewReloadTimer);
|
|
4725
|
+
}
|
|
4726
|
+
this.membershipViewReloadTimer = setTimeout(() => {
|
|
4727
|
+
this.membershipViewReloadTimer = null;
|
|
4728
|
+
log('Read-repair window passed since a sibling connected; reloading the membership views');
|
|
4729
|
+
this.controlDatabase?.invalidateMembershipViews();
|
|
4730
|
+
this.controlReconcileRuns?.trigger(0);
|
|
4731
|
+
}, MEMBERSHIP_VIEW_RECONNECT_RELOAD_MS);
|
|
4732
|
+
// Node-only: don't keep the event loop alive solely for this timer.
|
|
4733
|
+
this.membershipViewReloadTimer?.unref?.();
|
|
4375
4734
|
}
|
|
4376
4735
|
/**
|
|
4377
|
-
* Wire the control-connection
|
|
4378
|
-
*
|
|
4379
|
-
*
|
|
4380
|
-
*
|
|
4736
|
+
* Wire the control-connection listeners, which keep {@link identifiedControlPeers}: the
|
|
4737
|
+
* growth edge that drives write-while-alone re-replication, and the sibling reconcile
|
|
4738
|
+
* trigger ({@link onControlSiblingConnectionChange}). Wired in {@link start} (not the
|
|
4739
|
+
* delayed refresh path) so no early identify is missed, and reads back the peers that
|
|
4740
|
+
* identified before it ran ({@link noteIdentifiedControlPeers}); torn down in
|
|
4741
|
+
* {@link stopRecordRefresh}. Idempotent.
|
|
4381
4742
|
*/
|
|
4382
4743
|
wireControlConnectionListeners() {
|
|
4383
|
-
if (!this.controlNode || this.
|
|
4744
|
+
if (!this.controlNode || this.controlPeerIdentifyHandler) {
|
|
4384
4745
|
return;
|
|
4385
4746
|
}
|
|
4386
|
-
|
|
4387
|
-
this.
|
|
4388
|
-
|
|
4747
|
+
const partyId = this.config.controlNetwork.partyId;
|
|
4748
|
+
this.controlPeerIdentifyHandler = (event) => {
|
|
4749
|
+
if (servesControlDatabase(partyId, event.detail.protocols)) {
|
|
4750
|
+
this.onControlPeerIdentified(event.detail.peerId);
|
|
4751
|
+
}
|
|
4752
|
+
};
|
|
4753
|
+
this.controlConnectionCloseHandler = (event) => this.onControlConnectionClosed(event.detail.remotePeer);
|
|
4754
|
+
this.controlNode.addEventListener('peer:identify', this.controlPeerIdentifyHandler);
|
|
4389
4755
|
this.controlNode.addEventListener('connection:close', this.controlConnectionCloseHandler);
|
|
4756
|
+
void this.noteIdentifiedControlPeers();
|
|
4390
4757
|
}
|
|
4391
4758
|
// ============================================================================
|
|
4392
4759
|
// Device-token registry (control-network push-token publish + resolve)
|
|
@@ -4692,6 +5059,7 @@ export class CadreNode {
|
|
|
4692
5059
|
if (sAppId !== undefined) {
|
|
4693
5060
|
this.replicaSAppIds.set(strandId, sAppId);
|
|
4694
5061
|
log('Storage replica %s serves sApp %s', strandId, sAppId);
|
|
5062
|
+
this.strandWatcher?.sAppIdLearned(strandId);
|
|
4695
5063
|
}
|
|
4696
5064
|
}
|
|
4697
5065
|
catch (error) {
|
|
@@ -4800,6 +5168,7 @@ export class CadreNode {
|
|
|
4800
5168
|
this.strandLaunchRefusals.clear();
|
|
4801
5169
|
this.discoveredStrands.clear();
|
|
4802
5170
|
this.replicaSAppIds.clear();
|
|
5171
|
+
this.formedPartyKeys.clear();
|
|
4803
5172
|
// Drop delegate-admission state: the grants are scoped to the session that
|
|
4804
5173
|
// recorded them, so a stop()/start() cycle on this object must not keep
|
|
4805
5174
|
// admitting delegates announced under the previous one, nor carry stale
|
|
@@ -4820,6 +5189,10 @@ export class CadreNode {
|
|
|
4820
5189
|
// Detach the membership hub first: nothing this teardown does should drive a
|
|
4821
5190
|
// gate refresh, and a notification arriving after this point is a no-op.
|
|
4822
5191
|
if (this.controlDatabase) {
|
|
5192
|
+
// The database's registry would dispose the gate watch too; disposing it here, before
|
|
5193
|
+
// the listener goes, keeps a closing commit from scheduling a gate refresh.
|
|
5194
|
+
this.membershipGateWatch?.dispose();
|
|
5195
|
+
this.membershipGateWatch = null;
|
|
4823
5196
|
this.controlDatabase.setMembershipChangeListener(null);
|
|
4824
5197
|
this.controlDatabase.setGuardedDeleteListener(null);
|
|
4825
5198
|
this.controlDatabase.setControlWriteAbandonedListener(null);
|
|
@@ -5157,8 +5530,9 @@ export class CadreNode {
|
|
|
5157
5530
|
* registered, deliberately: both an explicit retry and the {@link StrandWatcher}'s
|
|
5158
5531
|
* automatic relaunch need it. A failed launch here hands the strand back to that
|
|
5159
5532
|
* relaunch — `StrandWatcher.forgetStrand` drops the id from the watcher's
|
|
5160
|
-
* `knownStrands
|
|
5161
|
-
*
|
|
5533
|
+
* `knownStrands`, records the backoff and wakes the watcher when it expires, so the
|
|
5534
|
+
* strand is re-attempted on the same ladder a watcher-driven failure gets
|
|
5535
|
+
* (`min(strandWatchInterval, 5 s) * 2^(failures-1)`, capped at 5 minutes). Because the
|
|
5162
5536
|
* config stays registered, the retry takes `handleStrandAdded`'s auto-launch branch:
|
|
5163
5537
|
* what the app sees is `strand:error` per failed retry and `strand:started` when one
|
|
5164
5538
|
* succeeds, never a second `strand:discovered`. {@link detachStrand} (reached via
|
|
@@ -5196,6 +5570,7 @@ export class CadreNode {
|
|
|
5196
5570
|
// A claim also overrides any earlier `stopStrand` of this id: the watcher must track
|
|
5197
5571
|
// the strand again, or a party-wide removal would never stop it here.
|
|
5198
5572
|
this.strandWatcher?.unsuppressStrand(strandRow.Id);
|
|
5573
|
+
this.strandWatcher?.sAppIdLearned(strandRow.Id);
|
|
5199
5574
|
log('Registered sAppConfig for strand %s (sApp: %s, founder: %s)', strandRow.Id, sAppConfig.id, founder ?? 'derived');
|
|
5200
5575
|
// Before the launch, so a foreign strand whose launch fails, or whose app is killed
|
|
5201
5576
|
// mid-launch, is still offered again: by the watcher's retry and by the next start.
|
|
@@ -5219,7 +5594,7 @@ export class CadreNode {
|
|
|
5219
5594
|
// Hand the strand back to the watcher's retry ladder. Harmless on the founder path
|
|
5220
5595
|
// (`foundStrand` → `publishStrand` + this call), where the watcher may never have
|
|
5221
5596
|
// offered the strand: the `knownStrands` delete is a no-op and the recorded backoff
|
|
5222
|
-
// only delays the watcher's own first attempt by one
|
|
5597
|
+
// only delays the watcher's own first attempt by one ladder step, which is what
|
|
5223
5598
|
// should happen right after an attempt that just failed.
|
|
5224
5599
|
//
|
|
5225
5600
|
// NOTE: the retry relaunches from the CONTROL row (or, for a strand joined from
|
|
@@ -5300,7 +5675,8 @@ export class CadreNode {
|
|
|
5300
5675
|
* (the key exists nowhere else). No app-data wipe is needed for either.
|
|
5301
5676
|
*
|
|
5302
5677
|
* The insert is signed through this node's owner signer ({@link getOwnerSigner}), in one
|
|
5303
|
-
* approval with a closed strand's party
|
|
5678
|
+
* approval and one control transaction with a closed strand's party key
|
|
5679
|
+
* ({@link ControlDatabase.insertFoundingRows}). The signer's key is persisted on the row
|
|
5304
5680
|
* as `FounderOwnerKey` (the schema pins the column to the verified signer), and this node's
|
|
5305
5681
|
* peer id as `FounderPeerId` (bound into the signature), which is what later lets any
|
|
5306
5682
|
* launch derive "this machine is the founder" from the row alone.
|
|
@@ -5311,7 +5687,8 @@ export class CadreNode {
|
|
|
5311
5687
|
*
|
|
5312
5688
|
* A CLOSED strand's publish also seats this party's own membership identity key —
|
|
5313
5689
|
* the `StrandPartyKey` row {@link ensureStrandPartyKey} documents — which is what the
|
|
5314
|
-
* founder bootstrap derives `Member.Key`/`Manager.MemberKey` from
|
|
5690
|
+
* founder bootstrap derives `Member.Key`/`Manager.MemberKey` from; a key the party already
|
|
5691
|
+
* holds for the id is kept. The
|
|
5315
5692
|
* `memberPrivateKey` ARGUMENT stays the strand-wide read secret formation hands to
|
|
5316
5693
|
* joiners; it derives nobody's identity.
|
|
5317
5694
|
*
|
|
@@ -5331,23 +5708,8 @@ export class CadreNode {
|
|
|
5331
5708
|
*/
|
|
5332
5709
|
async publishStrand(strandId, type = 'o', memberPrivateKey, sign) {
|
|
5333
5710
|
const signer = this.requireOwnerSigner(`publish strand ${strandId}`);
|
|
5334
|
-
|
|
5335
|
-
|
|
5336
|
-
const trimmed = requireNonBlank(strandId, 'strand id');
|
|
5337
|
-
// Refuse an unusable id HERE, not only at launch. `addStrand` already asserts, but it
|
|
5338
|
-
// runs after this row is written, and this row replicates: an id no node can turn into
|
|
5339
|
-
// a storage scope key would otherwise reach the whole party's control database and be
|
|
5340
|
-
// declined once per member. Asserted after the trim so the id checked is the id stored.
|
|
5341
|
-
assertStrandScopeKey(trimmed);
|
|
5342
|
-
// FounderPeerId records THIS machine as the founder, which is what later lets a relaunch
|
|
5343
|
-
// derive founder-ness; ControlDatabase.insertStrand writes the same id (its own node's).
|
|
5344
|
-
const desired = {
|
|
5345
|
-
Id: trimmed,
|
|
5346
|
-
Type: type,
|
|
5347
|
-
MemberPrivateKey: memberPrivateKey ?? null,
|
|
5348
|
-
FounderOwnerKey: signer.publicKey,
|
|
5349
|
-
FounderPeerId: this.controlNode.peerId.toString(),
|
|
5350
|
-
};
|
|
5711
|
+
const desired = this.founderStrandRow(strandId, type, memberPrivateKey, signer);
|
|
5712
|
+
const trimmed = desired.Id;
|
|
5351
5713
|
// Read first rather than leaning on the insert's collision for BOTH readings. The
|
|
5352
5714
|
// ordinary resume (an app killed after its own publish committed) then needs no error
|
|
5353
5715
|
// matching at all: the collision path can only recognise itself by the engine's message
|
|
@@ -5367,24 +5729,57 @@ export class CadreNode {
|
|
|
5367
5729
|
log('publishStrand(%s): already published (type %s) with identical content — no-op', trimmed, type);
|
|
5368
5730
|
// A closed strand THIS machine published must hold a party key (its own membership
|
|
5369
5731
|
// identity — see ensureStrandPartyKey). Insert-if-absent, so the ordinary repeat
|
|
5370
|
-
// is a read-only no-op and a
|
|
5371
|
-
//
|
|
5732
|
+
// is a read-only no-op and a closed row stored without its key heals here. A row
|
|
5733
|
+
// another machine published is that machine's to mint for.
|
|
5372
5734
|
if (type === 'c' && this.isSelfFoundedRow(matched)) {
|
|
5373
5735
|
await this.ensureStrandPartyKey(trimmed, undefined, sign);
|
|
5374
5736
|
}
|
|
5375
5737
|
return matched;
|
|
5376
5738
|
}
|
|
5377
|
-
return await
|
|
5739
|
+
return (await this.writeFoundingRows(desired, undefined, signer, sign)).strandRow;
|
|
5740
|
+
}
|
|
5741
|
+
/**
|
|
5742
|
+
* The `Strand` row this machine publishes for `strandId`, as {@link publishStrand} and
|
|
5743
|
+
* {@link foundStrand} write it: the id trimmed and checked, this node's owner key and peer id
|
|
5744
|
+
* as its founder.
|
|
5745
|
+
*/
|
|
5746
|
+
founderStrandRow(strandId, type, memberPrivateKey, signer) {
|
|
5747
|
+
// Trim/reject here so the id that lands matches the one unpublishStrand looks up: it
|
|
5748
|
+
// trims too, and an untrimmed row would be unreachable by the same string.
|
|
5749
|
+
const trimmed = requireNonBlank(strandId, 'strand id');
|
|
5750
|
+
// Refuse an unusable id HERE, not only at launch. `addStrand` already asserts, but it
|
|
5751
|
+
// runs after this row is written, and this row replicates: an id no node can turn into
|
|
5752
|
+
// a storage scope key would otherwise reach the whole party's control database and be
|
|
5753
|
+
// declined once per member. Asserted after the trim so the id checked is the id stored.
|
|
5754
|
+
assertStrandScopeKey(trimmed);
|
|
5755
|
+
// FounderPeerId records THIS machine as the founder, which is what later lets a relaunch
|
|
5756
|
+
// derive founder-ness; ControlDatabase.insertFoundingRows writes the same id (its own node's).
|
|
5757
|
+
return {
|
|
5758
|
+
Id: trimmed,
|
|
5759
|
+
Type: type,
|
|
5760
|
+
MemberPrivateKey: memberPrivateKey ?? null,
|
|
5761
|
+
FounderOwnerKey: signer.publicKey,
|
|
5762
|
+
FounderPeerId: this.controlNode.peerId.toString(),
|
|
5763
|
+
};
|
|
5378
5764
|
}
|
|
5379
5765
|
/**
|
|
5380
|
-
* The write half of {@link publishStrand}
|
|
5381
|
-
*
|
|
5382
|
-
*
|
|
5766
|
+
* The write half of {@link publishStrand} and {@link foundStrand}: under one owner approval,
|
|
5767
|
+
* insert `desired`, a closed strand's party key and the requested invitation in one control
|
|
5768
|
+
* transaction ({@link ControlDatabase.insertFoundingRows}), adopting a matching row a
|
|
5769
|
+
* concurrent founder landed first. Reads nothing first: the caller has just read the
|
|
5770
|
+
* `Strand` row, and a stored key or row is resolved from the insert's own rejection.
|
|
5383
5771
|
*/
|
|
5384
|
-
async
|
|
5385
|
-
const
|
|
5772
|
+
async writeFoundingRows(desired, invite, signer, sign) {
|
|
5773
|
+
const action = invite
|
|
5774
|
+
? `Publish the strand ${desired.Id} to the cadre, with an invitation to the app ${invite.sAppId}`
|
|
5775
|
+
: `Publish the strand ${desired.Id} to the cadre`;
|
|
5776
|
+
return await withinOwnerApproval(signer, { partyId: this.partyId, action }, sign, (approved) => this.insertOrAdoptFounding(desired, invite, signer, approved));
|
|
5777
|
+
}
|
|
5778
|
+
/** {@link writeFoundingRows} inside its approval. */
|
|
5779
|
+
async insertOrAdoptFounding(desired, invite, signer, sign) {
|
|
5780
|
+
let partyKey;
|
|
5386
5781
|
try {
|
|
5387
|
-
await this.
|
|
5782
|
+
partyKey = await this.seatFoundingRows(desired, invite, signer, sign);
|
|
5388
5783
|
}
|
|
5389
5784
|
catch (error) {
|
|
5390
5785
|
// Only the uniqueness collision is a candidate for the idempotent reading; every
|
|
@@ -5392,26 +5787,79 @@ export class CadreNode {
|
|
|
5392
5787
|
if (!isStrandIdConflict(error, 'Strand')) {
|
|
5393
5788
|
throw error;
|
|
5394
5789
|
}
|
|
5395
|
-
const
|
|
5396
|
-
|
|
5397
|
-
|
|
5398
|
-
|
|
5399
|
-
|
|
5790
|
+
const strandRow = await this.adoptConcurrentFounding(desired, error);
|
|
5791
|
+
// The rejection rolled the invitation back with the row; a caller asking for one gets one.
|
|
5792
|
+
if (invite) {
|
|
5793
|
+
await this.publishFoundingInviteIfAbsent(strandRow.Id, invite, sign);
|
|
5794
|
+
}
|
|
5795
|
+
return { strandRow };
|
|
5796
|
+
}
|
|
5797
|
+
log('Published strand %s (type %s) to control DB under owner %s', desired.Id, desired.Type, signer.publicKey);
|
|
5798
|
+
if (invite) {
|
|
5799
|
+
this.registerPublishedInvitation(invite.token, invite.expiresAtMs);
|
|
5800
|
+
}
|
|
5801
|
+
return { strandRow: desired, partyKey };
|
|
5802
|
+
}
|
|
5803
|
+
/**
|
|
5804
|
+
* Insert `desired`'s founding rows, minting a closed strand's party key — this party's own
|
|
5805
|
+
* membership identity, which the founder bootstrap derives `Member`/`Manager` from, distinct
|
|
5806
|
+
* from the row's shared `MemberPrivateKey` read secret. A key already stored for the id with
|
|
5807
|
+
* no strand row (one {@link ensureStrandPartyKey} seated, which it would adopt too) refuses
|
|
5808
|
+
* that transaction whole; the rows are then written again without a key.
|
|
5809
|
+
*
|
|
5810
|
+
* @returns The party key this seated; undefined when it seated none, so the launch reads the stored one.
|
|
5811
|
+
*/
|
|
5812
|
+
async seatFoundingRows(desired, invite, signer, sign) {
|
|
5813
|
+
const controlDatabase = this.controlDatabase;
|
|
5814
|
+
const strand = { id: desired.Id, type: desired.Type, memberPrivateKey: desired.MemberPrivateKey ?? undefined };
|
|
5815
|
+
if (desired.Type !== 'c') {
|
|
5816
|
+
await controlDatabase.insertFoundingRows({ strand, formationInvite: invite }, signer.publicKey, sign);
|
|
5817
|
+
return undefined;
|
|
5818
|
+
}
|
|
5819
|
+
const partyKey = await generateStrandMemberKey();
|
|
5820
|
+
try {
|
|
5821
|
+
await controlDatabase.insertFoundingRows({ strand, partyKey, formationInvite: invite }, signer.publicKey, sign);
|
|
5822
|
+
return partyKey;
|
|
5823
|
+
}
|
|
5824
|
+
catch (error) {
|
|
5825
|
+
if (!isStrandIdConflict(error, 'StrandPartyKey')) {
|
|
5400
5826
|
throw error;
|
|
5401
5827
|
}
|
|
5402
|
-
log('publishStrand(%s):
|
|
5403
|
-
|
|
5828
|
+
log('publishStrand(%s): this party already holds a key for the id; publishing the row and adopting that key', desired.Id);
|
|
5829
|
+
await controlDatabase.insertFoundingRows({ strand, formationInvite: invite }, signer.publicKey, sign);
|
|
5830
|
+
return undefined;
|
|
5404
5831
|
}
|
|
5405
|
-
|
|
5406
|
-
|
|
5407
|
-
|
|
5408
|
-
|
|
5409
|
-
|
|
5410
|
-
|
|
5411
|
-
|
|
5412
|
-
|
|
5832
|
+
}
|
|
5833
|
+
/**
|
|
5834
|
+
* Resolve a founding refused on `Strand.Id`: re-read the row that landed and adopt it when it
|
|
5835
|
+
* matches `desired` — a concurrent founder, such as two machines of one party on one id —
|
|
5836
|
+
* or throw naming the columns that differ.
|
|
5837
|
+
*/
|
|
5838
|
+
async adoptConcurrentFounding(desired, error) {
|
|
5839
|
+
const landed = await this.controlDatabase.queryStrand(desired.Id);
|
|
5840
|
+
if (!landed) {
|
|
5841
|
+
// Collided, yet no row is readable: not the race this branch handles (a row
|
|
5842
|
+
// deleted between the rejection and the re-read, or a stale local view). Surface
|
|
5843
|
+
// the original rather than guessing.
|
|
5844
|
+
throw error;
|
|
5413
5845
|
}
|
|
5414
|
-
|
|
5846
|
+
log('publishStrand(%s): lost a concurrent founding race; re-read the landed row', desired.Id);
|
|
5847
|
+
return requireMatchingStrandRow(landed, desired, 'landed concurrently from another founder');
|
|
5848
|
+
}
|
|
5849
|
+
/**
|
|
5850
|
+
* Publish a founding's invitation on its own, bound to `strandId`, for a founding that adopted
|
|
5851
|
+
* a stored row rather than writing its own — unless the invitation is already stored, bound to
|
|
5852
|
+
* this strand, as a repeat of a founding whose transaction committed leaves it.
|
|
5853
|
+
*
|
|
5854
|
+
* @param sign - The open approval's `sign`; omitted, the publish asks for its own.
|
|
5855
|
+
*/
|
|
5856
|
+
async publishFoundingInviteIfAbsent(strandId, invite, sign) {
|
|
5857
|
+
if ((await this.controlDatabase.queryFormationInvite(invite.token))?.strandId === strandId) {
|
|
5858
|
+
log('foundStrand(%s): the invitation is already published', strandId);
|
|
5859
|
+
return;
|
|
5860
|
+
}
|
|
5861
|
+
const { token, sAppId, ...options } = invite;
|
|
5862
|
+
await this.publishFormationInvite(token, sAppId, { ...options, strandId }, sign);
|
|
5415
5863
|
}
|
|
5416
5864
|
async foundStrand(config, options) {
|
|
5417
5865
|
const tTotal = performance.now();
|
|
@@ -5438,17 +5886,18 @@ export class CadreNode {
|
|
|
5438
5886
|
* so both forms of `foundStrand` reject on it before returning.
|
|
5439
5887
|
*/
|
|
5440
5888
|
async publishFoundingRow(config) {
|
|
5441
|
-
const { strandId, type = 'o', memberPrivateKey } = config;
|
|
5889
|
+
const { strandId, type = 'o', memberPrivateKey, formationInvite, sAppConfig } = config;
|
|
5442
5890
|
if (!this._running || !this.controlDatabase) {
|
|
5443
5891
|
throw new Error(`CadreNode must be started before attempting to found strand ${strandId}`);
|
|
5444
5892
|
}
|
|
5445
5893
|
const controlDatabase = this.controlDatabase;
|
|
5446
5894
|
const trimmed = requireNonBlank(strandId, 'strand id');
|
|
5895
|
+
const invite = formationInvite && { ...formationInvite, sAppId: sAppConfig.id };
|
|
5447
5896
|
const timed = (step, op) => timedStep('foundStrand', trimmed, step, op);
|
|
5448
5897
|
const published = await timed('queryStrand', () => controlDatabase.queryStrand(trimmed));
|
|
5449
|
-
const strandRow = published
|
|
5450
|
-
? this.
|
|
5451
|
-
: await timed('publishStrand', () => this.
|
|
5898
|
+
const { strandRow, partyKey } = published
|
|
5899
|
+
? await this.resumeFounding(published, type, memberPrivateKey, invite)
|
|
5900
|
+
: await timed('publishStrand', () => this.publishFoundingStrand(trimmed, type, memberPrivateKey, invite));
|
|
5452
5901
|
// Derived, not hardcoded `true`: adopting a row another machine published means that
|
|
5453
5902
|
// machine runs the bootstrap — this one must attach or it would write a second Header.
|
|
5454
5903
|
const founded = this.isSelfFoundedRow(strandRow);
|
|
@@ -5457,15 +5906,34 @@ export class CadreNode {
|
|
|
5457
5906
|
'(FounderPeerId is not this node\'s peer id) — attaching as a joiner; ' +
|
|
5458
5907
|
'the founder bootstrap runs on that machine', trimmed);
|
|
5459
5908
|
}
|
|
5460
|
-
return { strandRow, founded };
|
|
5909
|
+
return { strandRow, founded, partyKey };
|
|
5910
|
+
}
|
|
5911
|
+
/** {@link publishFoundingRow} when no row is stored: write the founding rows under this node's owner. */
|
|
5912
|
+
publishFoundingStrand(strandId, type, memberPrivateKey, invite) {
|
|
5913
|
+
const signer = this.requireOwnerSigner(`publish strand ${strandId}`);
|
|
5914
|
+
return this.writeFoundingRows(this.founderStrandRow(strandId, type, memberPrivateKey, signer), invite, signer);
|
|
5915
|
+
}
|
|
5916
|
+
/**
|
|
5917
|
+
* {@link publishFoundingRow} over a stored row: adopt it ({@link adoptPublishedStrand}), then
|
|
5918
|
+
* publish the requested invitation on its own, under its own approval.
|
|
5919
|
+
*/
|
|
5920
|
+
async resumeFounding(published, type, memberPrivateKey, invite) {
|
|
5921
|
+
const strandRow = this.adoptPublishedStrand(published, type, memberPrivateKey);
|
|
5922
|
+
if (invite) {
|
|
5923
|
+
await timedStep('foundStrand', strandRow.Id, 'publishFormationInvite', () => this.publishFoundingInviteIfAbsent(strandRow.Id, invite));
|
|
5924
|
+
}
|
|
5925
|
+
return { strandRow };
|
|
5461
5926
|
}
|
|
5462
5927
|
/**
|
|
5463
5928
|
* The start half of {@link foundStrand}. Its timing lines (`addStrand: start`, then the
|
|
5464
5929
|
* duration or `failed after`) are logged when the start settles, also in the background
|
|
5465
5930
|
* form, so a start that hangs still shows as a start line with no end line.
|
|
5931
|
+
*
|
|
5932
|
+
* A party key the publish half just seated is handed to the launch, which would otherwise
|
|
5933
|
+
* read back the row the same founding wrote.
|
|
5466
5934
|
*/
|
|
5467
|
-
attachFoundingRow({ strandRow, founded }, { sAppConfig, awaitFirstSync }) {
|
|
5468
|
-
return timedStep('foundStrand', strandRow.Id, 'addStrand', () => this.addStrand({ strandRow, sAppConfig, founder: founded, awaitFirstSync }));
|
|
5935
|
+
attachFoundingRow({ strandRow, founded, partyKey }, { sAppConfig, awaitFirstSync }) {
|
|
5936
|
+
return timedStep('foundStrand', strandRow.Id, 'addStrand', () => this.addStrand({ strandRow, sAppConfig, founder: founded, awaitFirstSync, partyMemberPrivateKey: partyKey }));
|
|
5469
5937
|
}
|
|
5470
5938
|
/**
|
|
5471
5939
|
* Resume onto an already-published `Strand` row: keep its stored content, reject a type
|
|
@@ -5494,9 +5962,10 @@ export class CadreNode {
|
|
|
5494
5962
|
* inverse of {@link publishStrand}.
|
|
5495
5963
|
*
|
|
5496
5964
|
* Party-wide, unlike {@link stopStrand}, which only stops the strand on THIS node: every
|
|
5497
|
-
* cadre node watching the table
|
|
5498
|
-
*
|
|
5499
|
-
*
|
|
5965
|
+
* cadre node watching the table hears the row's removal and stops its own instance (a
|
|
5966
|
+
* machine outside the `Strand` block's cohort, on a cadre larger than 16, only at its
|
|
5967
|
+
* watcher's backstop read, default 5 minutes). This node converges immediately — the
|
|
5968
|
+
* method forces a watcher run and stops any still-running local instance before resolving. Removing the row
|
|
5500
5969
|
* removes OUR party's participation only: other parties in the strand keep their own
|
|
5501
5970
|
* rows and the strand network carries on, so no cross-party sign-off is involved — this
|
|
5502
5971
|
* is an owner-signed control-plane write like every other, not "destroy the network".
|
|
@@ -5582,16 +6051,16 @@ export class CadreNode {
|
|
|
5582
6051
|
// deadline rather than dropping it (a binding cleared late is still worth more than
|
|
5583
6052
|
// one never cleared).
|
|
5584
6053
|
await this.strandManager.clearOwnMemberPeerBinding(trimmed);
|
|
5585
|
-
// Converge locally
|
|
5586
|
-
// onStrandRemoved for a strand it tracked; the explicit teardown below
|
|
5587
|
-
// whose strandFilter never admitted this strand (the watcher never knew
|
|
5588
|
-
// never fire). StrandInstanceManager.stopStrand no-ops on an unknown id, so the two
|
|
6054
|
+
// Converge locally before resolving rather than when the write wakes the watcher. The
|
|
6055
|
+
// watcher fires onStrandRemoved for a strand it tracked; the explicit teardown below
|
|
6056
|
+
// covers a node whose strandFilter never admitted this strand (the watcher never knew
|
|
6057
|
+
// it, so it will never fire). StrandInstanceManager.stopStrand no-ops on an unknown id, so the two
|
|
5589
6058
|
// paths cannot double-stop into an error.
|
|
5590
6059
|
//
|
|
5591
6060
|
// detachStrand, NOT stopStrand: this is a party-wide removal, not a local abandonment
|
|
5592
6061
|
// of a strand that still exists, so it must not SUPPRESS the id in the watcher. The
|
|
5593
6062
|
// row is already gone — there is nothing to suppress — and an id re-published before
|
|
5594
|
-
// the next
|
|
6063
|
+
// the next run's suppression cleanup would then never be offered again this session.
|
|
5595
6064
|
// (Same reasoning as handleStrandRemoved, which also avoids stopStrand's `_running`
|
|
5596
6065
|
// guard; here the guard is already satisfied by requireOwnerSigner above.)
|
|
5597
6066
|
await this.strandWatcher?.forcePoll();
|
|
@@ -5614,9 +6083,11 @@ export class CadreNode {
|
|
|
5614
6083
|
* as `MemberPrivateKey`; docs/strands.md → "Closed-Strand Member Key Handling") and is
|
|
5615
6084
|
* never put on the formation wire.
|
|
5616
6085
|
*
|
|
5617
|
-
*
|
|
5618
|
-
*
|
|
5619
|
-
*
|
|
6086
|
+
* A founding seats the key in the same transaction as the strand row, not through here.
|
|
6087
|
+
* Called internally by {@link publishStrand}'s repeat of a published closed strand and by a
|
|
6088
|
+
* founder launch that finds no row, which heal a closed strand row stored without its key
|
|
6089
|
+
* (one seated by `ControlDatabase.insertStrand` directly, or whose key was removed); public
|
|
6090
|
+
* so a test harness — or a joiner flow that persists a formation-issued identity — can seat
|
|
5620
6091
|
* a specific key deliberately.
|
|
5621
6092
|
*
|
|
5622
6093
|
* @param strandId - The strand the key is this party's identity for.
|
|
@@ -5664,8 +6135,8 @@ export class CadreNode {
|
|
|
5664
6135
|
* Resolve the party membership key a closed strand's launch threads into the founder
|
|
5665
6136
|
* bootstrap: the explicit attach-time key when given, else the party's persisted
|
|
5666
6137
|
* `StrandPartyKey` row, else — on the one machine the row names as founder (`FounderPeerId`)
|
|
5667
|
-
* — a freshly minted-and-persisted key, which seats the identity for a
|
|
5668
|
-
*
|
|
6138
|
+
* — a freshly minted-and-persisted key, which seats the identity for a closed row stored
|
|
6139
|
+
* without one (see {@link ensureStrandPartyKey}). Everyone else resolves undefined: non-founding
|
|
5669
6140
|
* machines never mint (no mint race between a party's machines), and only a founder
|
|
5670
6141
|
* bootstrap needs the key at all.
|
|
5671
6142
|
*
|
|
@@ -5685,6 +6156,11 @@ export class CadreNode {
|
|
|
5685
6156
|
if (explicitKey !== undefined) {
|
|
5686
6157
|
return explicitKey;
|
|
5687
6158
|
}
|
|
6159
|
+
const formed = this.formedPartyKeys.get(strand.Id);
|
|
6160
|
+
if (formed !== undefined) {
|
|
6161
|
+
this.formedPartyKeys.delete(strand.Id);
|
|
6162
|
+
return formed;
|
|
6163
|
+
}
|
|
5688
6164
|
if (!this.controlDatabase) {
|
|
5689
6165
|
return undefined;
|
|
5690
6166
|
}
|
|
@@ -5708,8 +6184,8 @@ export class CadreNode {
|
|
|
5708
6184
|
* Counterpart to {@link createOpenInvitation}, which only mints the
|
|
5709
6185
|
* out-of-band {@link OpenInvitation} envelope: persisting the matching
|
|
5710
6186
|
* `FormationInvite` row is what makes the token *redeemable* — the host's
|
|
5711
|
-
* {@link ControlFormationUsageRecorder}
|
|
5712
|
-
*
|
|
6187
|
+
* {@link ControlFormationUsageRecorder} reads it (`readToken`) to check a
|
|
6188
|
+
* redemption. A host minting a closed-strand invite does both (mint +
|
|
5713
6189
|
* publish), exactly as the integration harness's `createInvitation` does.
|
|
5714
6190
|
*
|
|
5715
6191
|
* Signs through the same owner signer as {@link publishStrand}, whose key must be an
|
|
@@ -5726,12 +6202,19 @@ export class CadreNode {
|
|
|
5726
6202
|
async publishFormationInvite(token, sAppId, options = {}, sign) {
|
|
5727
6203
|
const signer = this.requireOwnerSigner(`publish formation invite ${token}`);
|
|
5728
6204
|
await withinOwnerApproval(signer, { partyId: this.partyId, action: `Publish an invitation to the app ${sAppId}` }, sign, (approved) => this.controlDatabase.insertFormationInvite(token, sAppId, signer.publicKey, approved, options));
|
|
5729
|
-
|
|
5730
|
-
// it locally opens the connection gate's formation exemption immediately
|
|
5731
|
-
// rather than waiting for the durable row to become readable.
|
|
5732
|
-
this.strandSolicitationService?.registerMintedInvitation(token, options.expiresAtMs ?? Number.POSITIVE_INFINITY);
|
|
6205
|
+
this.registerPublishedInvitation(token, options.expiresAtMs);
|
|
5733
6206
|
log('Published formation invite %s (sApp %s) under owner %s', token, sAppId, signer.publicKey);
|
|
5734
6207
|
}
|
|
6208
|
+
/**
|
|
6209
|
+
* Register an invitation this node just published with the strand solicitation service. A
|
|
6210
|
+
* host may publish one whose token was minted elsewhere; registering it locally opens the
|
|
6211
|
+
* connection gate's formation exemption immediately rather than waiting for the durable row
|
|
6212
|
+
* to become readable. A token {@link createOpenInvitation} minted here is already registered,
|
|
6213
|
+
* and this records the same expiry again.
|
|
6214
|
+
*/
|
|
6215
|
+
registerPublishedInvitation(token, expiresAtMs) {
|
|
6216
|
+
this.strandSolicitationService?.registerMintedInvitation(token, expiresAtMs ?? Number.POSITIVE_INFINITY);
|
|
6217
|
+
}
|
|
5735
6218
|
/**
|
|
5736
6219
|
* Enroll an approver public key allowed to sign off on `ValidationUrl` redemptions.
|
|
5737
6220
|
*
|
|
@@ -5839,7 +6322,7 @@ export class CadreNode {
|
|
|
5839
6322
|
* fan-out, so a rediscovery costs nothing beyond the map lookup plus the (pure)
|
|
5840
6323
|
* founder derivation. NOT a silent no-op any more when the launch resolves as a
|
|
5841
6324
|
* FOUNDER: the tracked instance may have been launched first as a joiner (an app
|
|
5842
|
-
* attach, or this node's own watcher
|
|
6325
|
+
* attach, or this node's own watcher run winning the `resolveCohortSeed` window
|
|
5843
6326
|
* below), which used to drop the founder request and leave the strand headerless —
|
|
5844
6327
|
* now the tracked instance is founded in place
|
|
5845
6328
|
* ({@link StrandInstanceManager.foundExistingStrand}), waking it first if quiesced
|
|
@@ -5851,9 +6334,9 @@ export class CadreNode {
|
|
|
5851
6334
|
*
|
|
5852
6335
|
* An absent `sAppConfig` launches a storage replica ({@link CadreNodeConfig.hostUnclaimedStrands}),
|
|
5853
6336
|
* which is ALWAYS a joiner whatever the row says: the founder bootstrap writes the sApp
|
|
5854
|
-
* into `Strand.Header`, and a replica has none. So a self-founded row whose watcher
|
|
6337
|
+
* into `Strand.Header`, and a replica has none. So a self-founded row whose watcher run
|
|
5855
6338
|
* wins the race against its app's `addStrand` after a restart comes up as a replica — and
|
|
5856
|
-
* so does a `foundStrand` whose publish a watcher
|
|
6339
|
+
* so does a `foundStrand` whose publish a watcher run saw before its attach. A claim
|
|
5857
6340
|
* that finds a tracked replica upgrades it in place, over the same node and store
|
|
5858
6341
|
* ({@link StrandInstanceManager.attachSApp}), BEFORE any founding: the founder bootstrap
|
|
5859
6342
|
* needs the sApp the attach supplies.
|
|
@@ -5885,8 +6368,8 @@ export class CadreNode {
|
|
|
5885
6368
|
}
|
|
5886
6369
|
// A closed strand's launch carries the party's OWN membership identity key: the
|
|
5887
6370
|
// explicit attach-time key, else the control-layer StrandPartyKey row — minted here
|
|
5888
|
-
// when this machine is the row-derived founder and no row exists yet (a
|
|
5889
|
-
//
|
|
6371
|
+
// when this machine is the row-derived founder and no row exists yet (a closed row
|
|
6372
|
+
// stored without its key). A joiner with no persisted key threads undefined;
|
|
5890
6373
|
// only a FOUNDER bootstrap needs the key, and that path throws loudly without one
|
|
5891
6374
|
// (StrandDatabase).
|
|
5892
6375
|
const partyMemberPrivateKey = strand.Type === 'c'
|
|
@@ -6208,7 +6691,7 @@ export class CadreNode {
|
|
|
6208
6691
|
*
|
|
6209
6692
|
* Timestamps are recorded OPTIMISTICALLY, whatever `collectStrandAddrs` reports
|
|
6210
6693
|
* per peer: recording only on success would re-announce to an unreachable relay
|
|
6211
|
-
* on every reconcile
|
|
6694
|
+
* on every reconcile pass, one dial timeout each. A failed INITIAL announce costs
|
|
6212
6695
|
* the strand supervisor its first attempt only (the relay denies the
|
|
6213
6696
|
* reservation; every re-drive re-announces first through
|
|
6214
6697
|
* {@link announceDelegateToRelay}); a failed REFRESH retries within
|
|
@@ -6331,9 +6814,11 @@ export class CadreNode {
|
|
|
6331
6814
|
*
|
|
6332
6815
|
* Due times are per (sibling, strand) and set from each sibling's own outcome, so
|
|
6333
6816
|
* a sibling that connects late — a phone joining after the party's always-on
|
|
6334
|
-
* machines — is asked on the
|
|
6335
|
-
* (its view of the membership may not include us yet) is retried
|
|
6336
|
-
* {@link STRAND_PEER_ADDR_RETRY_MS} rather than
|
|
6817
|
+
* machines — is asked on the pass its connection starts, and one that could not
|
|
6818
|
+
* answer or refused (its view of the membership may not include us yet) is retried
|
|
6819
|
+
* within {@link STRAND_PEER_ADDR_RETRY_MS} rather than
|
|
6820
|
+
* {@link STRAND_PEER_ADDR_REFRESH_MS}; the pass that stamped the retry arms a wake
|
|
6821
|
+
* for it ({@link armReconcileWake}).
|
|
6337
6822
|
*
|
|
6338
6823
|
* Distinct from {@link refreshDelegateGrants}, deliberately: that pass covers
|
|
6339
6824
|
* RELAYS on a `DELEGATE_GRANT_TTL_MS / 2` throttle to keep circuit-relay
|
|
@@ -6382,15 +6867,17 @@ export class CadreNode {
|
|
|
6382
6867
|
*
|
|
6383
6868
|
* `connectedSiblingTargets`' membership read is unbounded, and with zero
|
|
6384
6869
|
* connections its answer is empty whatever the table holds, so decide it from the
|
|
6385
|
-
* connection list instead of paying for the read once a
|
|
6870
|
+
* connection list instead of paying for the read once a pass.
|
|
6386
6871
|
*
|
|
6387
|
-
* NOTE: otherwise the read runs on EVERY
|
|
6872
|
+
* NOTE: otherwise the read runs on EVERY pass with a running strand, not only when
|
|
6388
6873
|
* some (sibling, strand) is due: pruning a departed sibling needs the current
|
|
6389
6874
|
* target set, and whether anyone is due cannot be told without it (a connected
|
|
6390
|
-
* non-member never gets a due time to compare). One more `CadrePeer` read per
|
|
6391
|
-
*
|
|
6392
|
-
*
|
|
6393
|
-
*
|
|
6875
|
+
* non-member never gets a due time to compare). One more `CadrePeer` read per
|
|
6876
|
+
* pass, on top of the two `runReconcileControlCohort` already makes, all served
|
|
6877
|
+
* from the membership rows view: an in-memory scan while the view is current,
|
|
6878
|
+
* storage only after the table changed or the view's maximum age passed. Sharing
|
|
6879
|
+
* one row set across the pass would save no storage read and was declined (see
|
|
6880
|
+
* the NOTE there).
|
|
6394
6881
|
*
|
|
6395
6882
|
* NOTE: one strand-addr RPC per (running strand × due sibling) — each a tiny
|
|
6396
6883
|
* request/response on an already-open control connection. If a node ever runs
|
|
@@ -6473,8 +6960,8 @@ export class CadreNode {
|
|
|
6473
6960
|
*
|
|
6474
6961
|
* NOTE: every pass serialises the whole FRET table (`exportTable()`), verifies one
|
|
6475
6962
|
* signature per record and makes one `peerStore.merge` per peer, per running strand
|
|
6476
|
-
* per
|
|
6477
|
-
* nothing in libp2p's persistent peer store. If many strands or large tables make it
|
|
6963
|
+
* per pass — every change-driven pass and the 2-minute backstop. Fine at strand table
|
|
6964
|
+
* sizes, and an unchanged address set writes nothing in libp2p's persistent peer store. If many strands or large tables make it
|
|
6478
6965
|
* show up, read only the entries whose address stamp is near the one-hour mark.
|
|
6479
6966
|
*
|
|
6480
6967
|
* @returns the ids of the peers the table holds a usable record for.
|
|
@@ -6757,8 +7244,8 @@ export class CadreNode {
|
|
|
6757
7244
|
* The stop + emit are skipped when the strand manager holds no instance for `strandId` —
|
|
6758
7245
|
* e.g. a party owner that published a strand's row but never ran it locally, or an
|
|
6759
7246
|
* explicit {@link stopStrand} for an id this node never started. `hibernationManager`
|
|
6760
|
-
* untrack and the `sAppConfigs` / {@link strandLaunchRefusals} / {@link discoveredStrands}
|
|
6761
|
-
* deletes stay unconditional (all no-ops when there is nothing to remove), so a launch
|
|
7247
|
+
* untrack and the `sAppConfigs` / {@link strandLaunchRefusals} / {@link discoveredStrands} /
|
|
7248
|
+
* {@link formedPartyKeys} deletes stay unconditional (all no-ops when there is nothing to remove), so a launch
|
|
6762
7249
|
* that failed before an instance was ever tracked still gets its stray entries cleared.
|
|
6763
7250
|
*
|
|
6764
7251
|
* Dropping the {@link discoveredStrands} entry here covers both callers, and both
|
|
@@ -6775,6 +7262,7 @@ export class CadreNode {
|
|
|
6775
7262
|
this.sAppConfigs.delete(strandId);
|
|
6776
7263
|
this.strandLaunchRefusals.delete(strandId);
|
|
6777
7264
|
this.discoveredStrands.delete(strandId);
|
|
7265
|
+
this.formedPartyKeys.delete(strandId);
|
|
6778
7266
|
if (!this.strandManager.hasStrand(strandId)) {
|
|
6779
7267
|
log('detachStrand: strand %s not tracked locally — no-op (no stop, no strand:stopped)', strandId);
|
|
6780
7268
|
return;
|
|
@@ -7141,20 +7629,35 @@ export class CadreNode {
|
|
|
7141
7629
|
return this.controlDatabase;
|
|
7142
7630
|
}
|
|
7143
7631
|
/**
|
|
7144
|
-
* Open control-network connections right now
|
|
7145
|
-
*
|
|
7146
|
-
*
|
|
7147
|
-
*
|
|
7148
|
-
* It approximates the precise signal (the block's cluster size), and a caller that
|
|
7149
|
-
* samples it AFTER a write samples a slightly wider window than `committedAlone`
|
|
7150
|
-
* does inside the write itself. It exists so an embedder can warn an owner before a
|
|
7151
|
-
* control write that this machine currently sees none of its siblings, and report
|
|
7152
|
-
* after one that the write may not have travelled — a warning shown unconditionally
|
|
7153
|
-
* is a warning people learn to ignore.
|
|
7632
|
+
* Open control-network connections right now, of every kind: siblings, relays, bootstrap
|
|
7633
|
+
* nodes and peers still identifying. A report of connectivity, not of replication reach: a
|
|
7634
|
+
* control write can commit local-only with connections open, and whether one would is
|
|
7635
|
+
* {@link hasReachableControlSibling}.
|
|
7154
7636
|
*/
|
|
7155
7637
|
getControlConnectionCount() {
|
|
7156
7638
|
return this.controlNode?.getConnections().length ?? 0;
|
|
7157
7639
|
}
|
|
7640
|
+
/**
|
|
7641
|
+
* Whether a control write made now reaches another machine: a connected peer has identified
|
|
7642
|
+
* as serving the control database and is one this node sends control writes to (an
|
|
7643
|
+
* authorized member, or a device whose admission it is writing). Optimystic counts a peer
|
|
7644
|
+
* into a control block's cohort on those same terms; with none, a write commits on this node
|
|
7645
|
+
* alone and is queued for re-issue once one is reachable (docs/architecture.md → "Writes made
|
|
7646
|
+
* while alone").
|
|
7647
|
+
*
|
|
7648
|
+
* It exists so an embedder can warn an owner before a control write that this machine
|
|
7649
|
+
* currently reaches none of its siblings, and report after one that the write may not have
|
|
7650
|
+
* travelled; a warning shown unconditionally is a warning people learn to ignore. A caller
|
|
7651
|
+
* that samples it AFTER a write samples a slightly wider window than the write did.
|
|
7652
|
+
*/
|
|
7653
|
+
hasReachableControlSibling() {
|
|
7654
|
+
for (const peerId of this.identifiedControlPeers) {
|
|
7655
|
+
if (this.isControlCohortMember(peerId)) {
|
|
7656
|
+
return true;
|
|
7657
|
+
}
|
|
7658
|
+
}
|
|
7659
|
+
return false;
|
|
7660
|
+
}
|
|
7158
7661
|
/**
|
|
7159
7662
|
* Force a poll of the strand watcher (for testing)
|
|
7160
7663
|
*/
|
|
@@ -7231,7 +7734,8 @@ export class CadreNode {
|
|
|
7231
7734
|
}), this.controlNode, this.controlDatabase);
|
|
7232
7735
|
log('Owner signer wired (owner key %s)', signer.publicKey);
|
|
7233
7736
|
// Embedders wire the owner key after start(), so the runner's first pass usually found
|
|
7234
|
-
// this machine not yet an owner;
|
|
7737
|
+
// this machine not yet an owner; wiring a signer writes no table it watches, so without
|
|
7738
|
+
// this it would wait for its backstop.
|
|
7235
7739
|
this.pendingJoinRunner?.kick();
|
|
7236
7740
|
}
|
|
7237
7741
|
/**
|
|
@@ -7455,6 +7959,10 @@ export class CadreNode {
|
|
|
7455
7959
|
if (usage === undefined || invite === undefined) {
|
|
7456
7960
|
return false;
|
|
7457
7961
|
}
|
|
7962
|
+
// NOTE: accepted tradeoff — three ed25519 verifies per invitation-admitted row on every
|
|
7963
|
+
// refresh, CPU only since the rows come from the membership views; a memo of verdicts would
|
|
7964
|
+
// also have to follow the trusted-owner anchor, which syncOwnerAnchor rewrites. Revisit if
|
|
7965
|
+
// the gate's admission deadline ever shows it.
|
|
7458
7966
|
return verifyInvitationAdmission(this.partyId, row, usage, invite, key => this.trustedOwnerStore?.has(key) ?? false);
|
|
7459
7967
|
}
|
|
7460
7968
|
/**
|
|
@@ -7477,7 +7985,7 @@ export class CadreNode {
|
|
|
7477
7985
|
* builds its snapshot against that anchor, which is also what its previous snapshot was
|
|
7478
7986
|
* judged against. Caught here rather than failing the refresh, because the membership read
|
|
7479
7987
|
* tolerates an unreachable `Revocation` block on an isolated node
|
|
7480
|
-
* (`ControlDatabase.
|
|
7988
|
+
* (`ControlDatabase.readRevocationRowsOrNone`) and `queryRevocations` does not: failing the
|
|
7481
7989
|
* whole refresh on that would freeze such a node's gate snapshot.
|
|
7482
7990
|
* NOTE: on such a node the sync is skipped at every refresh until the block is held; if an
|
|
7483
7991
|
* isolated node ever has to derive owner additions while cut off, give `queryRevocations`
|
|
@@ -7486,9 +7994,12 @@ export class CadreNode {
|
|
|
7486
7994
|
* NOTE: cost per membership refresh is one `OwnerKey` read and one `Revocation` read (the
|
|
7487
7995
|
* tombstone list, whose `OwnerKey` stamps are handed to `queryOwnerKeyRows` so it does not
|
|
7488
7996
|
* read the retired set again, and which the hang-up of removed peers shares through
|
|
7489
|
-
* `loadTombstones`), plus the owner-grant chain when some owner row is
|
|
7490
|
-
*
|
|
7491
|
-
*
|
|
7997
|
+
* `loadTombstones`), plus the owner-grant chain's two reads when some owner row is
|
|
7998
|
+
* grant-seated. All are served from the control database's membership views, so they reach
|
|
7999
|
+
* storage only after their table changed or the view's maximum age passed
|
|
8000
|
+
* (`control-founding-consult-budget.spec.ts` pins the counts). On a party founded before
|
|
8001
|
+
* genesis created every control collection, whose `Revocation` table has never been
|
|
8002
|
+
* written, each `Revocation` load is a cohort consult until the first tombstone.
|
|
7492
8003
|
*
|
|
7493
8004
|
* @param retry - forwarded to every read; `false` on the refresh path, for the reason on
|
|
7494
8005
|
* {@link refreshAuthorizedControlPeers}.
|
|
@@ -7509,9 +8020,6 @@ export class CadreNode {
|
|
|
7509
8020
|
const tombstones = await loadTombstones();
|
|
7510
8021
|
const retired = new Set(tombstones.filter(tombstone => tombstone.tableName === 'OwnerKey').map(tombstone => tombstone.stampId));
|
|
7511
8022
|
const rows = await controlDatabase.queryOwnerKeyRows(retry, retired);
|
|
7512
|
-
// NOTE: the grant chain is two whole-table reads per membership refresh while any owner
|
|
7513
|
-
// row is grant-seated, over tables that are append-only; the memo suggested on
|
|
7514
|
-
// loadInvitationChain would cover it too.
|
|
7515
8023
|
const grantChain = rows.some(isConsentSeated) ? await this.loadOwnerGrantChain(controlDatabase, retry) : null;
|
|
7516
8024
|
const derivation = deriveOwnerAnchor({ partyId: this.partyId, anchor: before, rows, tombstones, grantChain });
|
|
7517
8025
|
if (derivation.refusedRemovals.length > 0) {
|
|
@@ -7569,15 +8077,8 @@ export class CadreNode {
|
|
|
7569
8077
|
/**
|
|
7570
8078
|
* The usage and invitation rows {@link hasAnchoredProof} resolves an invitation-admitted
|
|
7571
8079
|
* `CadrePeer` row through, keyed for lookup. Both reads honour `retry` for
|
|
7572
|
-
* {@link listAuthorizedMembers}' reason.
|
|
7573
|
-
*
|
|
7574
|
-
* NOTE: once invitations are the only admission route (ticket
|
|
7575
|
-
* `cadre-invitations-redeemable-by-any-member`), every cadre with a member not yet
|
|
7576
|
-
* re-vouched pays these two whole-table reads and three ed25519 verifies per row on every
|
|
7577
|
-
* membership refresh. Both tables are append-only and a verified chain never changes (expiry,
|
|
7578
|
-
* use count and withdrawal are not re-checked), so if the gate's admission deadline ever
|
|
7579
|
-
* shows this, memoise verified (row stamp → usage stamp) pairs per node and skip the reads
|
|
7580
|
-
* when every admitted row is already memoised.
|
|
8080
|
+
* {@link listAuthorizedMembers}' reason. The rows come from the membership views, so they
|
|
8081
|
+
* reach storage only after the table changed or the view's maximum age passed.
|
|
7581
8082
|
*/
|
|
7582
8083
|
async loadInvitationChain(controlDatabase, retry) {
|
|
7583
8084
|
const [usages, invites] = await Promise.all([
|
|
@@ -7731,16 +8232,16 @@ export class CadreNode {
|
|
|
7731
8232
|
* write-driven refresh never fires for it — yet applying it can ANCHOR a new
|
|
7732
8233
|
* owner key, which flips rows already present from unauthorized to authorized.
|
|
7733
8234
|
* Hence the explicit refresh here; without it a peer the freshly anchored owner
|
|
7734
|
-
* vouched for stays denied until the next
|
|
8235
|
+
* vouched for stays denied until the next membership change or the backstop pass.
|
|
7735
8236
|
*/
|
|
7736
8237
|
seedEventCallbacks() {
|
|
7737
8238
|
return {
|
|
7738
8239
|
onSeedReceived: (partyId, peerId) => this.emit('seed:received', { partyId, peerId }),
|
|
7739
8240
|
onSeedApplied: (partyId, peersAdded, seed) => {
|
|
7740
|
-
this.emit('seed:applied', { partyId, peersAdded });
|
|
7741
8241
|
// A wire-delivered seed never passes through the `applySeed` wrapper, so
|
|
7742
8242
|
// this is where its preferred peers become cold-start bootstrap targets.
|
|
7743
|
-
this.recordSeedBootstrapPeers(seed);
|
|
8243
|
+
const dialTargets = this.recordSeedBootstrapPeers(seed);
|
|
8244
|
+
this.emit('seed:applied', { partyId, peersAdded, dialTargets });
|
|
7744
8245
|
void this.refreshMembershipGate('seed-applied');
|
|
7745
8246
|
},
|
|
7746
8247
|
onSeedError: (partyId, error) => this.emit('seed:error', { partyId, error }),
|
|
@@ -7764,13 +8265,14 @@ export class CadreNode {
|
|
|
7764
8265
|
* NO `CadrePeer` writer needs to call this: the control database notifies it
|
|
7765
8266
|
* after every committed member-row write (`ControlDatabase.mutateCadrePeer`,
|
|
7766
8267
|
* wired in {@link start}), which is exactly what makes the refresh automatic
|
|
7767
|
-
* rather than a caller obligation.
|
|
7768
|
-
*
|
|
7769
|
-
*
|
|
7770
|
-
*
|
|
7771
|
-
*
|
|
7772
|
-
*
|
|
7773
|
-
*
|
|
8268
|
+
* rather than a caller obligation. A membership row that arrived by REPLICATION
|
|
8269
|
+
* refreshes it too, through the gate watch {@link start} takes on the membership
|
|
8270
|
+
* tables (within its 50 ms window; a change this node's storage never hears of
|
|
8271
|
+
* waits for the reconcile pass's backstop). It stays public for the one change
|
|
8272
|
+
* that writes no row and raises no table event: a newly anchored trusted owner
|
|
8273
|
+
* key ({@link applySeed}), which flips rows ALREADY present from unauthorized to
|
|
8274
|
+
* authorized without touching them. A caller may also await it to observe a
|
|
8275
|
+
* refresh that began after its own change.
|
|
7774
8276
|
*
|
|
7775
8277
|
* Coalescing: marks the snapshot stale and resolves once a refresh that began
|
|
7776
8278
|
* after this call has completed, so a caller that awaits it always observes
|
|
@@ -7795,13 +8297,10 @@ export class CadreNode {
|
|
|
7795
8297
|
* connected peer it newly covers — the strand-side counterpart to
|
|
7796
8298
|
* {@link refreshMembershipGate}.
|
|
7797
8299
|
*
|
|
7798
|
-
*
|
|
7799
|
-
*
|
|
7800
|
-
*
|
|
7801
|
-
*
|
|
7802
|
-
* The gate therefore polls (default 30 s). An app that has just called
|
|
7803
|
-
* `revokeMember` or `leaveStrand` should follow the write with this call so
|
|
7804
|
-
* the cut is immediate rather than up to one poll interval late.
|
|
8300
|
+
* The gate already refreshes on its own when the strand's `Member` or
|
|
8301
|
+
* `MemberPeer` table changes: at once for a `revokeMember` or `leaveStrand`
|
|
8302
|
+
* written on this machine, and when a replicated removal reaches this node's
|
|
8303
|
+
* storage. Call this to await the cut — it resolves after the sweep.
|
|
7805
8304
|
*
|
|
7806
8305
|
* Quiet no-op for a strand that is not running, is quiesced, is open, or has
|
|
7807
8306
|
* the gate disabled. Never rejects; resolves once the sweep has finished.
|
|
@@ -7852,8 +8351,8 @@ export class CadreNode {
|
|
|
7852
8351
|
* CONCURRENT external write's refresh for the life of the scope: that writer's
|
|
7853
8352
|
* promise resolves before its peer is admitted, and admission lands at scope
|
|
7854
8353
|
* exit instead. Acceptable — scope exit is milliseconds away (bounded by the
|
|
7855
|
-
* drain), versus the
|
|
7856
|
-
*
|
|
8354
|
+
* drain), versus the reconcile pass that was the alternative before any of this
|
|
8355
|
+
* existed — but it is the reason these scopes stay short and rare.
|
|
7857
8356
|
*/
|
|
7858
8357
|
async deferMembershipGateRefresh(reason, body) {
|
|
7859
8358
|
this.membershipGateDeferDepth++;
|
|
@@ -7989,7 +8488,7 @@ export class CadreNode {
|
|
|
7989
8488
|
throw new Error('CadreNode must be started before planning an owner key removal');
|
|
7990
8489
|
}
|
|
7991
8490
|
const key = requireEd25519PublicKeyB64(ownerPublicKeyB64, 'owner key');
|
|
7992
|
-
return
|
|
8491
|
+
return await this.planOwnerKeyRemovalWith(this.controlDatabase, key);
|
|
7993
8492
|
}
|
|
7994
8493
|
/**
|
|
7995
8494
|
* Remove an owner key, signed by this node's owner key (which must be a different owner): the
|
|
@@ -8022,7 +8521,7 @@ export class CadreNode {
|
|
|
8022
8521
|
throw new Error('Seed bootstrap service not initialized. Call initializeSeedBootstrap() first.');
|
|
8023
8522
|
}
|
|
8024
8523
|
const key = requireEd25519PublicKeyB64(ownerPublicKeyB64, 'owner key');
|
|
8025
|
-
const
|
|
8524
|
+
const plan = await this.planOwnerKeyRemovalWith(this.controlDatabase, key);
|
|
8026
8525
|
if (plan.refusal !== null) {
|
|
8027
8526
|
throw new OwnerKeyRemovalRefusedError(key, plan.refusal);
|
|
8028
8527
|
}
|
|
@@ -8037,18 +8536,18 @@ export class CadreNode {
|
|
|
8037
8536
|
}
|
|
8038
8537
|
const action = `Remove the owner key ${namedOwnerKey(key, plan.ownerKey?.label ?? null)} from the cadre`;
|
|
8039
8538
|
const removed = await signer.approve({ partyId: this.partyId, action }, async (sign) => {
|
|
8040
|
-
await this.revouchDependents(service, revouch,
|
|
8539
|
+
await this.revouchDependents(service, revouch, sign);
|
|
8041
8540
|
return await service.removeOwner(key, sign);
|
|
8042
8541
|
});
|
|
8043
8542
|
log('removeOwner(%s): removed=%s revouched owners=%d peers=%d', key, removed, revouch.owners.length, revouch.peers.length);
|
|
8044
8543
|
return { removed, revouched: revouch };
|
|
8045
8544
|
}
|
|
8046
|
-
/** {@link planOwnerKeyRemoval}
|
|
8545
|
+
/** {@link planOwnerKeyRemoval} over `controlDatabase`, for a caller that already holds it. */
|
|
8047
8546
|
async planOwnerKeyRemovalWith(controlDatabase, key) {
|
|
8048
8547
|
const [rows, owners, grants] = await Promise.all([
|
|
8049
8548
|
controlDatabase.queryCadrePeers(), controlDatabase.queryOwnerKeyRows(), controlDatabase.listOwnerGrantStatuses()
|
|
8050
8549
|
]);
|
|
8051
|
-
|
|
8550
|
+
return planOwnerKeyRemoval({
|
|
8052
8551
|
key,
|
|
8053
8552
|
rows,
|
|
8054
8553
|
owners,
|
|
@@ -8057,7 +8556,6 @@ export class CadreNode {
|
|
|
8057
8556
|
selfOwnerKey: this.ownerSignerEnrolledIn(new Set(owners.map((owner) => owner.key)))?.publicKey ?? null,
|
|
8058
8557
|
anchor: this.trustedOwnerStore?.all() ?? new Set(),
|
|
8059
8558
|
});
|
|
8060
|
-
return { plan, rows };
|
|
8061
8559
|
}
|
|
8062
8560
|
/**
|
|
8063
8561
|
* The re-vouch step of {@link removeOwner}: every chosen dependent row is rebound to this
|
|
@@ -8066,12 +8564,14 @@ export class CadreNode {
|
|
|
8066
8564
|
* the only rebinding `OwnerKey.NoUpdate` allows); peers through the owner re-touch
|
|
8067
8565
|
* (`reauthorizePeer`), which rewrites `VouchOwner`/`VouchSig` and clears `VouchUsage`.
|
|
8068
8566
|
*
|
|
8069
|
-
* The peer re-touch
|
|
8070
|
-
* `
|
|
8071
|
-
*
|
|
8072
|
-
*
|
|
8073
|
-
*
|
|
8074
|
-
*
|
|
8567
|
+
* The peer re-touch is the re-vouch mode of `reauthorizePeer`, which leaves the row's
|
|
8568
|
+
* `UpdatedAt` untouched and writes whether or not the machine has self-published. The owner
|
|
8569
|
+
* branch of `CadrePeer.AuthorizedUpdate` does not require a bump, and bumping a self-published
|
|
8570
|
+
* row would leave its `Sig` unverifiable, so unresolvable and unlisted as self-published, until
|
|
8571
|
+
* the machine's next heartbeat (the NOTE on that constraint); the write-while-alone drain's
|
|
8572
|
+
* re-broadcast mode bumps because it re-issues an otherwise unchanged row, whereas here the
|
|
8573
|
+
* voucher itself changes. A dependent whose row vanished since the plan is reported absent and
|
|
8574
|
+
* skipped.
|
|
8075
8575
|
*
|
|
8076
8576
|
* A failure rethrows and stops the call before the removal; re-vouches already made stay
|
|
8077
8577
|
* (the row is only re-signed by a current owner) and the caller retries. An owner re-vouch
|
|
@@ -8082,7 +8582,7 @@ export class CadreNode {
|
|
|
8082
8582
|
* rows anyway, so the other machines may get the removal's tombstone without the new voucher
|
|
8083
8583
|
* (`tickets/backlog/bug-owner-key-removal-revouch-made-alone-is-not-reissued`).
|
|
8084
8584
|
*/
|
|
8085
|
-
async revouchDependents(service, revouch,
|
|
8585
|
+
async revouchDependents(service, revouch, sign) {
|
|
8086
8586
|
for (const key of revouch.owners) {
|
|
8087
8587
|
await service.revouchOwner(key, sign);
|
|
8088
8588
|
if (this.committedAlone()) {
|
|
@@ -8091,9 +8591,7 @@ export class CadreNode {
|
|
|
8091
8591
|
}
|
|
8092
8592
|
}
|
|
8093
8593
|
for (const dependent of revouch.peers) {
|
|
8094
|
-
|
|
8095
|
-
const updatedAt = row !== undefined && row.updatedAt > 0 ? row.updatedAt : Date.now();
|
|
8096
|
-
await service.reauthorizePeer(dependent, updatedAt, sign);
|
|
8594
|
+
await service.reauthorizePeer(dependent, 'revouch', sign);
|
|
8097
8595
|
}
|
|
8098
8596
|
}
|
|
8099
8597
|
/**
|
|
@@ -8118,6 +8616,10 @@ export class CadreNode {
|
|
|
8118
8616
|
throw new Error('CadreNode must be started before planning a machine removal');
|
|
8119
8617
|
}
|
|
8120
8618
|
const machines = await this.readMachines(this.controlDatabase);
|
|
8619
|
+
// NOTE: the invitation and usage lists come from the membership views, so on a node that
|
|
8620
|
+
// heard of a row with no storage event (outside the block's cohort) an invitation issued
|
|
8621
|
+
// elsewhere in the last MEMBERSHIP_VIEW_MAX_AGE_MS can be missing and left unwithdrawn. If
|
|
8622
|
+
// removal must close that window, read both tables fresh here.
|
|
8121
8623
|
const [invites, usages, owners] = await Promise.all([
|
|
8122
8624
|
this.controlDatabase.listCadreInviteStatuses(),
|
|
8123
8625
|
this.controlDatabase.queryCadreInviteUsages(),
|
|
@@ -8328,12 +8830,13 @@ export class CadreNode {
|
|
|
8328
8830
|
* that connection from the addresses it was handed, on this launch or a later
|
|
8329
8831
|
* one.
|
|
8330
8832
|
*
|
|
8331
|
-
* Nothing is dialed here: the drone has not received the seed yet.
|
|
8332
|
-
*
|
|
8333
|
-
*
|
|
8334
|
-
*
|
|
8335
|
-
*
|
|
8336
|
-
*
|
|
8833
|
+
* Nothing is dialed here: the drone has not received the seed yet. Its `CadrePeer`
|
|
8834
|
+
* insert is a membership change, so a reconcile pass dials the retained addresses
|
|
8835
|
+
* within about 50 ms of this call returning, whether or not the caller runs one, and
|
|
8836
|
+
* a failed dial is retried on the redial ladder (15 s, 30 s, 60 s, then every
|
|
8837
|
+
* backstop). A caller that wants to await the connection calls
|
|
8838
|
+
* {@link reconcileControlCohort} itself, twice in a row: a call made while a pass is
|
|
8839
|
+
* in flight joins it, and that pass may have listed siblings before this add.
|
|
8337
8840
|
*/
|
|
8338
8841
|
async addDrone(options) {
|
|
8339
8842
|
if (!this.seedBootstrapService) {
|
|
@@ -8378,12 +8881,19 @@ export class CadreNode {
|
|
|
8378
8881
|
* by the node (its anchored branch), the row insert is a no-op on an existing row
|
|
8379
8882
|
* (`ControlDatabase.insertCadrePeer`), and the dial target is replaced, not duplicated.
|
|
8380
8883
|
*
|
|
8381
|
-
*
|
|
8382
|
-
*
|
|
8383
|
-
*
|
|
8384
|
-
*
|
|
8385
|
-
*
|
|
8386
|
-
*
|
|
8884
|
+
* Once it has acknowledged, the node restarts into this party (cadre-cli's
|
|
8885
|
+
* `restartIntoClaimedParty`), closing the connection the claim used. The dial target puts
|
|
8886
|
+
* the address that connection was dialed on first, so every later dial tries the address
|
|
8887
|
+
* that just worked before the others (a QR code lists public addresses before LAN ones, and
|
|
8888
|
+
* a home router often drops a connection to its own public address from inside). A
|
|
8889
|
+
* reconcile pass is kicked (not awaited) now, and the vouch's `CadrePeer` insert starts
|
|
8890
|
+
* another about 50 ms later; for the first {@link CONTROL_COHORT_REDIAL_MS} after the
|
|
8891
|
+
* acknowledgement this node also retries the node every second while it restarts
|
|
8892
|
+
* ({@link redialClaimedNode}). A caller that wants to wait for the connection awaits
|
|
8893
|
+
* {@link reconcileControlCohort} itself, with `addDrone`'s caveat that a pass already in
|
|
8894
|
+
* flight is joined rather than restarted. The claimed node dials this one back too, as the
|
|
8895
|
+
* seed's minter, when this node has a listen or relay address; a phone with neither is
|
|
8896
|
+
* reached by its own dials alone.
|
|
8387
8897
|
*
|
|
8388
8898
|
* @throws {ClaimRefusedError} when the node was reached and refused the claim.
|
|
8389
8899
|
* @throws {PeerUnreachableError} when the node was never reached.
|
|
@@ -8398,6 +8908,7 @@ export class CadreNode {
|
|
|
8398
8908
|
if (!signer) {
|
|
8399
8909
|
throw new Error('Owner key required to create seeds');
|
|
8400
8910
|
}
|
|
8911
|
+
let dialAddrs = target.multiaddrs;
|
|
8401
8912
|
await signer.approve({ partyId: this.partyId, action: `Claim the node ${target.peerId} for the cadre` }, async (sign) => {
|
|
8402
8913
|
const seed = await service.createSeed(sign);
|
|
8403
8914
|
const proof = claimProof(secret, target.peerId, seed.signerKey, seedDigest(seed));
|
|
@@ -8405,13 +8916,87 @@ export class CadreNode {
|
|
|
8405
8916
|
if (!ack.accepted) {
|
|
8406
8917
|
throw new ClaimRefusedError(target.peerId, ack.reason ?? 'no reason given', ack.code);
|
|
8407
8918
|
}
|
|
8919
|
+
// Read before anything else is awaited: the node's restart closes the connection.
|
|
8920
|
+
dialAddrs = this.connectedAddrFirst(target.peerId, target.multiaddrs);
|
|
8408
8921
|
log('claimNode: %s accepted the claim; vouching it into the cadre', target.peerId);
|
|
8409
8922
|
// The wrapper queues the write for re-replication: a phone claiming its first
|
|
8410
8923
|
// always-on node has no control connection yet, so the insert commits local-only.
|
|
8924
|
+
// NOTE: accepted tradeoff — the node's own signed record is not carried in the
|
|
8925
|
+
// acknowledgement and written with this vouch: it would be signed before the node
|
|
8926
|
+
// restarts, so its addresses may not be the restarted node's (a new relay reservation),
|
|
8927
|
+
// and the write-while-alone drain re-sends only rows whose `Sig` is null
|
|
8928
|
+
// (drainPendingPeerWrites, reconstructAuthoredMembership), so a vouch carrying it would
|
|
8929
|
+
// never reach this cadre's other members. The node publishes its record once it has
|
|
8930
|
+
// restarted. Revisit if the drain ever re-sends signed rows.
|
|
8411
8931
|
await this.vouchPeer(target.peerId, target.multiaddrs, sign);
|
|
8412
8932
|
});
|
|
8413
|
-
this.retainDialTarget(target.peerId,
|
|
8933
|
+
this.retainDialTarget(target.peerId, dialAddrs, 'claimNode');
|
|
8414
8934
|
void this.reconcileControlCohort().catch((error) => log('claimNode: reconcile kick after claiming %s failed: %o', target.peerId, error));
|
|
8935
|
+
void this.redialClaimedNode(target.peerId).catch((error) => log('claimNode: redialing %s while it restarts failed: %o', target.peerId, error));
|
|
8936
|
+
}
|
|
8937
|
+
/**
|
|
8938
|
+
* `addrs` with the address this node's open connection to `peerId` was dialed on moved to
|
|
8939
|
+
* the front, when it is one of them; `addrs` unchanged otherwise. Compared without a
|
|
8940
|
+
* trailing `/p2p/<peerId>`, which a QR code's address may or may not carry ({@link addrKey}).
|
|
8941
|
+
*/
|
|
8942
|
+
connectedAddrFirst(peerId, addrs) {
|
|
8943
|
+
const controlNode = this.controlNode;
|
|
8944
|
+
if (!controlNode) {
|
|
8945
|
+
return addrs;
|
|
8946
|
+
}
|
|
8947
|
+
const remote = peerIdFromString(peerId);
|
|
8948
|
+
const connected = new Set(controlNode.getConnections(remote).map((c) => addrKey(c.remoteAddr.toString(), remote)));
|
|
8949
|
+
const index = addrs.findIndex((addr) => connected.has(addrKey(addr, remote)));
|
|
8950
|
+
return index <= 0 ? addrs : [addrs[index], ...addrs.slice(0, index), ...addrs.slice(index + 1)];
|
|
8951
|
+
}
|
|
8952
|
+
/**
|
|
8953
|
+
* Retry a node this node has just claimed ({@link claimNode}) once every
|
|
8954
|
+
* {@link CLAIM_RESTART_REDIAL_STEP_MS} for {@link CONTROL_COHORT_REDIAL_MS} after it accepted.
|
|
8955
|
+
* The node restarts into this party right after it acknowledges, and while it does, a
|
|
8956
|
+
* reconcile pass's dial either fails or opens a connection the restarting node closes at
|
|
8957
|
+
* once. The first failure puts the pass's next redial 15 s out ({@link armReconcileWake}),
|
|
8958
|
+
* and an owner that cannot be dialed has no other way back to the node.
|
|
8959
|
+
*
|
|
8960
|
+
* Each step dials once from the retained dial target, unless a connection to the node is
|
|
8961
|
+
* open: the one the claim used until the restart closes it, or one to the restarted node
|
|
8962
|
+
* that has not identified yet. A dial that connects does not end the loop, because the
|
|
8963
|
+
* restarting node may still close it. The loop ends once the node has identified as serving
|
|
8964
|
+
* this party's control database ({@link identifiedControlPeers}, which the placeholder node
|
|
8965
|
+
* it ran before the claim never joins), at the window's end, when this node stops, or when
|
|
8966
|
+
* the target is forgotten ({@link removePeer}). Steps run one after another, so a dial
|
|
8967
|
+
* finishes on its own budget first. The redial streak is left alone: the ladder carries on
|
|
8968
|
+
* after the window. One loop per peer: a repeated claim while one runs starts none.
|
|
8969
|
+
*/
|
|
8970
|
+
async redialClaimedNode(peerId) {
|
|
8971
|
+
if (this.claimRedialPeers.has(peerId)) {
|
|
8972
|
+
return;
|
|
8973
|
+
}
|
|
8974
|
+
this.claimRedialPeers.add(peerId);
|
|
8975
|
+
try {
|
|
8976
|
+
const endsAt = Date.now() + CONTROL_COHORT_REDIAL_MS;
|
|
8977
|
+
while (Date.now() < endsAt && await this.redialClaimedNodeStep(peerId)) {
|
|
8978
|
+
await new Promise((resolve) => {
|
|
8979
|
+
const timer = setTimeout(resolve, CLAIM_RESTART_REDIAL_STEP_MS);
|
|
8980
|
+
// Node-only: a stop mid-window must not keep the event loop alive for the step.
|
|
8981
|
+
timer.unref?.();
|
|
8982
|
+
});
|
|
8983
|
+
}
|
|
8984
|
+
}
|
|
8985
|
+
finally {
|
|
8986
|
+
this.claimRedialPeers.delete(peerId);
|
|
8987
|
+
}
|
|
8988
|
+
}
|
|
8989
|
+
/** One step of {@link redialClaimedNode}; false ends the loop. */
|
|
8990
|
+
async redialClaimedNodeStep(peerId) {
|
|
8991
|
+
const controlNode = this.controlNode;
|
|
8992
|
+
const target = this.bootstrapPeerStore?.all().get(peerId);
|
|
8993
|
+
if (!this._running || !controlNode || !target || this.identifiedControlPeers.has(peerId)) {
|
|
8994
|
+
return false;
|
|
8995
|
+
}
|
|
8996
|
+
if (controlNode.getConnections(peerIdFromString(peerId)).length === 0) {
|
|
8997
|
+
await this.dialBootstrapPeer(peerId, target.addrs, 'claimNode(redial)');
|
|
8998
|
+
}
|
|
8999
|
+
return true;
|
|
8415
9000
|
}
|
|
8416
9001
|
/**
|
|
8417
9002
|
* Add a phone to the cadre with relay support.
|
|
@@ -8569,11 +9154,13 @@ export class CadreNode {
|
|
|
8569
9154
|
* Steps: refuse an invitation for another party; pin its `ownerKeys` into the node-local
|
|
8570
9155
|
* anchor (`'invite'` provenance — the pin sticks even if the redemption then fails, as it
|
|
8571
9156
|
* does for a seed invite, so a later attempt or seed from the same owner is anchored);
|
|
8572
|
-
* sign the request
|
|
8573
|
-
* acceptance verify the reply against the pinned
|
|
8574
|
-
* store, retain the answering member as a
|
|
8575
|
-
*
|
|
8576
|
-
* the connection it already holds, the same unified behaviour as after a seed.
|
|
9157
|
+
* sign the request, which carries this node's own signed address record; dial the listed
|
|
9158
|
+
* members in order (`redeemAtMembers`); on acceptance verify the reply against the pinned
|
|
9159
|
+
* keys, merge its dial hints into the peer store, retain the answering member as a
|
|
9160
|
+
* cold-start dial target and start a cohort reconcile pass. This node then syncs the control
|
|
9161
|
+
* database over the connection it already holds, the same unified behaviour as after a seed.
|
|
9162
|
+
* The member writes the record on this node's `CadrePeer` row when its signature verifies, so
|
|
9163
|
+
* the row resolves to this node's addresses from its admission.
|
|
8577
9164
|
*
|
|
8578
9165
|
* A device that is already a member (a restart with the invitation still configured) is
|
|
8579
9166
|
* answered as accepted by the member's idempotent redemption, with no second usage row.
|
|
@@ -8617,7 +9204,15 @@ export class CadreNode {
|
|
|
8617
9204
|
}
|
|
8618
9205
|
await this.trustOwnerKeys(invitation.ownerKeys, 'invite');
|
|
8619
9206
|
const pinned = new Set(invitation.ownerKeys);
|
|
8620
|
-
const
|
|
9207
|
+
const selfPeerId = controlNode.peerId.toString();
|
|
9208
|
+
// The addresses a publish would carry, sanitized before signing: the member stores the
|
|
9209
|
+
// signature only over the exact list it stores.
|
|
9210
|
+
const record = this.signSelfRecord(selfPeerId, signer, sanitizeAddrs(await this.collectSelfAddrs()), null);
|
|
9211
|
+
const request = signRedeemRequest(invitation, { peerKey: signer.publicKeyB64, peerPrivateKey: signer.privateKeyB64 }, generateStampId(selfPeerId), record);
|
|
9212
|
+
// Before dialing, not on acceptance: the member's commit can reach this node before its
|
|
9213
|
+
// reply does, and a reconcile pass that reads the row first must find it current. A record
|
|
9214
|
+
// no member wrote matches no row, so a failed redemption leaves nothing to undo.
|
|
9215
|
+
this.lastSignedSelfRecord = record;
|
|
8621
9216
|
const { reply, memberPeerId, memberAddr } = await redeemInPasses(controlNode, {
|
|
8622
9217
|
invitation,
|
|
8623
9218
|
request,
|
|
@@ -8628,7 +9223,6 @@ export class CadreNode {
|
|
|
8628
9223
|
...(options.onProgress ? { onProgress: options.onProgress } : {}),
|
|
8629
9224
|
});
|
|
8630
9225
|
log('Cadre invitation %s redeemed at %s', invitation.invite.key, memberAddr);
|
|
8631
|
-
const selfPeerId = controlNode.peerId.toString();
|
|
8632
9226
|
await mergeSeedPeers(controlNode, reply.peers.filter((peer) => peer.peerId !== selfPeerId));
|
|
8633
9227
|
if (memberPeerId !== null) {
|
|
8634
9228
|
// Every address the bundle gave the answering member, so a restart has the same choices.
|
|
@@ -8637,9 +9231,8 @@ export class CadreNode {
|
|
|
8637
9231
|
.map((addr) => addr.toString());
|
|
8638
9232
|
this.retainDialTarget(memberPeerId, memberAddrs, 'redeemCadreInvitation');
|
|
8639
9233
|
}
|
|
8640
|
-
//
|
|
8641
|
-
//
|
|
8642
|
-
await this.refreshMembershipGate('invitation-redeemed');
|
|
9234
|
+
// No gate refresh here: the rows about to replicate in are judged against the pins above
|
|
9235
|
+
// when they land, by the gate watch's refresh and the head refresh of this pass.
|
|
8643
9236
|
void this.reconcileControlCohort().catch((error) => log('redeemCadreInvitation: the post-redemption reconcile pass failed: %o', error));
|
|
8644
9237
|
return { peerId: memberPeerId, redeemedAt: new Date().toISOString() };
|
|
8645
9238
|
}
|
|
@@ -8811,25 +9404,31 @@ export class CadreNode {
|
|
|
8811
9404
|
return { ownerKey: signer.publicKey, alreadyOwner: reply.alreadyOwner, memberPeerId };
|
|
8812
9405
|
}
|
|
8813
9406
|
/**
|
|
8814
|
-
* {@link redeemOwnerGrant}'s local path: the alone refusal, the seat
|
|
8815
|
-
* then one approval around the consent signature and the
|
|
8816
|
-
* the approval, so a prompting signer never asks for a
|
|
9407
|
+
* {@link redeemOwnerGrant}'s local path: the alone refusal, the seat (only when this node holds
|
|
9408
|
+
* no grant row), the liveness refusal, then one approval around the consent signature and the
|
|
9409
|
+
* write. Every refusal comes before the approval, so a prompting signer never asks for a
|
|
9410
|
+
* redemption this node would refuse. The held row is read once and carried into the liveness
|
|
9411
|
+
* check and the write, as the member handler carries it (`MemberRedemptionHandler`).
|
|
8817
9412
|
*/
|
|
8818
9413
|
async redeemOwnerGrantLocally(controlDatabase, bundle, signer) {
|
|
8819
9414
|
if (await this.ownerKeyWriteAlone()) {
|
|
8820
9415
|
throw new OwnerKeyAddRefusedError(signer.publicKey, 'alone');
|
|
8821
9416
|
}
|
|
8822
|
-
|
|
9417
|
+
const held = await controlDatabase.queryOwnerGrant(bundle.grant.key);
|
|
9418
|
+
if (held === null) {
|
|
9419
|
+
await refusingUnsyncedGrant(() => controlDatabase.seatOwnerGrant(bundle.grant));
|
|
9420
|
+
}
|
|
8823
9421
|
// The write answers a key that is already an owner without reading the grant, so only a
|
|
8824
9422
|
// key that is not one needs the grant live (the same permissive pre-check the handler's
|
|
8825
9423
|
// admission uses; the write enforces every condition).
|
|
8826
|
-
if (!(await controlDatabase.getOwnerKeys()).has(signer.publicKey) && !await controlDatabase.isOwnerGrantLive(bundle.grant)) {
|
|
9424
|
+
if (!(await controlDatabase.getOwnerKeys()).has(signer.publicKey) && !await controlDatabase.isOwnerGrantLive(bundle.grant, held)) {
|
|
8827
9425
|
throw new OwnerGrantRejectedError('grant-spent', 'The grant is expired, withdrawn, exhausted, or its issuer is no longer an owner');
|
|
8828
9426
|
}
|
|
8829
9427
|
const { alreadyOwner } = await signer.approve(ownerGrantApproval(bundle), async (sign) => {
|
|
8830
9428
|
const request = await this.signOwnerGrantRequest(bundle, signer, sign);
|
|
8831
9429
|
return await refusingUnsyncedGrant(() => controlDatabase.redeemOwnerGrant({
|
|
8832
9430
|
grantKey: bundle.grant.key,
|
|
9431
|
+
...(held ? { held } : {}),
|
|
8833
9432
|
seatedKey: signer.publicKey,
|
|
8834
9433
|
usageStampId: request.usageStampId,
|
|
8835
9434
|
grantSig: request.grantSig,
|
|
@@ -9015,7 +9614,7 @@ export class CadreNode {
|
|
|
9015
9614
|
// Overridden on the same grounds: only this node holds the running strand
|
|
9016
9615
|
// instance's database and this party's `StrandPartyKey` identity, and a wrong
|
|
9017
9616
|
// issuer here would admit joiners under someone else's authority.
|
|
9018
|
-
issueMembershipInvite: (
|
|
9617
|
+
issueMembershipInvite: (host, signal) => this.issueStrandMembershipInvite(host, signal)
|
|
9019
9618
|
});
|
|
9020
9619
|
const previous = this.strandSolicitationService;
|
|
9021
9620
|
if (previous) {
|
|
@@ -9280,10 +9879,17 @@ export class CadreNode {
|
|
|
9280
9879
|
selfId: this.controlNode.peerId.toString(),
|
|
9281
9880
|
linkRoundTripMs: this.config.network?.linkRoundTripMs,
|
|
9282
9881
|
isOwner: () => this.isOwner(),
|
|
9283
|
-
// NOTE: one owner-key read and one pending-join read (three tables) per pass (30 s); for a
|
|
9284
|
-
// party that never asked for a join the second reads never-written blocks, which consults
|
|
9285
|
-
// the cohort. If it shows up in a device profile, poll more slowly while the tables read empty.
|
|
9286
9882
|
readRows: () => database().queryPendingJoins(),
|
|
9883
|
+
// What `isOwner` and `queryPendingJoins` read: the owner keys, the joins and their
|
|
9884
|
+
// outcomes, and the tombstones both filter by. A dismissal writes its tombstone with the
|
|
9885
|
+
// request's delete, but the two can replicate separately.
|
|
9886
|
+
watchChanges: (onChange, options) => database().watchTables([
|
|
9887
|
+
'CadreControl.JoinRequest',
|
|
9888
|
+
'CadreControl.JoinSuccess',
|
|
9889
|
+
'CadreControl.JoinFailure',
|
|
9890
|
+
'CadreControl.OwnerKey',
|
|
9891
|
+
'CadreControl.Revocation',
|
|
9892
|
+
], onChange, options),
|
|
9287
9893
|
readRow: (id) => database().queryPendingJoin(id),
|
|
9288
9894
|
attempt: (join) => this.formStrand(this.pendingJoinInvitation(join), parseStoredDisclosure(join)),
|
|
9289
9895
|
record: (expected, outcome) => ownerWrite(`Record the outcome of the pending join ${expected.Id}`, (ownerKey, sign) => database().recordJoinOutcome(expected, outcome, ownerKey, sign)),
|
|
@@ -9344,6 +9950,13 @@ export class CadreNode {
|
|
|
9344
9950
|
* token is spent, and a join no store names would vanish at the next restart.
|
|
9345
9951
|
*/
|
|
9346
9952
|
async rememberFormedStrand(result) {
|
|
9953
|
+
// NOTE: accepted tradeoff — the party-wide `JoinedStrand` row is not written here, in
|
|
9954
|
+
// formStrand's own approval, but later by an owner machine's connected reconcile pass
|
|
9955
|
+
// (`JoinedStrandSession.syncWithParty`). That publish is gated on a control connection so a
|
|
9956
|
+
// machine cut off from its own party never commits it alone, and during formStrand the
|
|
9957
|
+
// count always includes the connection to the other party's responder, so the gate would
|
|
9958
|
+
// always pass here. Non-owner and signer-less machines need this local path regardless.
|
|
9959
|
+
// Revisit if the gate can tell a connection to this party's own machines from any other.
|
|
9347
9960
|
try {
|
|
9348
9961
|
await this.joinedStrands.remember({
|
|
9349
9962
|
Id: result.strandId,
|
|
@@ -9383,7 +9996,7 @@ export class CadreNode {
|
|
|
9383
9996
|
*/
|
|
9384
9997
|
async adoptFormationMembershipInvite(strandId, invite) {
|
|
9385
9998
|
try {
|
|
9386
|
-
await this.ensureStrandPartyKey(strandId);
|
|
9999
|
+
this.formedPartyKeys.set(strandId, await this.ensureStrandPartyKey(strandId));
|
|
9387
10000
|
}
|
|
9388
10001
|
catch (error) {
|
|
9389
10002
|
throw new FormationPostApprovalError(strandId, `Formation for strand ${strandId} was approved (its one-time token is spent), but ` +
|
|
@@ -9428,6 +10041,10 @@ export class CadreNode {
|
|
|
9428
10041
|
* implements): mint a single-use `Strand.Invite` against the LIVE host strand so a
|
|
9429
10042
|
* validated joiner can seat its own `Strand.Member` row.
|
|
9430
10043
|
*
|
|
10044
|
+
* `host` is the strand as the formation session resolved it from its `Strand` row, read
|
|
10045
|
+
* once there; this method reads no `Strand` row of its own. A row removed in between is
|
|
10046
|
+
* refused when the consent row is written (the `StrandExists` CHECK).
|
|
10047
|
+
*
|
|
9431
10048
|
* - Open host strand → `null` (no members, nothing to invite into).
|
|
9432
10049
|
* - Closed host strand whose founder launch was refused as pre-split
|
|
9433
10050
|
* ({@link strandLaunchRefusals}) → rethrow the recorded `PreSplitStrandIdentityError`;
|
|
@@ -9475,17 +10092,12 @@ export class CadreNode {
|
|
|
9475
10092
|
* `MEMBERSHIP_INVITE_UNAVAILABLE_REASON` forever; that flow needs manager delegation,
|
|
9476
10093
|
* not a retry.
|
|
9477
10094
|
*/
|
|
9478
|
-
async issueStrandMembershipInvite(
|
|
10095
|
+
async issueStrandMembershipInvite(host, signal) {
|
|
10096
|
+
const { strandId } = host;
|
|
9479
10097
|
if (!this.controlDatabase) {
|
|
9480
10098
|
throw new Error(`Cannot issue a membership invitation for strand ${strandId}: control database unavailable`);
|
|
9481
10099
|
}
|
|
9482
|
-
|
|
9483
|
-
if (!row) {
|
|
9484
|
-
// The bound resolution saw this row moments ago; a vanished row is a concurrent
|
|
9485
|
-
// unpublish — reject rather than invite into a strand this party just removed.
|
|
9486
|
-
throw new Error(`Cannot issue a membership invitation for strand ${strandId}: its Strand row is gone`);
|
|
9487
|
-
}
|
|
9488
|
-
if (row.Type !== 'c') {
|
|
10100
|
+
if (host.strandType !== 'c') {
|
|
9489
10101
|
return null;
|
|
9490
10102
|
}
|
|
9491
10103
|
const refusal = this.strandLaunchRefusals.get(strandId);
|
|
@@ -9503,8 +10115,8 @@ export class CadreNode {
|
|
|
9503
10115
|
throw new Error(`Cannot issue a membership invitation for closed strand ${strandId}: its runtime is ` +
|
|
9504
10116
|
'not live on this responder (not launched, or still starting)');
|
|
9505
10117
|
}
|
|
9506
|
-
if (
|
|
9507
|
-
await assertNotPreSplitStrand(db, strandId, strandMemberKeyPair(
|
|
10118
|
+
if (host.memberPrivateKey) {
|
|
10119
|
+
await assertNotPreSplitStrand(db, strandId, strandMemberKeyPair(host.memberPrivateKey).publicKeyB64);
|
|
9508
10120
|
}
|
|
9509
10121
|
if (signal?.aborted) {
|
|
9510
10122
|
throw new Error(`Cannot issue a membership invitation for closed strand ${strandId}: the formation ` +
|