@serfab/cadre-core 1.17.0 → 1.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/dist/cadre-invite-protocol.d.ts +35 -13
- package/dist/cadre-invite-protocol.js +90 -25
- package/dist/cadre-invite-protocol.js.map +1 -1
- package/dist/cadre-node.d.ts +586 -255
- package/dist/cadre-node.js +1182 -566
- package/dist/cadre-node.js.map +1 -1
- package/dist/control-cohort.d.ts +66 -9
- package/dist/control-cohort.js +67 -9
- package/dist/control-cohort.js.map +1 -1
- package/dist/control-database.d.ts +451 -193
- package/dist/control-database.js +701 -359
- package/dist/control-database.js.map +1 -1
- package/dist/control-formation-recorder.d.ts +46 -43
- package/dist/control-formation-recorder.js +77 -78
- package/dist/control-formation-recorder.js.map +1 -1
- package/dist/control-protocol-guard.d.ts +6 -0
- package/dist/control-protocol-guard.js +14 -2
- package/dist/control-protocol-guard.js.map +1 -1
- package/dist/control-schema-version.d.ts +48 -31
- package/dist/control-schema-version.js +67 -45
- package/dist/control-schema-version.js.map +1 -1
- package/dist/control-schema.d.ts +8 -8
- package/dist/control-schema.js +16 -16
- package/dist/control-schema.js.map +1 -1
- package/dist/control-write-retry.d.ts +42 -7
- package/dist/control-write-retry.js +87 -19
- package/dist/control-write-retry.js.map +1 -1
- package/dist/enrolled-machine-store.d.ts +3 -3
- package/dist/enrolled-machine-store.js +3 -3
- package/dist/formation-token.d.ts +48 -0
- package/dist/formation-token.js +26 -0
- package/dist/formation-token.js.map +1 -0
- package/dist/index.d.ts +15 -14
- package/dist/index.js +17 -14
- package/dist/index.js.map +1 -1
- package/dist/joined-strand-store.d.ts +26 -9
- package/dist/joined-strand-store.js +63 -24
- package/dist/joined-strand-store.js.map +1 -1
- package/dist/link-budget.d.ts +9 -7
- package/dist/link-budget.js +9 -7
- package/dist/link-budget.js.map +1 -1
- package/dist/member-redemption.d.ts +15 -9
- package/dist/member-redemption.js +13 -8
- package/dist/member-redemption.js.map +1 -1
- package/dist/membership-view.d.ts +91 -0
- package/dist/membership-view.js +110 -0
- package/dist/membership-view.js.map +1 -0
- package/dist/owner-grant-protocol.d.ts +11 -7
- package/dist/owner-grant-protocol.js +31 -17
- package/dist/owner-grant-protocol.js.map +1 -1
- package/dist/peer-addr-book.d.ts +10 -1
- package/dist/peer-addr-book.js +4 -3
- package/dist/peer-addr-book.js.map +1 -1
- package/dist/peer-join-backfill.d.ts +96 -30
- package/dist/peer-join-backfill.js +272 -128
- package/dist/peer-join-backfill.js.map +1 -1
- package/dist/pending-join-runner.d.ts +48 -18
- package/dist/pending-join-runner.js +74 -56
- package/dist/pending-join-runner.js.map +1 -1
- package/dist/seed-bootstrap.d.ts +47 -40
- package/dist/seed-bootstrap.js +66 -61
- package/dist/seed-bootstrap.js.map +1 -1
- package/dist/self-address-update.d.ts +13 -0
- package/dist/self-address-update.js +19 -0
- package/dist/self-address-update.js.map +1 -0
- package/dist/strand-database.d.ts +23 -2
- package/dist/strand-database.js +26 -6
- package/dist/strand-database.js.map +1 -1
- package/dist/strand-first-sync-gate.d.ts +176 -89
- package/dist/strand-first-sync-gate.js +288 -122
- package/dist/strand-first-sync-gate.js.map +1 -1
- package/dist/strand-formation-deadlines.d.ts +9 -9
- package/dist/strand-formation-deadlines.js +9 -9
- package/dist/strand-formation-manager.d.ts +23 -17
- package/dist/strand-formation-manager.js +46 -44
- package/dist/strand-formation-manager.js.map +1 -1
- package/dist/strand-formation-protocol.d.ts +5 -8
- package/dist/strand-formation-protocol.js +101 -16
- package/dist/strand-formation-protocol.js.map +1 -1
- package/dist/strand-instance-manager.d.ts +26 -17
- package/dist/strand-instance-manager.js +119 -50
- package/dist/strand-instance-manager.js.map +1 -1
- package/dist/strand-membership-reconciler.d.ts +16 -13
- package/dist/strand-membership-reconciler.js +28 -22
- package/dist/strand-membership-reconciler.js.map +1 -1
- package/dist/strand-membership-writer.d.ts +40 -28
- package/dist/strand-membership-writer.js +212 -153
- package/dist/strand-membership-writer.js.map +1 -1
- package/dist/strand-move.js +5 -2
- package/dist/strand-move.js.map +1 -1
- package/dist/strand-revocation-enforcer.d.ts +134 -60
- package/dist/strand-revocation-enforcer.js +144 -71
- package/dist/strand-revocation-enforcer.js.map +1 -1
- package/dist/strand-solicitation.d.ts +50 -39
- package/dist/strand-solicitation.js +10 -3
- package/dist/strand-solicitation.js.map +1 -1
- package/dist/strand-watcher.d.ts +80 -13
- package/dist/strand-watcher.js +152 -48
- package/dist/strand-watcher.js.map +1 -1
- package/dist/table-change-watch.d.ts +63 -0
- package/dist/table-change-watch.js +209 -0
- package/dist/table-change-watch.js.map +1 -0
- package/dist/timeout-scheduler.d.ts +4 -3
- package/dist/timeout-scheduler.js +4 -3
- package/dist/timeout-scheduler.js.map +1 -1
- package/dist/types.d.ts +90 -49
- package/dist/types.js +4 -4
- package/dist/types.js.map +1 -1
- package/package.json +8 -8
- package/src/cadre-invite-protocol.ts +119 -31
- package/src/cadre-node.ts +1289 -588
- package/src/control-cohort.ts +86 -9
- package/src/control-database.ts +848 -403
- package/src/control-formation-recorder.ts +359 -353
- package/src/control-protocol-guard.ts +16 -2
- package/src/control-schema-version.ts +101 -51
- package/src/control-schema.ts +16 -16
- package/src/control-write-retry.ts +91 -19
- package/src/delegate-admission.ts +351 -351
- package/src/enrolled-machine-store.ts +3 -3
- package/src/formation-token.ts +63 -0
- package/src/index.ts +30 -18
- package/src/joined-strand-store.ts +69 -24
- package/src/link-budget.ts +9 -7
- package/src/member-redemption.ts +20 -13
- package/src/membership-view.ts +144 -0
- package/src/owner-grant-protocol.ts +33 -18
- package/src/peer-addr-book.ts +4 -3
- package/src/peer-join-backfill.ts +961 -761
- package/src/pending-join-runner.ts +76 -53
- package/src/seed-bootstrap.ts +72 -67
- package/src/self-address-update.ts +19 -0
- package/src/strand-database.ts +352 -322
- package/src/strand-first-sync-gate.ts +334 -134
- package/src/strand-formation-deadlines.ts +227 -227
- package/src/strand-formation-manager.ts +731 -725
- package/src/strand-formation-protocol.ts +122 -24
- package/src/strand-instance-manager.ts +2092 -2026
- package/src/strand-membership-reconciler.ts +31 -25
- package/src/strand-membership-writer.ts +2093 -2019
- package/src/strand-move.ts +5 -2
- package/src/strand-revocation-enforcer.ts +740 -654
- package/src/strand-solicitation.ts +61 -34
- package/src/strand-watcher.ts +175 -56
- package/src/table-change-watch.ts +285 -0
- package/src/timeout-scheduler.ts +29 -28
- package/src/types.ts +89 -51
package/dist/cadre-node.js
CHANGED
|
@@ -21,7 +21,7 @@ import { MemoryBootstrapPeerStore } from './bootstrap-peer-store.js';
|
|
|
21
21
|
import { MemoryStrandNetworkStateStore } from './strand-network-state.js';
|
|
22
22
|
import { MemoryEnrolledMachineStore } from './enrolled-machine-store.js';
|
|
23
23
|
import { JoinedStrandSession, KeyStoreJoinedStrandStore, MemoryJoinedStrandStore } from './joined-strand-store.js';
|
|
24
|
-
import { mergePeerAddrs, groupAddrsByPeerId } from './peer-addr-book.js';
|
|
24
|
+
import { addrKey, mergePeerAddrs, groupAddrsByPeerId } from './peer-addr-book.js';
|
|
25
25
|
import { strandFretPeerAddrs } from './strand-fret-addrs.js';
|
|
26
26
|
import { verifyCadrePeerVoucher, verifyInvitationAdmission, verifyRevocationSigner } from './peer-authorization.js';
|
|
27
27
|
import { deriveOwnerAnchor } from './owner-anchor-sync.js';
|
|
@@ -30,32 +30,35 @@ import { describeOwnerKeys, namedOwnerKey, planOwnerKeyRemoval, revouchWithinPla
|
|
|
30
30
|
import { ed25519PublicKeyB64FromPeerId } from './ed25519-key.js';
|
|
31
31
|
import { signPeerRecord, verifyPeerRecordSignature, isPeerRecordFresh, orderSignalingFirst, isSignalingAddr, currentMemberTrustPolicy, DEFAULT_PEER_RECORD_MAX_AGE_MS, DEFAULT_PEER_RECORD_HEARTBEAT_MS } from './peer-record.js';
|
|
32
32
|
import { signDeviceTokenRecord, verifyDeviceTokenSignature, isPushPlatform } from './device-token.js';
|
|
33
|
-
import { StrandWatcher } from './strand-watcher.js';
|
|
33
|
+
import { StrandWatcher, DEFAULT_STRAND_WATCH_BACKSTOP_MS } from './strand-watcher.js';
|
|
34
34
|
import { StrandInstanceManager, liveStrandStatus } from './strand-instance-manager.js';
|
|
35
35
|
import { PeerJoinBackfill } from './peer-join-backfill.js';
|
|
36
36
|
import { deriveCohortMembers } from './strand-cohort.js';
|
|
37
|
-
import { selectControlCohortDials, DEFAULT_CONTROL_COHORT_RECONCILE_MS, DEFAULT_CONTROL_COHORT_TARGET_DEGREE } from './control-cohort.js';
|
|
37
|
+
import { selectControlCohortDials, nextReconcileWakeMs, CLAIM_RESTART_REDIAL_STEP_MS, CONTROL_COHORT_REDIAL_MS, DEFAULT_CONTROL_COHORT_RECONCILE_MS, DEFAULT_CONTROL_COHORT_TARGET_DEGREE } from './control-cohort.js';
|
|
38
|
+
import { scheduleRuns } from './table-change-watch.js';
|
|
39
|
+
import { MEMBERSHIP_VIEW_RECONNECT_RELOAD_MS } from './membership-view.js';
|
|
38
40
|
import { dialPeerAddrs, SelfRelayOnlyError, CONTROL_COHORT_DIAL_ADDRESS_ATTEMPTS, trailingPeerId, withTrailingPeerId } from './peer-dial.js';
|
|
39
41
|
import { ADMISSION_DECISION_TIMEOUT_MS, declaredCohortReadDeadlineMs, optimysticDialLimits, peerJoinPushBudget, relayReservationBudgetMs, relayedDialBudgetMs, relayedRequestBudgetMs, resolveLinkRoundTripMs } from './link-budget.js';
|
|
40
42
|
import { EnrollmentService } from './enrollment.js';
|
|
41
43
|
import { HibernationManager } from './hibernation-manager.js';
|
|
42
44
|
import { ControlDatabase, generateStampId, isPendingJoinConflict, isStrandIdConflict, IssuedRowNotHereYetError, pendingJoinId } from './control-database.js';
|
|
43
|
-
import { CadreInviteHandler, encodeCadreInvitation, redeemInPasses, signRedeemRequest } from './cadre-invite-protocol.js';
|
|
45
|
+
import { CadreInviteHandler, encodeCadreInvitation, redeemInPasses, sanitizeAddrs, signRedeemRequest } from './cadre-invite-protocol.js';
|
|
44
46
|
import { OwnerGrantHandler, OwnerGrantRejectedError, encodeOwnerGrant, redeemOwnerGrantInPasses, signOwnerGrantRequest } from './owner-grant-protocol.js';
|
|
45
47
|
import { FormationPostApprovalError } from './strand-formation-rejection.js';
|
|
46
48
|
import { PendingJoinRunner, membershipInvitesToStage, parseStoredDisclosure, requestedPendingJoin } from './pending-join-runner.js';
|
|
47
|
-
import { SeedBootstrapService, mergeSeedPeers, seedDigest } from './seed-bootstrap.js';
|
|
49
|
+
import { SeedBootstrapService, mergeSeedPeers, seedDialTargets, seedDigest } from './seed-bootstrap.js';
|
|
48
50
|
import { claimSecretTrustPolicy } from './seed-trust-policy.js';
|
|
49
51
|
import { ClaimRefusedError, claimProof, parseClaimSecret } from './claim-proof.js';
|
|
50
52
|
import { StrandSolicitationService } from './strand-solicitation.js';
|
|
51
53
|
import { ControlFormationUsageRecorder } from './control-formation-recorder.js';
|
|
52
54
|
import { cadreInvitationReach, selectCadreInvitationMembers, selectInvitationSiblingAddrs } from './invitation-bootstrap.js';
|
|
53
55
|
import { createMembershipConnectionGater, UnauthorizedReservationBudget } from './membership-connection-gater.js';
|
|
54
|
-
import { controlNetworkName, installControlProtocolGuard } from './control-protocol-guard.js';
|
|
56
|
+
import { controlNetworkName, installControlProtocolGuard, servesControlDatabase } from './control-protocol-guard.js';
|
|
55
57
|
import { StrandWakeService, dialWake } from './strand-wake-protocol.js';
|
|
56
58
|
import { StrandAddrService, collectStrandAddrs } from './strand-addr-protocol.js';
|
|
57
59
|
import { DelegateAdmissionStore, extractCircuitRelayTargets, dueRelayAnnounces, prunePeerStrandKeys, peerStrandKey, recordRelayRefusals, relaysNotRefusing } from './delegate-admission.js';
|
|
58
60
|
import { relayCircuitAddrs, resolveListenAddrs, resolveTransportOptions, RelayReservationFailedError } from './relay-addrs.js';
|
|
61
|
+
import { selfAddressesChanged } from './self-address-update.js';
|
|
59
62
|
import { nodeTransportsOption } from './node-transports.js';
|
|
60
63
|
import { resolveRelayServer } from './relay-server.js';
|
|
61
64
|
import { replacesAdvertisedAddrs, resolveAnnounceAddrs } from './announce-addrs.js';
|
|
@@ -73,8 +76,8 @@ const timing = debug('sereus:cadre:timing');
|
|
|
73
76
|
*
|
|
74
77
|
* Ten minutes sits comfortably inside the peerStore's one-hour address expiry
|
|
75
78
|
* (`MAX_ADDRESS_AGE`, see `peer-addr-book.ts`) with headroom for several missed
|
|
76
|
-
* passes, and far above the
|
|
77
|
-
* strand-addr RPC fan-out stays cheap.
|
|
79
|
+
* passes, and far above the reconcile pass's 2-minute backstop the refresh rides
|
|
80
|
+
* on, so the strand-addr RPC fan-out stays cheap.
|
|
78
81
|
*/
|
|
79
82
|
export const STRAND_PEER_ADDR_REFRESH_MS = 10 * 60 * 1000;
|
|
80
83
|
/**
|
|
@@ -83,8 +86,9 @@ export const STRAND_PEER_ADDR_REFRESH_MS = 10 * 60 * 1000;
|
|
|
83
86
|
* phone asking before its `CadrePeer` row has replicated to the sibling is refused
|
|
84
87
|
* and its delegate grant goes unrecorded, and waiting the full
|
|
85
88
|
* {@link STRAND_PEER_ADDR_REFRESH_MS} for the next try is the delay this bounds.
|
|
86
|
-
*
|
|
87
|
-
*
|
|
89
|
+
* A pass arms its own wake for the earliest due time it leaves behind
|
|
90
|
+
* (`nextReconcileWakeMs`), so a sibling that keeps failing costs one timed-out
|
|
91
|
+
* RPC a minute, with no pass in between.
|
|
88
92
|
*/
|
|
89
93
|
export const STRAND_PEER_ADDR_RETRY_MS = 60 * 1000;
|
|
90
94
|
/**
|
|
@@ -451,8 +455,8 @@ export class CadreNode {
|
|
|
451
455
|
*
|
|
452
456
|
* Cleared by {@link cleanup} only, deliberately NOT by {@link detachStrand}: the watcher's
|
|
453
457
|
* filter rejection stops a replica through that very method, and forgetting the id there
|
|
454
|
-
* would have the next
|
|
455
|
-
* relaunch the replica —
|
|
458
|
+
* would have the next watcher run find it unknown again, re-admit the strand provisionally
|
|
459
|
+
* and relaunch the replica — on every other run, forever. A strand id's sApp never changes,
|
|
456
460
|
* and the map is bounded by the strands this party has published, like
|
|
457
461
|
* {@link discoveredStrands}.
|
|
458
462
|
*/
|
|
@@ -480,19 +484,24 @@ export class CadreNode {
|
|
|
480
484
|
* wait that ends in mutual denial (the upstream gate is fail-closed on
|
|
481
485
|
* timeout, unlike the fail-open connection gater). Optimystic's predicate is
|
|
482
486
|
* synchronous by contract. So the set is refreshed OUT OF BAND instead — after
|
|
483
|
-
* start,
|
|
484
|
-
*
|
|
485
|
-
*
|
|
486
|
-
*
|
|
487
|
+
* start, at the head of every {@link reconcileControlCohort} pass, whenever a
|
|
488
|
+
* membership table changes (the gate watch {@link start} takes through
|
|
489
|
+
* `ControlDatabase.watchMembershipTables`, which hears a local commit and a change
|
|
490
|
+
* replicated into this node's storage alike, within a 50 ms window), and immediately
|
|
491
|
+
* after every LOCAL membership mutation so a just-vouched peer is admitted
|
|
492
|
+
* before the writer's call returns. That last refresh is AUTOMATIC: the
|
|
493
|
+
* control database notifies {@link refreshMembershipGate} after every
|
|
487
494
|
* committed `CadrePeer` write (see `ControlDatabase.mutateCadrePeer`), so no
|
|
488
|
-
* writer has to remember. A change
|
|
489
|
-
*
|
|
490
|
-
*
|
|
491
|
-
*
|
|
492
|
-
*
|
|
493
|
-
*
|
|
494
|
-
*
|
|
495
|
-
*
|
|
495
|
+
* writer has to remember. A change this node's storage never hears of — a
|
|
496
|
+
* machine outside the block's cohort on a party larger than the replication
|
|
497
|
+
* breadth — is picked up at the pass's 2-minute backstop: bounded staleness,
|
|
498
|
+
* acceptable because this gate is defense in depth: rows an unadmitted peer
|
|
499
|
+
* manages to write are still disbelieved at read time by the voucher-anchored
|
|
500
|
+
* predicate. For cohort choice the same window means a member whose row
|
|
501
|
+
* arrived by replication is left out of this node's control-block cohorts
|
|
502
|
+
* until that refresh. Optimystic reads the predicate live on every lookup, so
|
|
503
|
+
* replacing this set is the whole re-evaluation; there is no per-peer
|
|
504
|
+
* reclassify call to make.
|
|
496
505
|
*/
|
|
497
506
|
this.authorizedControlPeers = new Set();
|
|
498
507
|
/**
|
|
@@ -530,9 +539,9 @@ export class CadreNode {
|
|
|
530
539
|
* When this node last ANNOUNCED a delegate, keyed `<targetPeerId>\n<strandId>`
|
|
531
540
|
* (the peer announced TO, not the delegate). Throttles
|
|
532
541
|
* {@link refreshDelegateGrants} to once per `DELEGATE_GRANT_TTL_MS / 2` per
|
|
533
|
-
* (relay, strand) so
|
|
534
|
-
* chatter; the launch/resume announce passes record here too.
|
|
535
|
-
* strand is no longer running are pruned on each reconcile pass.
|
|
542
|
+
* (relay, strand) so a run of change-driven reconcile passes never becomes
|
|
543
|
+
* per-pass RPC chatter; the launch/resume announce passes record here too.
|
|
544
|
+
* Keys whose strand is no longer running are pruned on each reconcile pass.
|
|
536
545
|
*/
|
|
537
546
|
this.delegateAnnounceAt = new Map();
|
|
538
547
|
/**
|
|
@@ -559,12 +568,13 @@ export class CadreNode {
|
|
|
559
568
|
* strand, keyed `peerStrandKey(siblingControlPeerId, strandId)` → epoch ms; a
|
|
560
569
|
* missing key is due. Bounds {@link refreshStrandPeerAddrs}'s fan-out per
|
|
561
570
|
* (sibling, strand) rather than per strand, so a sibling that connects after a
|
|
562
|
-
* pass is asked on the
|
|
563
|
-
* sibling's answer set. An answer (even an empty one) makes the sibling
|
|
564
|
-
* in {@link STRAND_PEER_ADDR_REFRESH_MS}; no answer, in
|
|
565
|
-
* {@link STRAND_PEER_ADDR_RETRY_MS}
|
|
566
|
-
*
|
|
567
|
-
*
|
|
571
|
+
* pass is asked on the pass its connection starts instead of waiting out a stamp
|
|
572
|
+
* another sibling's answer set. An answer (even an empty one) makes the sibling
|
|
573
|
+
* due again in {@link STRAND_PEER_ADDR_REFRESH_MS}; no answer, in
|
|
574
|
+
* {@link STRAND_PEER_ADDR_RETRY_MS}, and each pass arms its own wake for the
|
|
575
|
+
* earliest due time it leaves behind (`nextReconcileWakeMs`). Keys whose strand
|
|
576
|
+
* stopped running or whose sibling is no longer connected are pruned every pass,
|
|
577
|
+
* so a resumed strand and a reconnected sibling are both asked at once.
|
|
568
578
|
*/
|
|
569
579
|
this.strandAddrAskDueAt = new Map();
|
|
570
580
|
/**
|
|
@@ -646,6 +656,18 @@ export class CadreNode {
|
|
|
646
656
|
* `unpublishStrand` / `forgetJoinedStrand` as {@link formationStrandAddrs} is.
|
|
647
657
|
*/
|
|
648
658
|
this.pendingMembershipInvites = new Map();
|
|
659
|
+
/**
|
|
660
|
+
* The party key {@link adoptFormationMembershipInvite} just seated or read, by strand id,
|
|
661
|
+
* so the launch that follows a join does not read the `StrandPartyKey` row again. Taken
|
|
662
|
+
* (deleted) by the {@link resolveStrandPartyKey} that uses it, so any later launch reads
|
|
663
|
+
* the row; dropped by {@link detachStrand}. The joiner's counterpart of the founder's
|
|
664
|
+
* `StrandConfig.partyMemberPrivateKey`, without the app carrying the secret.
|
|
665
|
+
*
|
|
666
|
+
* NOTE: an entry is not dropped when the row is removed, because no `CadreNode` path removes
|
|
667
|
+
* it; an embedder calling `getControlDatabase().deleteStrandPartyKey` between a join and its
|
|
668
|
+
* launch would get the deleted key. If a node-level rotation is added, drop the entry there.
|
|
669
|
+
*/
|
|
670
|
+
this.formedPartyKeys = new Map();
|
|
649
671
|
/**
|
|
650
672
|
* Invite keys of the membership invitations this process has staged, by
|
|
651
673
|
* {@link adoptFormationMembershipInvite} or from a `JoinSuccess` row
|
|
@@ -739,7 +761,7 @@ export class CadreNode {
|
|
|
739
761
|
this.joinedStrandStore = null;
|
|
740
762
|
/**
|
|
741
763
|
* This session's joined strands: {@link joinedStrandStore} and the party-wide
|
|
742
|
-
* `JoinedStrand` table, unioned with the control rows each watcher
|
|
764
|
+
* `JoinedStrand` table, unioned with the control rows each watcher run, published and
|
|
743
765
|
* removed by the reconcile pass. Per SESSION, unlike the store — rebuilt at every
|
|
744
766
|
* {@link start}, which is what ends a revoked join's "keep offering until this session
|
|
745
767
|
* ends" (see `JoinedStrandSession.forgetAfterThisSession`).
|
|
@@ -752,17 +774,41 @@ export class CadreNode {
|
|
|
752
774
|
/** Listener that re-publishes the self record when reachable addresses change. */
|
|
753
775
|
this.selfPeerUpdateHandler = null;
|
|
754
776
|
/**
|
|
755
|
-
*
|
|
756
|
-
*
|
|
757
|
-
*
|
|
758
|
-
* {@link
|
|
777
|
+
* The reconcile pass's schedule (see {@link reconcileControlCohort}): single-flight
|
|
778
|
+
* runs on `table-change-watch.ts`'s rule, with no table subscription of its own — the
|
|
779
|
+
* gate watch ({@link membershipGateWatch}), the control connection listeners,
|
|
780
|
+
* `self:peer:update`, and the pass's own wake ({@link armReconcileWake}) `trigger` it,
|
|
781
|
+
* and its backstop (`network.controlCohort.reconcileMs`, default
|
|
782
|
+
* `DEFAULT_CONTROL_COHORT_RECONCILE_MS`) runs it when nothing has. Built alongside
|
|
783
|
+
* {@link recordRefreshTimer} in {@link startRecordRefresh}, about 1 s after start, and
|
|
784
|
+
* disposed in {@link stopRecordRefresh}; its timers never keep the loop alive.
|
|
785
|
+
*/
|
|
786
|
+
this.controlReconcileRuns = null;
|
|
787
|
+
/**
|
|
788
|
+
* The subscription on the membership tables (`ControlDatabase.watchMembershipTables`) that
|
|
789
|
+
* re-materializes the membership gate and triggers {@link controlReconcileRuns} on every
|
|
790
|
+
* change ({@link onMembershipGateTablesChanged}).
|
|
791
|
+
* Taken in {@link start} right after the membership listener is wired, disposed in
|
|
792
|
+
* {@link cleanup} before the control database closes.
|
|
793
|
+
*/
|
|
794
|
+
this.membershipGateWatch = null;
|
|
795
|
+
/** The second membership-view drop after a sibling connects ({@link armMembershipViewReload}); cleared in {@link stopRecordRefresh}. */
|
|
796
|
+
this.membershipViewReloadTimer = null;
|
|
797
|
+
/**
|
|
798
|
+
* Consecutive passes whose dial of a selected sibling or a cold-start bootstrap peer
|
|
799
|
+
* failed (`ControlCohortPassOutcome.unconnected`), for the redial ladder
|
|
800
|
+
* ({@link armReconcileWake}); 0 after a pass none of whose dials did. Per session:
|
|
801
|
+
* reset in {@link stopRecordRefresh}.
|
|
759
802
|
*/
|
|
760
|
-
this.
|
|
803
|
+
this.reconcileRedialStreak = 0;
|
|
804
|
+
/** Peers a {@link redialClaimedNode} loop is running for, so a repeated claim starts no second one. */
|
|
805
|
+
this.claimRedialPeers = new Set();
|
|
761
806
|
/**
|
|
762
|
-
* Single-flight guard for {@link reconcileControlCohort}.
|
|
763
|
-
*
|
|
764
|
-
*
|
|
765
|
-
*
|
|
807
|
+
* Single-flight guard for {@link reconcileControlCohort}. An explicit call, the
|
|
808
|
+
* `self:peer:update` trigger and a scheduled run can fire close together; collapsing
|
|
809
|
+
* concurrent passes into one in-flight run prevents two passes from double-dialing the
|
|
810
|
+
* same siblings (mirrors {@link registerSelfInFlight}). A scheduled run waits for this
|
|
811
|
+
* pass and then starts its own rather than joining it.
|
|
766
812
|
*/
|
|
767
813
|
this.reconcileControlCohortInFlight = null;
|
|
768
814
|
/**
|
|
@@ -785,19 +831,27 @@ export class CadreNode {
|
|
|
785
831
|
* machine in the party is already discarding this node's address as stale.
|
|
786
832
|
*/
|
|
787
833
|
this.lastSelfRecordPublishAt = null;
|
|
834
|
+
/**
|
|
835
|
+
* The newest record this process signed for its own `CadrePeer` row: written by either path
|
|
836
|
+
* of {@link publishSelfRecord}, or handed to a member to write by {@link redeemCadreInvitation}.
|
|
837
|
+
* What {@link publishSelfRecordUnlessCurrent} compares the row against. Per session, cleared
|
|
838
|
+
* with {@link lastSelfRecordPublishAt}: a row carried over from an earlier process is never
|
|
839
|
+
* taken as current.
|
|
840
|
+
*/
|
|
841
|
+
this.lastSignedSelfRecord = null;
|
|
788
842
|
/** Say-once latch for {@link escalateIfSelfRecordStale}; re-armed by the next publish. */
|
|
789
843
|
this.selfRecordStaleWarned = false;
|
|
790
844
|
// ── Write-while-alone re-replication (control-write-ensure-replicated) ──────
|
|
791
845
|
/**
|
|
792
846
|
* Re-replication queue for owner `authorizePeer` writes that committed while
|
|
793
|
-
* this node was alone
|
|
794
|
-
*
|
|
795
|
-
*
|
|
847
|
+
* this node was alone ({@link committedAlone}) — the Optimystic commit was
|
|
848
|
+
* local-only (the block's cohort was this node) and never broadcast. Maps the
|
|
849
|
+
* affected subject peerId → 'authorize', to
|
|
796
850
|
* re-issue once the cohort grows. Removals are NOT tracked here — a delete
|
|
797
851
|
* leaves a `Revocation` tombstone, tracked by {@link pendingRevocations}; a
|
|
798
852
|
* removePeer also clears any queued authorize for the same subject (the row is
|
|
799
|
-
* gone, so re-issuing the insert would be wrong). Drained
|
|
800
|
-
*
|
|
853
|
+
* gone, so re-issuing the insert would be wrong). Drained when a sibling becomes
|
|
854
|
+
* reachable ({@link updateControlSiblingReachability}) by {@link drainPendingControlReplication}.
|
|
801
855
|
*/
|
|
802
856
|
this.pendingPeerWrites = new Map();
|
|
803
857
|
/**
|
|
@@ -821,14 +875,6 @@ export class CadreNode {
|
|
|
821
875
|
* exactly a "removals from before this lifetime" case).
|
|
822
876
|
*/
|
|
823
877
|
this.reissuedHeldRevocations = false;
|
|
824
|
-
/**
|
|
825
|
-
* Whether this process has seen the singleton `Revocation` ledger marker filed — its own
|
|
826
|
-
* `'opened'` or an `'already-open'` answer, in {@link openRevocationLedgerIfDue}. A
|
|
827
|
-
* record of work done, not a cached authorization answer: the marker can never be
|
|
828
|
-
* deleted (`NoDelete`), so this can only go stale if the database itself is replaced,
|
|
829
|
-
* which is why stop() clears it with the other per-process replication state.
|
|
830
|
-
*/
|
|
831
|
-
this.revocationLedgerOpened = false;
|
|
832
878
|
/** This node's own `CadrePeer` self-write committed local-only (re-touched on growth). */
|
|
833
879
|
this.pendingSelfPeerWrite = false;
|
|
834
880
|
/** This node's own `DeviceToken` self-write committed local-only (re-touched on growth). */
|
|
@@ -843,14 +889,24 @@ export class CadreNode {
|
|
|
843
889
|
/** Single-flight guard for the re-replication drain (mirrors {@link registerSelfInFlight}). */
|
|
844
890
|
this.drainControlReplicationInFlight = null;
|
|
845
891
|
/**
|
|
846
|
-
*
|
|
847
|
-
*
|
|
848
|
-
*
|
|
892
|
+
* Connected control peers whose identify says they serve the control database
|
|
893
|
+
* ({@link onControlPeerIdentified}): the peers Optimystic can count into a control block's
|
|
894
|
+
* cohort, before its membership check. A peer joins on its first `peer:identify` of a
|
|
895
|
+
* connected period, not on `connection:open`: until it identifies, every control block's
|
|
896
|
+
* cohort is this node alone. It leaves when its last connection closes. Read through
|
|
897
|
+
* {@link hasReachableControlSibling}, which applies the membership check.
|
|
849
898
|
*/
|
|
850
|
-
this.
|
|
851
|
-
/**
|
|
852
|
-
|
|
853
|
-
|
|
899
|
+
this.identifiedControlPeers = new Set();
|
|
900
|
+
/**
|
|
901
|
+
* {@link hasReachableControlSibling} at its last evaluation, so the drain fires on the
|
|
902
|
+
* false→true edge only (the earliest point a re-issue reaches the cohort), not on every later
|
|
903
|
+
* identify. Re-armed to false when no reachable sibling remains
|
|
904
|
+
* ({@link updateControlSiblingReachability}).
|
|
905
|
+
*/
|
|
906
|
+
this.controlSiblingReachable = false;
|
|
907
|
+
/** `peer:identify` listener: {@link onControlPeerIdentified} (teardown in {@link stopRecordRefresh}). */
|
|
908
|
+
this.controlPeerIdentifyHandler = null;
|
|
909
|
+
/** `connection:close` listener: {@link onControlConnectionClosed} (teardown in {@link stopRecordRefresh}). */
|
|
854
910
|
this.controlConnectionCloseHandler = null;
|
|
855
911
|
this.config = config;
|
|
856
912
|
this.hostUnclaimedStrands = config.hostUnclaimedStrands ?? config.profile === 'storage';
|
|
@@ -1013,7 +1069,7 @@ export class CadreNode {
|
|
|
1013
1069
|
// closed before any network bring-up, and the retained dial targets are
|
|
1014
1070
|
// loaded before the first reconcile pass could consult them.
|
|
1015
1071
|
this.initializeBootstrapPeerStore();
|
|
1016
|
-
// The remembered cross-party joins, which the strand watcher built below
|
|
1072
|
+
// The remembered cross-party joins, which the strand watcher built below reads
|
|
1017
1073
|
// beside the control rows. Before network bring-up, so a mis-scoped injected store
|
|
1018
1074
|
// fails closed like the two above.
|
|
1019
1075
|
this.initializeJoinedStrandStore();
|
|
@@ -1092,6 +1148,20 @@ export class CadreNode {
|
|
|
1092
1148
|
// rest of start() and the refresh early-returns while it is — a write in that
|
|
1093
1149
|
// window is covered by start's own pass below.
|
|
1094
1150
|
this.controlDatabase.setMembershipChangeListener((reason) => this.refreshMembershipGate(reason));
|
|
1151
|
+
// The gate watch: a change to any membership table, replicated in or committed here,
|
|
1152
|
+
// re-materializes the gate and starts a reconcile pass. Separate from the pass so a
|
|
1153
|
+
// replicated revocation does not wait behind a pass spending its dial budget on an
|
|
1154
|
+
// unreachable sibling. The watch handler only arms a timer, so nothing runs under the
|
|
1155
|
+
// write lock the membership listener is called with; `onChange` runs from that timer.
|
|
1156
|
+
//
|
|
1157
|
+
// NOTE: a table this `Database` has not yet touched hears local commits only
|
|
1158
|
+
// (`table-change-watch.ts`). The start refresh reads `CadrePeer`, `OwnerKey` and
|
|
1159
|
+
// `Revocation`; the four invitation and grant tables are first read when some row is
|
|
1160
|
+
// admitted through one, and a load the committed-read path served is not kept and
|
|
1161
|
+
// arms no bridge, so a chain row that replicates in before then waits for the next
|
|
1162
|
+
// change on a touched table or the backstop. Nothing depends on a chain row until a
|
|
1163
|
+
// `CadrePeer` or `OwnerKey` row refers to it, and that row is itself such a change.
|
|
1164
|
+
this.membershipGateWatch = this.controlDatabase.watchMembershipTables(() => this.onMembershipGateTablesChanged(), { label: 'membership-gate' });
|
|
1095
1165
|
// Committed-delete seam → write-while-alone revocation queue. Same wiring
|
|
1096
1166
|
// window argument as the membership listener above.
|
|
1097
1167
|
this.controlDatabase.setGuardedDeleteListener((revocation) => this.noteGuardedDelete(revocation));
|
|
@@ -1104,7 +1174,7 @@ export class CadreNode {
|
|
|
1104
1174
|
this.strandWatcher = new StrandWatcher(queryable, {
|
|
1105
1175
|
onStrandAdded: async (strand) => this.handleStrandAdded(strand),
|
|
1106
1176
|
onStrandRemoved: async (strandId) => this.handleStrandRemoved(strandId)
|
|
1107
|
-
}, this.config.strandFilter ?? { mode: 'all' }, this.config.strandWatchInterval ??
|
|
1177
|
+
}, this.config.strandFilter ?? { mode: 'all' }, this.config.strandWatchInterval ?? DEFAULT_STRAND_WATCH_BACKSTOP_MS, this // CadreNode implements SAppIdLookup
|
|
1108
1178
|
);
|
|
1109
1179
|
t0 = performance.now();
|
|
1110
1180
|
await this.strandWatcher.start();
|
|
@@ -1174,8 +1244,8 @@ export class CadreNode {
|
|
|
1174
1244
|
timing('[start] total: %dms', Math.round(performance.now() - tTotal));
|
|
1175
1245
|
log('CadreNode started successfully');
|
|
1176
1246
|
// Wire the control-connection growth listeners immediately (not via the
|
|
1177
|
-
// delayed refresh path) so no early `
|
|
1178
|
-
// write-while-alone re-replication drain must fire
|
|
1247
|
+
// delayed refresh path) so no early `peer:identify` is missed — the
|
|
1248
|
+
// write-while-alone re-replication drain must fire when the first sibling is reachable.
|
|
1179
1249
|
this.wireControlConnectionListeners();
|
|
1180
1250
|
// Arm the control network's peer-join block catch-up now that the control
|
|
1181
1251
|
// DB (which the push-time membership gate reads) is up. Armed even when
|
|
@@ -1524,7 +1594,7 @@ export class CadreNode {
|
|
|
1524
1594
|
`but this node serves party ${partyId} — refusing to offer another party's joined strands`);
|
|
1525
1595
|
}
|
|
1526
1596
|
this.joinedStrandStore = store;
|
|
1527
|
-
this.joinedStrands = new JoinedStrandSession(store, this.createPartyJoinedStrandLedger());
|
|
1597
|
+
this.joinedStrands = new JoinedStrandSession(store, this.createPartyJoinedStrandLedger(), () => this.strandWatcher?.refresh());
|
|
1528
1598
|
}
|
|
1529
1599
|
/**
|
|
1530
1600
|
* Construct (or adopt) the node-local strand network state (see
|
|
@@ -1665,8 +1735,8 @@ export class CadreNode {
|
|
|
1665
1735
|
}
|
|
1666
1736
|
/**
|
|
1667
1737
|
* Arm the CONTROL network's peer-join block catch-up: push every block in the
|
|
1668
|
-
* control database's own raw store to each
|
|
1669
|
-
*
|
|
1738
|
+
* control database's own raw store to each AUTHORIZED member on each
|
|
1739
|
+
* (re)connection, skipping what it is known to hold. This is what physically replicates
|
|
1670
1740
|
* control blocks committed while the writer was alone — the named
|
|
1671
1741
|
* collection-header blocks written once at genesis above all, whose revision
|
|
1672
1742
|
* never moves again, so no later commit ever carries them to a member that
|
|
@@ -1675,7 +1745,9 @@ export class CadreNode {
|
|
|
1675
1745
|
* for peers it knew about before the restart).
|
|
1676
1746
|
*
|
|
1677
1747
|
* Unlike the per-strand instances (see `strand-instance-manager.ts`), pushes
|
|
1678
|
-
* are gated on {@link
|
|
1748
|
+
* are gated on the authorized-member snapshot ({@link authorizedControlPeers},
|
|
1749
|
+
* which {@link refreshAuthorizedControlPeers} materializes from the same rows and
|
|
1750
|
+
* rule {@link isAuthorizedMember} queries), judged at push time: the control
|
|
1679
1751
|
* network's inbound connection gate deliberately admits non-members in
|
|
1680
1752
|
* several states (seed delivery to an un-enrolled node, an open enrollment
|
|
1681
1753
|
* window, an outstanding invitation, configured bootstrap/relay peers), and
|
|
@@ -1691,22 +1763,18 @@ export class CadreNode {
|
|
|
1691
1763
|
* The one non-member the gate admits is a device whose invitation
|
|
1692
1764
|
* redemption this member is admitting ({@link admitRedeemingDevice}).
|
|
1693
1765
|
*
|
|
1694
|
-
* No-ops (logged) when
|
|
1695
|
-
*
|
|
1696
|
-
* no way to dial.
|
|
1766
|
+
* No-ops (logged) when {@link controlBackfillStorage} answers nothing (disabled,
|
|
1767
|
+
* or no control storage configured) or the node exposes no key network — the
|
|
1768
|
+
* backfill would have nothing to read or no way to dial.
|
|
1697
1769
|
*/
|
|
1698
1770
|
startControlBackfill() {
|
|
1699
|
-
if (
|
|
1771
|
+
if (!this.controlNode) {
|
|
1700
1772
|
return;
|
|
1701
1773
|
}
|
|
1702
|
-
|
|
1703
|
-
// idempotent (it returns the same wrapped store `buildControlNodeOptions` already
|
|
1704
|
-
// resolved), so the catch-up cannot be silently disarmed by a future reordering that
|
|
1705
|
-
// moves this call ahead of the one that populated the field.
|
|
1706
|
-
const storage = this.resolveControlStorage();
|
|
1774
|
+
const storage = this.controlBackfillStorage();
|
|
1707
1775
|
const keyNetwork = this.controlNode.keyNetwork;
|
|
1708
1776
|
if (!storage || !keyNetwork) {
|
|
1709
|
-
log('Control peer-join block catch-up is inert: %s', !storage ? 'no control storage configured' : 'control node exposes no keyNetwork');
|
|
1777
|
+
log('Control peer-join block catch-up is inert: %s', !storage ? 'disabled, or no control storage configured' : 'control node exposes no keyNetwork');
|
|
1710
1778
|
return;
|
|
1711
1779
|
}
|
|
1712
1780
|
const networkName = this.controlNetworkName();
|
|
@@ -1719,7 +1787,11 @@ export class CadreNode {
|
|
|
1719
1787
|
// under — derived from the same networkName binding the node options
|
|
1720
1788
|
// used, never re-spelled here.
|
|
1721
1789
|
protocolPrefix: `/optimystic/${networkName}`,
|
|
1722
|
-
|
|
1790
|
+
reportBlockHeld: this.controlNode.reportBlockHeld,
|
|
1791
|
+
// The membership snapshot, not {@link isControlNetworkMember}: that predicate admits
|
|
1792
|
+
// everyone while the snapshot is empty and admits bootstrap/relay peers outright, and
|
|
1793
|
+
// neither may receive the control store.
|
|
1794
|
+
authorizePeer: async (peerId) => this.admittingDevices.has(peerId) || this.authorizedControlPeers.has(peerId)
|
|
1723
1795
|
}, {
|
|
1724
1796
|
// A shorter settle window than the strand default (1000 ms): the control
|
|
1725
1797
|
// store is small (a party's membership — dozens of blocks), a re-push is
|
|
@@ -1774,7 +1846,7 @@ export class CadreNode {
|
|
|
1774
1846
|
const release = this.countAdmittingDevice(peerId);
|
|
1775
1847
|
try {
|
|
1776
1848
|
const result = await backfill.forceCatchUpPeer(peerIdFromString(peerId));
|
|
1777
|
-
log('Device %s caught up before its admission: offered=%d accepted=%d rejected=%d', peerId, result.offered, result.accepted, result.rejected.length);
|
|
1849
|
+
log('Device %s caught up before its admission: offered=%d alreadyHeld=%d accepted=%d rejected=%d', peerId, result.offered, result.alreadyHeld, result.accepted, result.rejected.length);
|
|
1778
1850
|
return release;
|
|
1779
1851
|
}
|
|
1780
1852
|
catch (error) {
|
|
@@ -1838,6 +1910,19 @@ export class CadreNode {
|
|
|
1838
1910
|
this.controlStorage = wrapStorageWithCache(resolved, scope);
|
|
1839
1911
|
return this.controlStorage;
|
|
1840
1912
|
}
|
|
1913
|
+
/**
|
|
1914
|
+
* The store the control network's peer-join block catch-up reads, or `undefined` when this
|
|
1915
|
+
* runtime does not arm it: `controlBackfill.enabled: false`, or no control storage. Both of
|
|
1916
|
+
* its readers ask here so they cannot disagree: {@link startControlBackfill} arms on it, and
|
|
1917
|
+
* {@link buildControlNodeOptions} turns Optimystic's rebalance reaction off on it.
|
|
1918
|
+
*
|
|
1919
|
+
* Through the resolver, not the {@link controlStorage} memo field: the resolver is
|
|
1920
|
+
* idempotent, so the catch-up cannot be silently disarmed by a future reordering that moves
|
|
1921
|
+
* a read ahead of the one that populated the field.
|
|
1922
|
+
*/
|
|
1923
|
+
controlBackfillStorage() {
|
|
1924
|
+
return this.config.controlBackfill?.enabled === false ? undefined : this.resolveControlStorage();
|
|
1925
|
+
}
|
|
1841
1926
|
/**
|
|
1842
1927
|
* Warn the operator when `network.announceAddrs` will silently cost this node the
|
|
1843
1928
|
* relay reachability it also configured.
|
|
@@ -1940,6 +2025,19 @@ export class CadreNode {
|
|
|
1940
2025
|
cohortQueryTimeoutMs: declaredCohortReadDeadlineMs(network)
|
|
1941
2026
|
}),
|
|
1942
2027
|
arachnode: { enableRingZulu: profile === 'storage' },
|
|
2028
|
+
// One join catch-up per node, for the strand node's reason (`strand-instance-manager.ts`,
|
|
2029
|
+
// at its `rebalance` option): Optimystic's rebalance growth arm would push a joining
|
|
2030
|
+
// member each control block again, one request per block. Every peer the growth arm
|
|
2031
|
+
// pushes to, the control catch-up pushes to as well, except the ones it deliberately
|
|
2032
|
+
// withholds the control store from: the cohort predicate (`isNetworkMember` below) admits
|
|
2033
|
+
// everyone while the membership snapshot is empty, and bootstrap/relay peers outright.
|
|
2034
|
+
// With the catch-up disabled the reaction stays on.
|
|
2035
|
+
//
|
|
2036
|
+
// NOTE: no ring shifts and no reclaimable shed blocks on this node either, unused today.
|
|
2037
|
+
// If storage nodes ever need them, turn the reaction back on: the catch-up already reports
|
|
2038
|
+
// its accepted pushes to the rebalance monitor (`reportBlockHeld`); measure whether the
|
|
2039
|
+
// growth arm still re-sends.
|
|
2040
|
+
...(this.controlBackfillStorage() && { rebalance: { enabled: false } }),
|
|
1943
2041
|
...(identityKey && { privateKey: identityKey }),
|
|
1944
2042
|
// With `network.relayAddrs` set, built with libp2p's relay discovery off (given a
|
|
1945
2043
|
// transport list to wrap — `node-transports.ts`), so the supervisor
|
|
@@ -2002,10 +2100,8 @@ export class CadreNode {
|
|
|
2002
2100
|
// Who this node SENDS control writes to (cohort and coordinator choice), where
|
|
2003
2101
|
// `authorizeInboundStream` is who may send to it. Without it Optimystic picks cohort
|
|
2004
2102
|
// members from the protocols a peer's peer-store record lists, which any connected
|
|
2005
|
-
// peer can fill.
|
|
2006
|
-
|
|
2007
|
-
// whose verified redemption is being written, which must hold the rows that admit it.
|
|
2008
|
-
isNetworkMember: (peerId) => this.isControlNetworkMember(peerId) || this.admittingDevices.has(peerId)
|
|
2103
|
+
// peer can fill.
|
|
2104
|
+
isNetworkMember: (peerId) => this.isControlCohortMember(peerId)
|
|
2009
2105
|
};
|
|
2010
2106
|
return nodeOptions;
|
|
2011
2107
|
}
|
|
@@ -2085,8 +2181,8 @@ export class CadreNode {
|
|
|
2085
2181
|
* (it redeems here by seating the row from its bundle, after which check 7
|
|
2086
2182
|
* admits) or from a sibling whose membership row is still replicating (check 5
|
|
2087
2183
|
* admits once it lands; if it has not landed by the deadline, the connection
|
|
2088
|
-
* is closed and
|
|
2089
|
-
* dials are never gated). The protocol guard refuses all of them every
|
|
2184
|
+
* is closed and the reconcile pass the close starts on either side re-forms it,
|
|
2185
|
+
* since outbound dials are never gated). The protocol guard refuses all of them every
|
|
2090
2186
|
* members-only protocol meanwhile. A circuit-relay reservation admitted by
|
|
2091
2187
|
* {@link admitControlRelayReservation} disarms the deadline (see
|
|
2092
2188
|
* `membership-connection-gater.ts` → "The relay-reservation seam").
|
|
@@ -2310,6 +2406,17 @@ export class CadreNode {
|
|
|
2310
2406
|
}
|
|
2311
2407
|
return this.authorizedControlPeers.size === 0 || this.authorizedControlPeers.has(remotePeerId);
|
|
2312
2408
|
}
|
|
2409
|
+
/**
|
|
2410
|
+
* Whether this node sends control writes to `remotePeerId`: Optimystic's `isNetworkMember`
|
|
2411
|
+
* for the control node (cohort and coordinator choice), and the membership half of
|
|
2412
|
+
* {@link hasReachableControlSibling}. The stream gate's rule ({@link isControlNetworkMember}),
|
|
2413
|
+
* unlogged, because Optimystic asks several times per peer per cohort assembly and counts
|
|
2414
|
+
* refusals on its own debug line; plus a device whose verified redemption is being written
|
|
2415
|
+
* ({@link admittingDevices}), which must hold the rows that admit it.
|
|
2416
|
+
*/
|
|
2417
|
+
isControlCohortMember(remotePeerId) {
|
|
2418
|
+
return this.isControlNetworkMember(remotePeerId) || this.admittingDevices.has(remotePeerId);
|
|
2419
|
+
}
|
|
2313
2420
|
/**
|
|
2314
2421
|
* Per-stream authorization for the protocol guard's `members-snapshot` class
|
|
2315
2422
|
* (`control-protocol-guard.ts`): the four Optimystic control-DB protocols
|
|
@@ -2349,7 +2456,8 @@ export class CadreNode {
|
|
|
2349
2456
|
* this listener with its write lock HELD (`notifyMembershipChanged`), so a retrying
|
|
2350
2457
|
* read here would sleep its backoff holding the lock and stall every other local
|
|
2351
2458
|
* writer. Nothing is lost — this refresh already keeps the previous snapshot on
|
|
2352
|
-
* failure and is re-driven by the next membership
|
|
2459
|
+
* failure and is re-driven by the next membership change (local or replicated, through
|
|
2460
|
+
* the gate watch) and by the reconcile pass's backstop.
|
|
2353
2461
|
*
|
|
2354
2462
|
* The trusted-owner anchor sync ({@link syncOwnerAnchor}) runs FIRST, so the snapshot
|
|
2355
2463
|
* built here judges `CadrePeer` rows against the anchor as the `OwnerKey` table now
|
|
@@ -2424,15 +2532,20 @@ export class CadreNode {
|
|
|
2424
2532
|
// commit is what would authorize it), and the gate re-judges at push time.
|
|
2425
2533
|
//
|
|
2426
2534
|
// NOTE: every refresh re-arms every connected peer that is not yet caught
|
|
2427
|
-
// up
|
|
2428
|
-
// connection that is admitted but never authorized — a configured
|
|
2535
|
+
// up. A connection that is admitted but never authorized — a configured
|
|
2429
2536
|
// bootstrap or relay peer, the steady state for a NAT'd node — is
|
|
2430
|
-
// therefore re-judged on every
|
|
2431
|
-
//
|
|
2432
|
-
//
|
|
2433
|
-
//
|
|
2434
|
-
//
|
|
2537
|
+
// therefore re-judged on every refresh, forever: every local membership
|
|
2538
|
+
// write, every replicated membership change (the gate watch), and every
|
|
2539
|
+
// reconcile pass, down to the 2-minute backstop on an idle party. Each
|
|
2540
|
+
// such pass is one timer and a lookup in the snapshot just set (the gate
|
|
2541
|
+
// reads no control database). If a node ever holds many such connections,
|
|
2542
|
+
// or refreshes get frequent, skip re-arming peers whose last pass was
|
|
2543
|
+
// denied under the SAME membership snapshot instead of re-arming
|
|
2544
|
+
// unconditionally.
|
|
2435
2545
|
void this.controlBackfill?.scheduleConnectedPeers();
|
|
2546
|
+
// Same join order for the growth edge: a sibling that identified before this commit
|
|
2547
|
+
// authorized it becomes reachable here.
|
|
2548
|
+
this.updateControlSiblingReachability();
|
|
2436
2549
|
await this.hangUpRevokedControlPeers(rows, loadTombstones);
|
|
2437
2550
|
await this.noteOwnRemoval(rows, loadTombstones);
|
|
2438
2551
|
}
|
|
@@ -2619,9 +2732,10 @@ export class CadreNode {
|
|
|
2619
2732
|
return this.bootstrapPeerIds;
|
|
2620
2733
|
}
|
|
2621
2734
|
/**
|
|
2622
|
-
* What the strand watcher
|
|
2735
|
+
* What the strand watcher reads: this party's control rows plus a row per cross-party
|
|
2623
2736
|
* join (see {@link joinedStrands}), so a joined strand is offered and relaunched on
|
|
2624
|
-
* the same path as the party's own.
|
|
2737
|
+
* the same path as the party's own. It runs on a change to either table; the
|
|
2738
|
+
* machine-local joins wake it through {@link JoinedStrandSession}'s change callback.
|
|
2625
2739
|
*/
|
|
2626
2740
|
createStrandQueryable() {
|
|
2627
2741
|
return {
|
|
@@ -2633,7 +2747,14 @@ export class CadreNode {
|
|
|
2633
2747
|
log('Querying strands from control database');
|
|
2634
2748
|
const control = await this.controlDatabase.queryStrands();
|
|
2635
2749
|
return this.joinedStrands ? await this.joinedStrands.withControlRows(control) : control;
|
|
2636
|
-
}
|
|
2750
|
+
},
|
|
2751
|
+
watchChanges: (onChange, options) => {
|
|
2752
|
+
if (!this.controlDatabase) {
|
|
2753
|
+
throw new Error('CadreNode must initialize its control database before watching its strand tables');
|
|
2754
|
+
}
|
|
2755
|
+
return this.controlDatabase.watchTables(['CadreControl.Strand', 'CadreControl.JoinedStrand'], onChange, options);
|
|
2756
|
+
},
|
|
2757
|
+
lastReadPartial: () => this.joinedStrands?.lastReadPartial() ?? false
|
|
2637
2758
|
};
|
|
2638
2759
|
}
|
|
2639
2760
|
/**
|
|
@@ -2648,10 +2769,6 @@ export class CadreNode {
|
|
|
2648
2769
|
return this.controlDatabase;
|
|
2649
2770
|
};
|
|
2650
2771
|
return {
|
|
2651
|
-
// NOTE: one more control read per watcher poll (5 s), and for a party that never
|
|
2652
|
-
// joined anything it reads a never-written block, which consults the cohort. If poll
|
|
2653
|
-
// cost shows up in a device profile, cache the list and re-read it on the reconcile
|
|
2654
|
-
// cadence.
|
|
2655
2772
|
list: () => database().queryJoinedStrands(),
|
|
2656
2773
|
names: async (strandId) => (await database().queryStrand(strandId)) !== null
|
|
2657
2774
|
|| (await database().queryJoinedStrand(strandId)) !== null,
|
|
@@ -2729,26 +2846,30 @@ export class CadreNode {
|
|
|
2729
2846
|
* after start (non-blocking). {@link registerSelf} is idempotent and safely
|
|
2730
2847
|
* no-ops when it cannot yet sign/insert (e.g. owner key not installed),
|
|
2731
2848
|
* so the timer is harmless even when registration only becomes possible later.
|
|
2849
|
+
* A record this process already put in place that is still current (a founding owner's
|
|
2850
|
+
* explicit publish, a redeemed invitation's admission, a restart's first-growth re-touch) is
|
|
2851
|
+
* not written again ({@link publishSelfRecordUnlessCurrent}). Whether the node is connected
|
|
2852
|
+
* yet does not matter: a device that redeems within a second of starting is.
|
|
2732
2853
|
*/
|
|
2733
2854
|
scheduleSelfRegistration() {
|
|
2734
2855
|
this.selfRegistrationTimer = setTimeout(() => {
|
|
2735
2856
|
void (async () => {
|
|
2736
2857
|
try {
|
|
2737
|
-
await this.
|
|
2858
|
+
await this.publishSelfRecordUnlessCurrent();
|
|
2738
2859
|
}
|
|
2739
2860
|
catch (error) {
|
|
2740
|
-
// Background task — a failed publish must not crash the node. A
|
|
2741
|
-
//
|
|
2742
|
-
//
|
|
2743
|
-
//
|
|
2861
|
+
// Background task — a failed publish must not crash the node. A publish that loses
|
|
2862
|
+
// its row to another writer — the owner's re-touch, or a restarted machine's own row
|
|
2863
|
+
// replicating in while the update runs — is re-run by the write funnel over the row
|
|
2864
|
+
// it finds (`ControlDatabase.updateSelfPeerRecord`); one that lost past the retry
|
|
2865
|
+
// budget fails here, and the next connected reconcile pass is the fallback
|
|
2744
2866
|
// (publishFirstSelfRecord).
|
|
2745
2867
|
log('Self-registration failed: %o', error);
|
|
2746
2868
|
}
|
|
2747
|
-
// Wire the ongoing refresh
|
|
2748
|
-
// an eager reconcile pass
|
|
2749
|
-
// interval was just armed inside startRecordRefresh).
|
|
2869
|
+
// Wire the ongoing refresh and the control-cohort reconcile schedule, then
|
|
2870
|
+
// run an eager reconcile pass now that the node has settled.
|
|
2750
2871
|
this.startRecordRefresh();
|
|
2751
|
-
|
|
2872
|
+
this.controlReconcileRuns?.trigger(0);
|
|
2752
2873
|
})();
|
|
2753
2874
|
}, 1000); // Small delay to ensure node is fully started
|
|
2754
2875
|
// Node-only: don't keep the event loop alive solely for this timer.
|
|
@@ -2820,16 +2941,15 @@ export class CadreNode {
|
|
|
2820
2941
|
return 'skipped';
|
|
2821
2942
|
}
|
|
2822
2943
|
// Collected only past that skip, which a node waiting for its row replicated in takes on
|
|
2823
|
-
//
|
|
2944
|
+
// reconcile pass made while a sibling is reachable ({@link publishFirstSelfRecord}).
|
|
2824
2945
|
const addrs = await this.collectSelfAddrs();
|
|
2825
2946
|
if (!existing && ownerService) {
|
|
2826
2947
|
// First-time row: requires an owner signature (the node is its own
|
|
2827
2948
|
// owner). insertSelfPeerRecord throws if no owner key is present.
|
|
2828
2949
|
const record = this.signSelfRecord(peerId, signingKey, addrs, null);
|
|
2950
|
+
const aloneBefore = this.committedAlone();
|
|
2829
2951
|
if (await ownerService.insertSelfPeerRecord(record)) {
|
|
2830
|
-
|
|
2831
|
-
this.pendingSelfPeerWrite = true;
|
|
2832
|
-
this.noteSelfRecordPublished();
|
|
2952
|
+
this.noteSelfPeerWritten(record, aloneBefore);
|
|
2833
2953
|
log('registerSelf: inserted own CadrePeer record (owner-signed, updatedAt=%d, %d addrs, sig=%s…)', record.updatedAt, addrs.length, record.sig.slice(0, 16));
|
|
2834
2954
|
return 'inserted';
|
|
2835
2955
|
}
|
|
@@ -2843,43 +2963,97 @@ export class CadreNode {
|
|
|
2843
2963
|
// first publish", add a fourth SelfRegistrationOutcome rather than re-labelling.
|
|
2844
2964
|
log('registerSelf: own CadrePeer row appeared mid-publish (concurrent authorize); self-updating to carry the signature');
|
|
2845
2965
|
}
|
|
2846
|
-
//
|
|
2847
|
-
//
|
|
2848
|
-
// the
|
|
2849
|
-
//
|
|
2850
|
-
//
|
|
2851
|
-
//
|
|
2852
|
-
|
|
2853
|
-
|
|
2854
|
-
|
|
2855
|
-
const current = existing ?? await this.controlDatabase.queryPeerRecord(peerId);
|
|
2856
|
-
if (!current) {
|
|
2966
|
+
// The update body reads the row under the write lock and signs over what it finds, on
|
|
2967
|
+
// every attempt — so neither `existing` (a read that predates this publish) nor a first
|
|
2968
|
+
// attempt that lost the row to another writer decides the `UpdatedAt` the strictly-greater
|
|
2969
|
+
// self-update rule is measured against, and a removal landing mid-publish re-reads as
|
|
2970
|
+
// absent instead of matching no rows under a 'refreshed'. The record it returns is the
|
|
2971
|
+
// one in the row.
|
|
2972
|
+
const aloneBefore = this.committedAlone();
|
|
2973
|
+
const record = await this.controlDatabase.updateSelfPeerRecord(peerId, (current) => this.signSelfRecord(peerId, signingKey, addrs, current));
|
|
2974
|
+
if (record === null) {
|
|
2857
2975
|
// Two causes, and the second is a steady state rather than a race: the row really
|
|
2858
2976
|
// vanished mid-publish (next refresh re-inserts), or this node was REVOKED — its row
|
|
2859
2977
|
// is physically present, so insertCadrePeer's in-lock existence check no-op'd above,
|
|
2860
|
-
// but
|
|
2861
|
-
//
|
|
2862
|
-
//
|
|
2978
|
+
// but the body reads it as absent because its stamp is retired. The revoked case
|
|
2979
|
+
// repeats every heartbeat and never re-inserts; that is intended (a revoked node has
|
|
2980
|
+
// nothing to publish), so this stays a skip, not an error.
|
|
2863
2981
|
log('registerSelf: own CadrePeer row not readable (vanished mid-publish, or this node is revoked and its stamp retired); skipping');
|
|
2864
2982
|
return 'skipped';
|
|
2865
2983
|
}
|
|
2866
|
-
|
|
2867
|
-
await this.controlDatabase.updateSelfPeerRecord(record);
|
|
2868
|
-
if (this.committedAlone())
|
|
2869
|
-
this.pendingSelfPeerWrite = true;
|
|
2870
|
-
this.noteSelfRecordPublished();
|
|
2984
|
+
this.noteSelfPeerWritten(record, aloneBefore);
|
|
2871
2985
|
log('registerSelf: refreshed own CadrePeer record (updatedAt=%d, %d addrs, sig=%s…)', record.updatedAt, addrs.length, record.sig.slice(0, 16));
|
|
2872
2986
|
return 'refreshed';
|
|
2873
2987
|
}
|
|
2874
2988
|
/**
|
|
2875
|
-
*
|
|
2876
|
-
*
|
|
2877
|
-
*
|
|
2989
|
+
* Bookkeeping for a committed write of this node's own `CadrePeer` row by {@link publishSelfRecord}:
|
|
2990
|
+
* `record` is the one the row holds (for the update path, the one its body signed on the
|
|
2991
|
+
* attempt that committed, not the first it signed). Queued for the drain's re-touch
|
|
2992
|
+
* ({@link pendingSelfPeerWrite}) when no sibling was reachable when the write started or when
|
|
2993
|
+
* it ended: a sibling that identifies while the write is out may do so after the write's cohort
|
|
2994
|
+
* was chosen, and the first-growth drain skips a row holding {@link lastSignedSelfRecord} unless
|
|
2995
|
+
* this flag is set. Both fields are set here together so no write can leave the record without
|
|
2996
|
+
* the flag.
|
|
2997
|
+
*/
|
|
2998
|
+
noteSelfPeerWritten(record, aloneBefore) {
|
|
2999
|
+
if (aloneBefore || this.committedAlone())
|
|
3000
|
+
this.pendingSelfPeerWrite = true;
|
|
3001
|
+
this.lastSignedSelfRecord = record;
|
|
3002
|
+
this.noteSelfRecordPublished();
|
|
3003
|
+
}
|
|
3004
|
+
/**
|
|
3005
|
+
* Record that this node's own address record is in place as of `at`, and re-arm the
|
|
3006
|
+
* stale-record warning. Called for BOTH publishing paths of {@link publishSelfRecord}
|
|
3007
|
+
* ({@link noteSelfPeerWritten}), and by {@link publishSelfRecordUnlessCurrent} with the stamp
|
|
3008
|
+
* of a record already in place — a `skipped` publish wrote nothing and found nothing, so it
|
|
3009
|
+
* must not refresh the stamp.
|
|
2878
3010
|
*/
|
|
2879
|
-
noteSelfRecordPublished() {
|
|
2880
|
-
this.lastSelfRecordPublishAt =
|
|
3011
|
+
noteSelfRecordPublished(at = Date.now()) {
|
|
3012
|
+
this.lastSelfRecordPublishAt = at;
|
|
2881
3013
|
this.selfRecordStaleWarned = false;
|
|
2882
3014
|
}
|
|
3015
|
+
/**
|
|
3016
|
+
* {@link registerSelf}, unless the own row already holds the record this process last signed
|
|
3017
|
+
* ({@link lastSignedSelfRecord}) and that record is current ({@link isCurrentSelfRecord});
|
|
3018
|
+
* then nothing is written and the record counts as published at its own stamp. For a publish
|
|
3019
|
+
* that puts a record in place (the boot timer, the reconcile pass's first publish, the
|
|
3020
|
+
* first-growth drain), never the heartbeat, which exists to re-stamp freshness. `'current'`
|
|
3021
|
+
* is this method's own answer; {@link registerSelf} always writes.
|
|
3022
|
+
*/
|
|
3023
|
+
async publishSelfRecordUnlessCurrent() {
|
|
3024
|
+
if (this.registerSelfInFlight) {
|
|
3025
|
+
return this.registerSelfInFlight;
|
|
3026
|
+
}
|
|
3027
|
+
const record = this.lastSignedSelfRecord;
|
|
3028
|
+
if (record !== null && await this.isCurrentSelfRecord(record)) {
|
|
3029
|
+
this.noteSelfRecordPublished(record.updatedAt);
|
|
3030
|
+
log('registerSelf: own CadrePeer record already current (updatedAt=%d); not written again', record.updatedAt);
|
|
3031
|
+
return 'current';
|
|
3032
|
+
}
|
|
3033
|
+
return this.registerSelf();
|
|
3034
|
+
}
|
|
3035
|
+
/**
|
|
3036
|
+
* Is `record` still the one to publish: signed less than half a heartbeat ago, with this
|
|
3037
|
+
* node's profile, over the addresses it would publish now, and the signature the own row
|
|
3038
|
+
* holds? The age bound keeps a record that is not re-written fresh for resolvers: at most half
|
|
3039
|
+
* a heartbeat old here and re-stamped by the heartbeat within one more, so never older than
|
|
3040
|
+
* 1.5 heartbeats (11.25 min) against {@link DEFAULT_PEER_RECORD_MAX_AGE_MS}. Cheap checks
|
|
3041
|
+
* first: the row comes from the membership view, the addresses from the app's resolver.
|
|
3042
|
+
*/
|
|
3043
|
+
async isCurrentSelfRecord(record) {
|
|
3044
|
+
if (Date.now() - record.updatedAt >= DEFAULT_PEER_RECORD_HEARTBEAT_MS / 2 || record.profile !== this.config.profile) {
|
|
3045
|
+
return false;
|
|
3046
|
+
}
|
|
3047
|
+
const row = await this.controlDatabase?.queryPeerRecord(record.peerId);
|
|
3048
|
+
if (row?.sig !== record.sig) {
|
|
3049
|
+
return false;
|
|
3050
|
+
}
|
|
3051
|
+
// Sanitized, as the record a redemption carries is (`signRedeemRequest`); a published
|
|
3052
|
+
// record is not, so one whose list sanitizing would change (over 16 addresses) is never
|
|
3053
|
+
// current and is written as before.
|
|
3054
|
+
const addrs = sanitizeAddrs(await this.collectSelfAddrs());
|
|
3055
|
+
return addrs.length === record.addrs.length && addrs.every((addr, i) => addr === record.addrs[i]);
|
|
3056
|
+
}
|
|
2883
3057
|
/**
|
|
2884
3058
|
* Report an abandoned control write to the embedding app.
|
|
2885
3059
|
*
|
|
@@ -3007,19 +3181,24 @@ export class CadreNode {
|
|
|
3007
3181
|
* process does not learn it was removed ({@link noteOwnRemoval}), and pays that read on every
|
|
3008
3182
|
* pass.
|
|
3009
3183
|
*
|
|
3010
|
-
*
|
|
3011
|
-
*
|
|
3012
|
-
*
|
|
3013
|
-
* then never see rows written while it
|
|
3014
|
-
*
|
|
3015
|
-
*
|
|
3184
|
+
* Only while a sibling is reachable ({@link hasReachableControlSibling}): a row written
|
|
3185
|
+
* elsewhere arrives from a sibling, and an owner's publish committed alone is local-only and
|
|
3186
|
+
* forks its own history — a restarted owner whose embedder runs a pass before it reconnects
|
|
3187
|
+
* would write its row on the stale side of the fork, and then never see rows written while it
|
|
3188
|
+
* was down (`cadre-invite-any-member` runs that pass and fails every time without this
|
|
3189
|
+
* condition). A connection is not enough: until the sibling identifies, the write's cohort is
|
|
3190
|
+
* this node alone. The boot-time publish and the first-growth drain already try at the
|
|
3191
|
+
* node's lone and first-sibling moments; this retries on the passes after.
|
|
3192
|
+
*
|
|
3193
|
+
* A machine admitted by invitation finds its row already holding the record its redemption
|
|
3194
|
+
* carried, and writes nothing ({@link publishSelfRecordUnlessCurrent}).
|
|
3016
3195
|
*/
|
|
3017
3196
|
async publishFirstSelfRecord() {
|
|
3018
3197
|
if (this.lastSelfRecordPublishAt !== null || this.isAwaitingClaim() || this.removal !== null
|
|
3019
|
-
|| this.
|
|
3198
|
+
|| !this.hasReachableControlSibling()) {
|
|
3020
3199
|
return;
|
|
3021
3200
|
}
|
|
3022
|
-
const outcome = await this.
|
|
3201
|
+
const outcome = await this.publishSelfRecordUnlessCurrent().catch((error) => {
|
|
3023
3202
|
log('reconcileControlCohort: first self-record publish failed (continuing): %o', error);
|
|
3024
3203
|
return 'skipped';
|
|
3025
3204
|
});
|
|
@@ -3055,15 +3234,23 @@ export class CadreNode {
|
|
|
3055
3234
|
this.escalateIfSelfRecordStale(reason, error);
|
|
3056
3235
|
});
|
|
3057
3236
|
};
|
|
3058
|
-
const
|
|
3059
|
-
|
|
3060
|
-
|
|
3237
|
+
const backstopMs = this.controlReconcileBackstopMs();
|
|
3238
|
+
// The pass's schedule. A run never joins a pass already in flight: that pass may have
|
|
3239
|
+
// read its rows before the change that woke this run, so joining it would drop the
|
|
3240
|
+
// change until the backstop. It waits for the pass, then starts its own.
|
|
3241
|
+
this.controlReconcileRuns = scheduleRuns(async () => {
|
|
3242
|
+
await this.reconcileControlCohortInFlight?.catch(() => undefined);
|
|
3243
|
+
await this.reconcileControlCohort().catch((error) => log('Control-cohort reconcile (scheduled) failed: %o', error));
|
|
3244
|
+
}, { label: 'control-reconcile', backstopMs });
|
|
3061
3245
|
// Address churn re-publishes the self record AND re-checks the control cohort:
|
|
3062
3246
|
// a relay-reservation rotation / NAT change can drop a sibling connection, so
|
|
3063
|
-
// the next pass should re-observe and re-dial it.
|
|
3064
|
-
this
|
|
3247
|
+
// the next pass should re-observe and re-dial it. Only address churn: libp2p also
|
|
3248
|
+
// fires this for a protocol handler registered or removed (see selfAddressesChanged).
|
|
3249
|
+
this.selfPeerUpdateHandler = (event) => {
|
|
3250
|
+
if (!selfAddressesChanged(event.detail))
|
|
3251
|
+
return;
|
|
3065
3252
|
republish('self:peer:update');
|
|
3066
|
-
|
|
3253
|
+
this.controlReconcileRuns?.trigger(0);
|
|
3067
3254
|
};
|
|
3068
3255
|
this.controlNode.addEventListener('self:peer:update', this.selfPeerUpdateHandler);
|
|
3069
3256
|
// A node AUTHORIZED AFTER IT BOOTED (seeded, claimed, invited) gets its first
|
|
@@ -3083,12 +3270,28 @@ export class CadreNode {
|
|
|
3083
3270
|
// dependable there.
|
|
3084
3271
|
this.recordRefreshTimer = setInterval(() => republish('heartbeat'), DEFAULT_PEER_RECORD_HEARTBEAT_MS);
|
|
3085
3272
|
this.recordRefreshTimer?.unref?.();
|
|
3086
|
-
|
|
3087
|
-
this.controlCohortReconcileTimer = setInterval(() => reconcile('interval'), reconcileMs);
|
|
3088
|
-
this.controlCohortReconcileTimer?.unref?.();
|
|
3089
|
-
log('Record refresh wired (heartbeat=%dms, cohortReconcile=%dms)', DEFAULT_PEER_RECORD_HEARTBEAT_MS, reconcileMs);
|
|
3273
|
+
log('Record refresh wired (heartbeat=%dms, cohortReconcileBackstop=%dms)', DEFAULT_PEER_RECORD_HEARTBEAT_MS, backstopMs);
|
|
3090
3274
|
}
|
|
3091
|
-
/**
|
|
3275
|
+
/** The reconcile schedule's backstop: `network.controlCohort.reconcileMs`, default `DEFAULT_CONTROL_COHORT_RECONCILE_MS`. */
|
|
3276
|
+
controlReconcileBackstopMs() {
|
|
3277
|
+
return this.config.network?.controlCohort?.reconcileMs ?? DEFAULT_CONTROL_COHORT_RECONCILE_MS;
|
|
3278
|
+
}
|
|
3279
|
+
/**
|
|
3280
|
+
* The gate watch's run ({@link membershipGateWatch}): re-materialize the gate, so a
|
|
3281
|
+
* replicated revocation hangs up the removed machine now (`hangUpRevokedControlPeers`
|
|
3282
|
+
* ends every refresh), then start a reconcile pass for whatever else the change means
|
|
3283
|
+
* (a new sibling to dial, a tombstoned row to reap). A local membership write refreshes
|
|
3284
|
+
* the gate twice — once from the awaited membership listener, so the writer sees its
|
|
3285
|
+
* change before returning, and once here, 50 ms later. The listener's refresh runs under
|
|
3286
|
+
* the write lock, so its reads take the committed-read path and the view keeps nothing
|
|
3287
|
+
* from them; this refresh is the read that refills the view after the write, which some
|
|
3288
|
+
* later caller would otherwise pay. Accepted: one reload per change, not two.
|
|
3289
|
+
*/
|
|
3290
|
+
async onMembershipGateTablesChanged() {
|
|
3291
|
+
await this.refreshMembershipGate('table-change');
|
|
3292
|
+
this.controlReconcileRuns?.trigger(0);
|
|
3293
|
+
}
|
|
3294
|
+
/** Tear down the self-record refresh timers, the reconcile schedule and the listeners (see {@link cleanup}). */
|
|
3092
3295
|
stopRecordRefresh() {
|
|
3093
3296
|
if (this.selfRegistrationTimer) {
|
|
3094
3297
|
clearTimeout(this.selfRegistrationTimer);
|
|
@@ -3098,24 +3301,30 @@ export class CadreNode {
|
|
|
3098
3301
|
clearInterval(this.recordRefreshTimer);
|
|
3099
3302
|
this.recordRefreshTimer = null;
|
|
3100
3303
|
}
|
|
3101
|
-
if (this.
|
|
3102
|
-
|
|
3103
|
-
this.
|
|
3304
|
+
if (this.controlReconcileRuns) {
|
|
3305
|
+
this.controlReconcileRuns.dispose();
|
|
3306
|
+
this.controlReconcileRuns = null;
|
|
3307
|
+
}
|
|
3308
|
+
if (this.membershipViewReloadTimer) {
|
|
3309
|
+
clearTimeout(this.membershipViewReloadTimer);
|
|
3310
|
+
this.membershipViewReloadTimer = null;
|
|
3104
3311
|
}
|
|
3312
|
+
this.reconcileRedialStreak = 0;
|
|
3105
3313
|
if (this.controlNode) {
|
|
3106
3314
|
if (this.selfPeerUpdateHandler) {
|
|
3107
3315
|
this.controlNode.removeEventListener('self:peer:update', this.selfPeerUpdateHandler);
|
|
3108
3316
|
}
|
|
3109
|
-
if (this.
|
|
3110
|
-
this.controlNode.removeEventListener('
|
|
3317
|
+
if (this.controlPeerIdentifyHandler) {
|
|
3318
|
+
this.controlNode.removeEventListener('peer:identify', this.controlPeerIdentifyHandler);
|
|
3111
3319
|
}
|
|
3112
3320
|
if (this.controlConnectionCloseHandler) {
|
|
3113
3321
|
this.controlNode.removeEventListener('connection:close', this.controlConnectionCloseHandler);
|
|
3114
3322
|
}
|
|
3115
3323
|
}
|
|
3116
3324
|
this.selfPeerUpdateHandler = null;
|
|
3117
|
-
this.
|
|
3325
|
+
this.controlPeerIdentifyHandler = null;
|
|
3118
3326
|
this.controlConnectionCloseHandler = null;
|
|
3327
|
+
this.identifiedControlPeers.clear();
|
|
3119
3328
|
// Reset the write-while-alone re-replication state so a stop()→start() cycle
|
|
3120
3329
|
// re-arms the growth edge and re-runs BOTH one-shot passes. Any queued
|
|
3121
3330
|
// local-only writes are moot once the node is torn down: owner inserts are
|
|
@@ -3123,10 +3332,9 @@ export class CadreNode {
|
|
|
3123
3332
|
// first-growth revocation sweep — which is why the sweep flag must reset too,
|
|
3124
3333
|
// or the second lifetime would skip the sweep and strand exactly the
|
|
3125
3334
|
// committed-alone tombstones it exists to carry.
|
|
3126
|
-
this.
|
|
3335
|
+
this.controlSiblingReachable = false;
|
|
3127
3336
|
this.reconstructedLocalOnlyWrites = false;
|
|
3128
3337
|
this.reissuedHeldRevocations = false;
|
|
3129
|
-
this.revocationLedgerOpened = false;
|
|
3130
3338
|
this.pendingPeerWrites.clear();
|
|
3131
3339
|
this.pendingRevocations.clear();
|
|
3132
3340
|
this.pendingSelfPeerWrite = false;
|
|
@@ -3135,6 +3343,7 @@ export class CadreNode {
|
|
|
3135
3343
|
// from scratch, and carrying the previous lifetime's stamp forward would either
|
|
3136
3344
|
// suppress the stale-record warning or fire it on a node that just came up.
|
|
3137
3345
|
this.lastSelfRecordPublishAt = null;
|
|
3346
|
+
this.lastSignedSelfRecord = null;
|
|
3138
3347
|
this.selfRecordStaleWarned = false;
|
|
3139
3348
|
}
|
|
3140
3349
|
/**
|
|
@@ -3311,26 +3520,37 @@ export class CadreNode {
|
|
|
3311
3520
|
* Run one proactive control-cohort dial pass to keep this node connected to its
|
|
3312
3521
|
* cadre siblings (so the `CadreControl` collections form a replicating cohort).
|
|
3313
3522
|
* Public so the cohort-growth-driven re-replication path
|
|
3314
|
-
* (`control-write-ensure-replicated`)
|
|
3315
|
-
*
|
|
3316
|
-
*
|
|
3523
|
+
* (`control-write-ensure-replicated`), `addDrone`'s callers, `claimNode`,
|
|
3524
|
+
* `redeemCadreInvitation` and tests can run a pass on demand. On its own a pass
|
|
3525
|
+
* runs when something it acts on changes, and otherwise at a backstop
|
|
3526
|
+
* ({@link controlReconcileRuns}): once after start; on `self:peer:update`; when a
|
|
3527
|
+
* membership table changes, local commit or replicated ({@link membershipGateWatch});
|
|
3528
|
+
* when a control connection to an authorized sibling opens or closes
|
|
3529
|
+
* ({@link onControlSiblingConnectionChange}); on the redial and strand-address wakes
|
|
3530
|
+
* a pass arms for itself ({@link armReconcileWake}); and at the backstop,
|
|
3531
|
+
* `network.controlCohort.reconcileMs` (default `DEFAULT_CONTROL_COHORT_RECONCILE_MS`,
|
|
3532
|
+
* 2 minutes) after the last pass settled. A connection to a relay, a bootstrap node or
|
|
3533
|
+
* another party's machine starts no pass: the pass acts on none of them, beyond
|
|
3534
|
+
* counting them as "connected" for its write steps, which therefore run at the next
|
|
3535
|
+
* membership change or backstop on a one-machine party whose only connection is a
|
|
3536
|
+
* relay.
|
|
3317
3537
|
*
|
|
3318
3538
|
* Concurrent triggers collapse into a single in-flight pass
|
|
3319
3539
|
* (see {@link reconcileControlCohortInFlight}) so two passes never double-dial;
|
|
3320
|
-
* a call that joins an in-flight pass resolves to THAT pass's result
|
|
3540
|
+
* a call that joins an in-flight pass resolves to THAT pass's result (a scheduled
|
|
3541
|
+
* run waits for it instead, so the change that woke the run is seen).
|
|
3321
3542
|
* Best-effort throughout: a failure to resolve/dial any one sibling is logged
|
|
3322
3543
|
* and the pass continues; the whole pass is a no-op when the node is alone.
|
|
3323
3544
|
*
|
|
3324
3545
|
* Resolves to the peers the pass dialled ({@link ControlCohortReconcileResult}),
|
|
3325
3546
|
* so a caller can tell a link this pass opened from one something else opened —
|
|
3326
|
-
* the pass skips a peer that is already connected. The
|
|
3327
|
-
* discard it.
|
|
3547
|
+
* the pass skips a peer that is already connected. The scheduled runs discard it.
|
|
3328
3548
|
*/
|
|
3329
3549
|
async reconcileControlCohort() {
|
|
3330
3550
|
if (this.reconcileControlCohortInFlight) {
|
|
3331
3551
|
return this.reconcileControlCohortInFlight;
|
|
3332
3552
|
}
|
|
3333
|
-
const op = this.
|
|
3553
|
+
const op = this.runReconcileControlCohortAndArmWake();
|
|
3334
3554
|
this.reconcileControlCohortInFlight = op;
|
|
3335
3555
|
try {
|
|
3336
3556
|
return await op;
|
|
@@ -3339,38 +3559,75 @@ export class CadreNode {
|
|
|
3339
3559
|
this.reconcileControlCohortInFlight = null;
|
|
3340
3560
|
}
|
|
3341
3561
|
}
|
|
3562
|
+
/** One pass, then the wake it earns ({@link armReconcileWake}); the public result is the pass's `dialed`. */
|
|
3563
|
+
async runReconcileControlCohortAndArmWake() {
|
|
3564
|
+
const startedAt = Date.now();
|
|
3565
|
+
const outcome = await this.runReconcileControlCohort();
|
|
3566
|
+
this.armReconcileWake(outcome, startedAt);
|
|
3567
|
+
return { dialed: outcome.dialed };
|
|
3568
|
+
}
|
|
3569
|
+
/**
|
|
3570
|
+
* After every pass, explicit or scheduled: advance or reset the redial streak and
|
|
3571
|
+
* `trigger` the schedule for the earliest of the pass's own reasons to run again
|
|
3572
|
+
* (`nextReconcileWakeMs`): a redial for a sibling or bootstrap peer it left
|
|
3573
|
+
* unconnected, or a strand-address ask falling due before the backstop. Nothing else
|
|
3574
|
+
* the pass does needs a wake sooner than the backstop: delegate grants re-announce
|
|
3575
|
+
* every 15 minutes inside a 30-minute TTL, the FRET and formation address re-merges
|
|
3576
|
+
* guard a one-hour expiry, and the self record has its own heartbeat. A pass that
|
|
3577
|
+
* runs before the schedule exists (an explicit call during the first second after
|
|
3578
|
+
* start) still counts for the streak.
|
|
3579
|
+
*/
|
|
3580
|
+
armReconcileWake(outcome, startedAt) {
|
|
3581
|
+
this.reconcileRedialStreak = outcome.unconnected ? this.reconcileRedialStreak + 1 : 0;
|
|
3582
|
+
const runs = this.controlReconcileRuns;
|
|
3583
|
+
if (!runs) {
|
|
3584
|
+
return;
|
|
3585
|
+
}
|
|
3586
|
+
const wakeMs = nextReconcileWakeMs({
|
|
3587
|
+
redialStreak: this.reconcileRedialStreak,
|
|
3588
|
+
passStartedAt: startedAt,
|
|
3589
|
+
strandAddrDueAt: this.strandAddrAskDueAt.values(),
|
|
3590
|
+
now: Date.now(),
|
|
3591
|
+
backstopMs: this.controlReconcileBackstopMs(),
|
|
3592
|
+
});
|
|
3593
|
+
if (wakeMs !== undefined) {
|
|
3594
|
+
log('reconcileControlCohort: next pass in %dms (redial streak %d)', wakeMs, this.reconcileRedialStreak);
|
|
3595
|
+
runs.trigger(wakeMs);
|
|
3596
|
+
}
|
|
3597
|
+
}
|
|
3342
3598
|
/** Body of {@link reconcileControlCohort}; serialised by its single-flight guard. */
|
|
3343
3599
|
async runReconcileControlCohort() {
|
|
3344
3600
|
// Shutdown / not-yet-started guard (mirrors publishSelfRecord). A pass that
|
|
3345
3601
|
// fires after stop() began must early-return rather than touch a torn-down node.
|
|
3346
3602
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3347
|
-
return { dialed: [] };
|
|
3348
|
-
}
|
|
3349
|
-
//
|
|
3350
|
-
//
|
|
3351
|
-
//
|
|
3352
|
-
//
|
|
3353
|
-
// pass's length when that is longer (passes are single-flight, and one
|
|
3354
|
-
// unreachable sibling's dial alone can take `controlDialBudget().totalMs`).
|
|
3603
|
+
return { dialed: [], unconnected: false };
|
|
3604
|
+
}
|
|
3605
|
+
// Refresh the per-stream gate's materialized authorized set at the head of every
|
|
3606
|
+
// pass, so the sibling enumeration below judges rows the same way the gate does.
|
|
3607
|
+
// Changes reach the gate on their own through the gate watch; this refresh is the
|
|
3608
|
+
// backstop's, for a change this node's storage never heard of.
|
|
3355
3609
|
// NOTE: this refresh and the sibling enumeration below each run their own
|
|
3356
3610
|
// CadrePeer query (two reads per pass), plus a third from
|
|
3357
3611
|
// `refreshStrandPeerAddrs` on every pass where a strand is running AND this
|
|
3358
3612
|
// node holds a control connection (one read for the whole pass, not one per
|
|
3359
3613
|
// strand; it runs even when no sibling is due, because pruning departed
|
|
3360
3614
|
// siblings needs the current target set). Each also reads Revocation first.
|
|
3361
|
-
//
|
|
3362
|
-
//
|
|
3363
|
-
//
|
|
3364
|
-
//
|
|
3365
|
-
// control-founding-consult-budget.spec.ts).
|
|
3366
|
-
//
|
|
3615
|
+
// All of them are served from the control database's membership views
|
|
3616
|
+
// (`ControlDatabase.readMembershipRows`): an in-memory scan while the view is
|
|
3617
|
+
// current, storage only for the first read after the table changed or after
|
|
3618
|
+
// `MEMBERSHIP_VIEW_MAX_AGE_MS`, and an idle pass makes no cohort consult
|
|
3619
|
+
// (pinned in control-founding-consult-budget.spec.ts). Threading one row set
|
|
3620
|
+
// through the refresh, the reap, the enumeration and the strand-address targets
|
|
3621
|
+
// would save no storage read and was declined
|
|
3622
|
+
// (`control-reconcile-woken-by-changes-reads-once`); rows going stale during a
|
|
3623
|
+
// pass is harmless, since a change during the pass starts another one.
|
|
3367
3624
|
await this.refreshMembershipGate('reconcile');
|
|
3368
3625
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3369
|
-
return { dialed: [] };
|
|
3626
|
+
return { dialed: [], unconnected: false };
|
|
3370
3627
|
}
|
|
3371
3628
|
await this.publishFirstSelfRecord();
|
|
3372
3629
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3373
|
-
return { dialed: [] };
|
|
3630
|
+
return { dialed: [], unconnected: false };
|
|
3374
3631
|
}
|
|
3375
3632
|
// Refresh relay delegate grants BEFORE the sibling enumeration below: a
|
|
3376
3633
|
// solo cadre with a party relay has no siblings to dial but must still keep
|
|
@@ -3378,7 +3635,7 @@ export class CadreNode {
|
|
|
3378
3635
|
// the reservation and faces the connection gate again).
|
|
3379
3636
|
await this.refreshDelegateGrants();
|
|
3380
3637
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3381
|
-
return { dialed: [] };
|
|
3638
|
+
return { dialed: [], unconnected: false };
|
|
3382
3639
|
}
|
|
3383
3640
|
// Then re-warm each running strand's own address book from its FRET address
|
|
3384
3641
|
// records and from whichever siblings are due an ask. Same reasoning as
|
|
@@ -3386,14 +3643,15 @@ export class CadreNode {
|
|
|
3386
3643
|
// network, and every layer under cadre-core dials strand peers by bare peer id.
|
|
3387
3644
|
await this.refreshStrandPeerAddrs();
|
|
3388
3645
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3389
|
-
return { dialed: [] };
|
|
3646
|
+
return { dialed: [], unconnected: false };
|
|
3390
3647
|
}
|
|
3391
3648
|
const selfPeerId = this.controlNode.peerId.toString();
|
|
3392
3649
|
// Reap guarded rows this node still holds for incarnations an already-committed
|
|
3393
3650
|
// Revocation tombstone retires — the half of a removal replication cannot carry
|
|
3394
|
-
// (a delete of an already-absent row replays as nothing).
|
|
3395
|
-
//
|
|
3396
|
-
//
|
|
3651
|
+
// (a delete of an already-absent row replays as nothing). Only while a sibling is
|
|
3652
|
+
// reachable (hasReachableControlSibling, not merely connected): a reap is a write, and a
|
|
3653
|
+
// write committed alone is local-only and forks this node's own history, which is the
|
|
3654
|
+
// condition this work exists to stop creating
|
|
3397
3655
|
// (tickets/blocked/forked-control-collection-sync-livelocks.md). Nothing is urgent
|
|
3398
3656
|
// here — the stale row already reads as absent everywhere — so waiting for
|
|
3399
3657
|
// connectivity costs nothing and keeps the reap fork-CLOSING rather than
|
|
@@ -3415,7 +3673,7 @@ export class CadreNode {
|
|
|
3415
3673
|
// avoid. Bounded by how long a sweep runs, which is bounded by the backlog of
|
|
3416
3674
|
// unreaped rows (0 in steady state). If that backlog can ever be large, move the
|
|
3417
3675
|
// check inside reapRevokedRows' loop via an injected predicate.
|
|
3418
|
-
if (this.
|
|
3676
|
+
if (this.hasReachableControlSibling()) {
|
|
3419
3677
|
try {
|
|
3420
3678
|
const reaped = await this.controlDatabase.reapRevokedRows(selfPeerId);
|
|
3421
3679
|
if (reaped > 0) {
|
|
@@ -3428,32 +3686,14 @@ export class CadreNode {
|
|
|
3428
3686
|
log('reconcileControlCohort: reap pass failed (continuing): %o', error);
|
|
3429
3687
|
}
|
|
3430
3688
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3431
|
-
return { dialed: [] };
|
|
3432
|
-
}
|
|
3433
|
-
// File the Revocation ledger marker once, so that table stops being a never-written
|
|
3434
|
-
// block, which the storage layer re-checks with the cohort on every read (and every
|
|
3435
|
-
// membership lookup and guarded insert reads it). Connected-only for the reap's
|
|
3436
|
-
// reason above: a marker committed alone is local-only, and one filed by a
|
|
3437
|
-
// disconnected owner while another machine creates the same collection is exactly
|
|
3438
|
-
// that fork. Owner-only, and at most once per process — see openRevocationLedgerIfDue.
|
|
3439
|
-
//
|
|
3440
|
-
// NOTE: accepted tradeoff — a solo founder does not file the marker until it holds a
|
|
3441
|
-
// control connection (normally its first sibling; a relay or bootstrap connection also
|
|
3442
|
-
// counts — see getControlConnectionCount — and is the reap's same residual), so until
|
|
3443
|
-
// then it keeps paying one local findCluster per read of
|
|
3444
|
-
// the missing Revocation block (0.009 ms each, measured upstream; no network work on
|
|
3445
|
-
// a cohort of one). Fork safety weighed over that and kept; revisit if findCluster
|
|
3446
|
-
// ever shows up as material in a device profile, or if a solo-founding marker can be
|
|
3447
|
-
// made fork-safe (e.g. filed inside the genesis transaction, before any other machine
|
|
3448
|
-
// can hold the party's collections).
|
|
3449
|
-
await this.openRevocationLedgerIfDue();
|
|
3450
|
-
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3451
|
-
return { dialed: [] };
|
|
3689
|
+
return { dialed: [], unconnected: false };
|
|
3452
3690
|
}
|
|
3453
3691
|
// Publish this machine's unpublished cross-party joins party-wide and remove the rows a
|
|
3454
|
-
// self-revocation queued (JoinedStrandSession.syncWithParty).
|
|
3455
|
-
//
|
|
3456
|
-
// a join
|
|
3692
|
+
// self-revocation queued (JoinedStrandSession.syncWithParty). Behind the reap's gate,
|
|
3693
|
+
// for its reason; owner-only by the session's own signer check. Nothing is urgent:
|
|
3694
|
+
// a join recorded locally starts no pass, so it reaches the party on the next pass
|
|
3695
|
+
// while a sibling is reachable — a membership change, a sibling connecting, or the
|
|
3696
|
+
// backstop, 2 minutes at most.
|
|
3457
3697
|
try {
|
|
3458
3698
|
await this.joinedStrands?.syncWithParty();
|
|
3459
3699
|
}
|
|
@@ -3461,7 +3701,7 @@ export class CadreNode {
|
|
|
3461
3701
|
log('reconcileControlCohort: joined-strand sync failed (retrying next pass): %o', error);
|
|
3462
3702
|
}
|
|
3463
3703
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3464
|
-
return { dialed: [] };
|
|
3704
|
+
return { dialed: [], unconnected: false };
|
|
3465
3705
|
}
|
|
3466
3706
|
}
|
|
3467
3707
|
// 1. Enumerate known siblings. This membership read is itself a pull-on-read
|
|
@@ -3475,11 +3715,11 @@ export class CadreNode {
|
|
|
3475
3715
|
// because filling this table needs a connection and getting a connection
|
|
3476
3716
|
// needs this table. Fall back to the seed's bootstrap addresses; a solo
|
|
3477
3717
|
// cadre has none and this is a no-op. Must not throw or busy-loop.
|
|
3478
|
-
return
|
|
3718
|
+
return this.dialColdStartBootstrap();
|
|
3479
3719
|
}
|
|
3480
3720
|
// Re-guard after the await: a stop() may have raced the membership read.
|
|
3481
3721
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3482
|
-
return { dialed: [] };
|
|
3722
|
+
return { dialed: [], unconnected: false };
|
|
3483
3723
|
}
|
|
3484
3724
|
// 2. Classify backbone (always-on) members and select a bounded dial set.
|
|
3485
3725
|
const targetDegree = this.config.network?.controlCohort?.targetDegree
|
|
@@ -3493,51 +3733,32 @@ export class CadreNode {
|
|
|
3493
3733
|
// keeping what resolved for the dial loop below.
|
|
3494
3734
|
const resolved = await this.warmSiblingAddrBook(siblings);
|
|
3495
3735
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3496
|
-
return { dialed: [] };
|
|
3736
|
+
return { dialed: [], unconnected: false };
|
|
3497
3737
|
}
|
|
3498
3738
|
// 4. Skip already-connected peers (no re-dial / churn for live connections).
|
|
3499
3739
|
const connected = new Set(this.controlNode.getConnections().map((c) => c.remotePeer.toString()));
|
|
3500
3740
|
// 5. Dial each selected, not-yet-connected sibling, best-effort, from the
|
|
3501
|
-
// addresses step 3 already resolved for it.
|
|
3741
|
+
// addresses step 3 already resolved for it. A failed dial earns the pass a
|
|
3742
|
+
// redial wake (armReconcileWake); a sibling with nothing to dial does not.
|
|
3502
3743
|
const dialed = [];
|
|
3744
|
+
let unconnected = false;
|
|
3503
3745
|
for (const sibling of dials) {
|
|
3504
3746
|
if (!this._running || !this.controlNode) {
|
|
3505
|
-
return { dialed };
|
|
3747
|
+
return { dialed, unconnected: false };
|
|
3506
3748
|
}
|
|
3507
3749
|
if (connected.has(sibling.peerId)) {
|
|
3508
3750
|
continue;
|
|
3509
3751
|
}
|
|
3510
|
-
|
|
3752
|
+
const outcome = await this.dialControlSibling(sibling, resolved.get(sibling.peerId) ?? []);
|
|
3753
|
+
if (outcome === 'connected') {
|
|
3511
3754
|
dialed.push(sibling.peerId);
|
|
3512
3755
|
}
|
|
3756
|
+
else if (outcome === 'failed') {
|
|
3757
|
+
unconnected = true;
|
|
3758
|
+
}
|
|
3513
3759
|
}
|
|
3514
|
-
log('reconcileControlCohort: pass complete (siblings=%d, selected=%d, dialed=%d)', siblings.length, dials.length, dialed.length);
|
|
3515
|
-
return { dialed };
|
|
3516
|
-
}
|
|
3517
|
-
/**
|
|
3518
|
-
* The owner-only half of the reconcile pass's ledger-marker step: file the singleton
|
|
3519
|
-
* `Revocation` marker ({@link SeedBootstrapService.openRevocationLedger}) unless this
|
|
3520
|
-
* process has already seen it filed. The connectivity gate is the caller's
|
|
3521
|
-
* ({@link runReconcileControlCohort}); why the marker exists is on
|
|
3522
|
-
* {@link ControlDatabase.openRevocationLedger}.
|
|
3523
|
-
*
|
|
3524
|
-
* Sets {@link revocationLedgerOpened} on `'opened'` or `'already-open'`. A node that
|
|
3525
|
-
* cannot sign as an owner does nothing: it picks the owner's marker up the first time it
|
|
3526
|
-
* reads the table, like any other replicated row. Best-effort like every step of the
|
|
3527
|
-
* pass — a failure is logged and leaves the flag clear, so the next connected pass retries.
|
|
3528
|
-
*/
|
|
3529
|
-
async openRevocationLedgerIfDue() {
|
|
3530
|
-
if (this.revocationLedgerOpened || !this.seedBootstrapService?.canAuthorize()) {
|
|
3531
|
-
return;
|
|
3532
|
-
}
|
|
3533
|
-
try {
|
|
3534
|
-
const outcome = await this.seedBootstrapService.openRevocationLedger();
|
|
3535
|
-
this.revocationLedgerOpened = true;
|
|
3536
|
-
log('reconcileControlCohort: revocation ledger marker %s', outcome);
|
|
3537
|
-
}
|
|
3538
|
-
catch (error) {
|
|
3539
|
-
log('reconcileControlCohort: filing the revocation ledger marker failed (retrying next pass): %o', error);
|
|
3540
|
-
}
|
|
3760
|
+
log('reconcileControlCohort: pass complete (siblings=%d, selected=%d, dialed=%d, unconnected=%s)', siblings.length, dials.length, dialed.length, unconnected);
|
|
3761
|
+
return { dialed, unconnected };
|
|
3541
3762
|
}
|
|
3542
3763
|
/**
|
|
3543
3764
|
* Resolve every sibling's signed address record once and merge what resolves
|
|
@@ -3562,8 +3783,9 @@ export class CadreNode {
|
|
|
3562
3783
|
* current ({@link refreshDialHint}).
|
|
3563
3784
|
*
|
|
3564
3785
|
* NOTE: this resolves EVERY sibling serially before the dial loop below runs,
|
|
3565
|
-
* so it costs one record query per sibling per reconcile pass (
|
|
3566
|
-
*
|
|
3786
|
+
* so it costs one record query per sibling per reconcile pass (served from the
|
|
3787
|
+
* membership rows view while it is current) and each one delays the pass's first
|
|
3788
|
+
* dial. A cadre is a handful of devices, so today
|
|
3567
3789
|
* that is a few extra local reads; if cadres ever grow large, batch the records
|
|
3568
3790
|
* into one query, merge only on change, or move the warm pass after the dials.
|
|
3569
3791
|
*/
|
|
@@ -3688,40 +3910,41 @@ export class CadreNode {
|
|
|
3688
3910
|
});
|
|
3689
3911
|
}
|
|
3690
3912
|
/**
|
|
3691
|
-
* Dial one sibling from its already-resolved addresses, best-effort.
|
|
3692
|
-
* whether the dial resolved (false when no address resolves or the dial fails).
|
|
3913
|
+
* Dial one sibling from its already-resolved addresses, best-effort.
|
|
3693
3914
|
*
|
|
3694
3915
|
* A per-peer failure (NAT, offline, relay down, connection-gater denial, or
|
|
3695
3916
|
* the {@link controlDialBudget} expiring) is logged and swallowed so one
|
|
3696
3917
|
* unreachable sibling never aborts the pass — exactly like
|
|
3697
3918
|
* {@link SeedBootstrapService.applySeed}'s owner-dial loop. A failed dial is
|
|
3698
|
-
*
|
|
3699
|
-
*
|
|
3700
|
-
*
|
|
3919
|
+
* retried on the pass's redial ladder ({@link armReconcileWake}: 15 s, 30 s, 60 s,
|
|
3920
|
+
* then every backstop) and on any pass a change starts sooner. A sibling with no
|
|
3921
|
+
* dialable address, or whose every address relays through this node
|
|
3922
|
+
* ({@link SelfRelayOnlyError}), is not dialed at all and is `undialable`: it gets a
|
|
3923
|
+
* one-line log and no redial, since only a change — its record replicating in, or
|
|
3924
|
+
* the sibling connecting — can make it dialable, and each of those starts a pass.
|
|
3701
3925
|
*/
|
|
3702
3926
|
async dialControlSibling(sibling, resolved) {
|
|
3703
3927
|
const controlNode = this.controlNode;
|
|
3704
3928
|
if (!controlNode) {
|
|
3705
|
-
return
|
|
3929
|
+
return 'undialable';
|
|
3706
3930
|
}
|
|
3707
3931
|
const addrs = await this.resolveControlDialAddrs(sibling.peerId, resolved);
|
|
3708
3932
|
if (addrs.length === 0) {
|
|
3709
3933
|
log('reconcileControlCohort: no dialable control address for sibling %s; skipping', sibling.peerId);
|
|
3710
|
-
return
|
|
3934
|
+
return 'undialable';
|
|
3711
3935
|
}
|
|
3712
3936
|
try {
|
|
3713
3937
|
log('reconcileControlCohort: dialing sibling %s (%d addr(s))', sibling.peerId, addrs.length);
|
|
3714
3938
|
await dialPeerAddrs(controlNode, sibling.peerId, addrs, this.controlDialBudget(), `reconcileControlCohort dial of sibling ${sibling.peerId}`);
|
|
3715
|
-
return
|
|
3939
|
+
return 'connected';
|
|
3716
3940
|
}
|
|
3717
3941
|
catch (error) {
|
|
3718
3942
|
if (error instanceof SelfRelayOnlyError) {
|
|
3719
3943
|
log('reconcileControlCohort: sibling %s is reachable only by relaying through this node; waiting for it to reconnect', sibling.peerId);
|
|
3944
|
+
return 'undialable';
|
|
3720
3945
|
}
|
|
3721
|
-
|
|
3722
|
-
|
|
3723
|
-
}
|
|
3724
|
-
return false;
|
|
3946
|
+
log('reconcileControlCohort: dial of sibling %s failed (continuing): %o', sibling.peerId, error);
|
|
3947
|
+
return 'failed';
|
|
3725
3948
|
}
|
|
3726
3949
|
}
|
|
3727
3950
|
/**
|
|
@@ -3774,7 +3997,7 @@ export class CadreNode {
|
|
|
3774
3997
|
* These addresses are never merged into the libp2p address book: they are
|
|
3775
3998
|
* unverified, and the address book takes only verified ones (see
|
|
3776
3999
|
* `mergePeerAddrs`). Layers that dial by bare peer id use the connection this
|
|
3777
|
-
* dial opens, and after it drops the
|
|
4000
|
+
* dial opens, and after it drops the reconcile pass the close starts dials again.
|
|
3778
4001
|
*
|
|
3779
4002
|
* NOTE: an entry is consulted here only for a current sibling, so a peer revoked
|
|
3780
4003
|
* by another owner is not dialed from it, even though the entry stays (only this
|
|
@@ -3825,8 +4048,7 @@ export class CadreNode {
|
|
|
3825
4048
|
* Called for every seed this node accepts, on BOTH intake paths — the
|
|
3826
4049
|
* {@link applySeed} wrapper (which may run on a throwaway service, so the
|
|
3827
4050
|
* service itself is the wrong place to keep this) and the inbound
|
|
3828
|
-
* `/sereus/seed/1.0.0` handler via `onSeedApplied`.
|
|
3829
|
-
* skipped: there is nothing to dial.
|
|
4051
|
+
* `/sereus/seed/1.0.0` handler via `onSeedApplied`.
|
|
3830
4052
|
*
|
|
3831
4053
|
* `preferred` is the seed's own claim about its peers, exactly as
|
|
3832
4054
|
* `SeedBootstrapService.applySeed` uses it to choose its dial targets. That is
|
|
@@ -3834,14 +4056,15 @@ export class CadreNode {
|
|
|
3834
4056
|
* signature-checked against a trust-anchored signer before this runs, and the
|
|
3835
4057
|
* flag only *selects a dial target*; a dial grants no authority.
|
|
3836
4058
|
*
|
|
3837
|
-
*
|
|
4059
|
+
* The peers retained are `seedDialTargets` (addressless peers and self dropped), the same
|
|
4060
|
+
* ones the service dials; they are returned for the `seed:applied` payload.
|
|
3838
4061
|
*/
|
|
3839
4062
|
recordSeedBootstrapPeers(seed) {
|
|
3840
|
-
|
|
3841
|
-
|
|
3842
|
-
|
|
3843
|
-
}
|
|
4063
|
+
const targets = seedDialTargets(seed, this.controlNode?.peerId.toString());
|
|
4064
|
+
for (const { peerId, multiaddrs } of targets) {
|
|
4065
|
+
this.retainDialTarget(peerId, multiaddrs, 'recordSeedBootstrapPeers');
|
|
3844
4066
|
}
|
|
4067
|
+
return targets;
|
|
3845
4068
|
}
|
|
3846
4069
|
/**
|
|
3847
4070
|
* Retain (or replace) one peer's out-of-band dial target in
|
|
@@ -3855,9 +4078,8 @@ export class CadreNode {
|
|
|
3855
4078
|
* failing a seed or an add that has already been accepted.
|
|
3856
4079
|
*
|
|
3857
4080
|
* Two peers are never retained. One with no address: there is nothing to dial.
|
|
3858
|
-
* And self: `
|
|
3859
|
-
*
|
|
3860
|
-
* could be handed this node's own id. Retaining self would make the cold-start
|
|
4081
|
+
* And self: `addDrone` could be handed this node's own id (seed intake arrives
|
|
4082
|
+
* already filtered by `seedDialTargets`). Retaining self would make the cold-start
|
|
3861
4083
|
* pass dial this node forever (the steady-state pass filters self out of its
|
|
3862
4084
|
* sibling list for the same reason).
|
|
3863
4085
|
*/
|
|
@@ -3891,15 +4113,16 @@ export class CadreNode {
|
|
|
3891
4113
|
* help: it dials only siblings enumerated from that very table. Retrying here
|
|
3892
4114
|
* is the only way back in.
|
|
3893
4115
|
*
|
|
3894
|
-
* Unbounded
|
|
3895
|
-
*
|
|
3896
|
-
*
|
|
3897
|
-
* a
|
|
3898
|
-
*
|
|
3899
|
-
*
|
|
3900
|
-
*
|
|
3901
|
-
*
|
|
3902
|
-
*
|
|
4116
|
+
* Unbounded: the branch is already gated by "the control database has no
|
|
4117
|
+
* siblings", so it stops the moment the node is actually in the party, and a node
|
|
4118
|
+
* that is stranded MUST keep trying — a give-up rule would turn a transient outage
|
|
4119
|
+
* into a permanent one. A failed dial earns the pass a redial wake on the ladder
|
|
4120
|
+
* {@link armReconcileWake} runs (15 s, 30 s, 60 s, then every backstop, 2 minutes
|
|
4121
|
+
* by default), so the steady cost is one dial per bootstrap peer per backstop (a
|
|
4122
|
+
* seed prefers its minter and the party's always-on machines, a few at most), each
|
|
4123
|
+
* failing fast against an unreachable address.
|
|
4124
|
+
* NOTE: the ladder is per pass, not per peer; if seeds ever carry many preferred
|
|
4125
|
+
* peers, add per-peer backoff here.
|
|
3903
4126
|
*
|
|
3904
4127
|
* Best-effort per peer, exactly like {@link dialControlSibling}: one dead
|
|
3905
4128
|
* address never aborts the pass.
|
|
@@ -3910,14 +4133,15 @@ export class CadreNode {
|
|
|
3910
4133
|
// cannot mutate what we are iterating.
|
|
3911
4134
|
const targets = this.bootstrapPeerStore?.all();
|
|
3912
4135
|
if (!controlNode || !targets || targets.size === 0) {
|
|
3913
|
-
return [];
|
|
4136
|
+
return { dialed: [], unconnected: false };
|
|
3914
4137
|
}
|
|
3915
4138
|
// Same skip rule as step 3 of the steady-state pass: no churn on live links.
|
|
3916
4139
|
const connected = new Set(controlNode.getConnections().map((c) => c.remotePeer.toString()));
|
|
3917
4140
|
const dialed = [];
|
|
4141
|
+
let unconnected = false;
|
|
3918
4142
|
for (const [peerId, entry] of targets) {
|
|
3919
4143
|
if (!this._running || !this.controlNode) {
|
|
3920
|
-
return dialed;
|
|
4144
|
+
return { dialed, unconnected: false };
|
|
3921
4145
|
}
|
|
3922
4146
|
// NOTE: a connection this node still holds in `status: 'open'` after the remote
|
|
3923
4147
|
// has already aborted it counts as connected here, so the cold-start retry stays
|
|
@@ -3932,9 +4156,12 @@ export class CadreNode {
|
|
|
3932
4156
|
if (await this.dialBootstrapPeer(peerId, entry.addrs)) {
|
|
3933
4157
|
dialed.push(peerId);
|
|
3934
4158
|
}
|
|
4159
|
+
else {
|
|
4160
|
+
unconnected = true;
|
|
4161
|
+
}
|
|
3935
4162
|
}
|
|
3936
|
-
log('reconcileControlCohort: cold-start pass complete (bootstrap=%d, connected=%d, dialed=%d)', targets.size, connected.size, dialed.length);
|
|
3937
|
-
return dialed;
|
|
4163
|
+
log('reconcileControlCohort: cold-start pass complete (bootstrap=%d, connected=%d, dialed=%d, unconnected=%s)', targets.size, connected.size, dialed.length, unconnected);
|
|
4164
|
+
return { dialed, unconnected };
|
|
3938
4165
|
}
|
|
3939
4166
|
/**
|
|
3940
4167
|
* Bind a retained dial target's addresses (see {@link bootstrapPeerStore}) to
|
|
@@ -3951,28 +4178,32 @@ export class CadreNode {
|
|
|
3951
4178
|
bootstrapDialAddrs(peerId, addrs) {
|
|
3952
4179
|
return this.normalizeDialAddrs(this.parseMultiaddrs(addrs), peerId);
|
|
3953
4180
|
}
|
|
3954
|
-
/**
|
|
3955
|
-
|
|
4181
|
+
/**
|
|
4182
|
+
* Dial one peer from its retained dial target's addresses, best-effort: a cold-start bootstrap
|
|
4183
|
+
* peer, or a just-claimed node ({@link redialClaimedNode}), which `context` names in the logs.
|
|
4184
|
+
* Returns whether it connected.
|
|
4185
|
+
*/
|
|
4186
|
+
async dialBootstrapPeer(peerId, addrs, context = 'reconcileControlCohort(cold-start)') {
|
|
3956
4187
|
const controlNode = this.controlNode;
|
|
3957
4188
|
if (!controlNode) {
|
|
3958
4189
|
return false;
|
|
3959
4190
|
}
|
|
3960
4191
|
const parsed = this.bootstrapDialAddrs(peerId, addrs);
|
|
3961
4192
|
if (parsed.length === 0) {
|
|
3962
|
-
log('
|
|
4193
|
+
log('%s: no dialable address for bootstrap peer %s; skipping', context, peerId);
|
|
3963
4194
|
return false;
|
|
3964
4195
|
}
|
|
3965
4196
|
try {
|
|
3966
|
-
log('
|
|
3967
|
-
await dialPeerAddrs(controlNode, peerId, parsed, this.controlDialBudget(),
|
|
4197
|
+
log('%s: dialing bootstrap peer %s (%d addr(s))', context, peerId, parsed.length);
|
|
4198
|
+
await dialPeerAddrs(controlNode, peerId, parsed, this.controlDialBudget(), `${context} dial of bootstrap peer ${peerId}`);
|
|
3968
4199
|
return true;
|
|
3969
4200
|
}
|
|
3970
4201
|
catch (error) {
|
|
3971
4202
|
if (error instanceof SelfRelayOnlyError) {
|
|
3972
|
-
log('
|
|
4203
|
+
log('%s: bootstrap peer %s is reachable only by relaying through this node; waiting for it to reconnect', context, peerId);
|
|
3973
4204
|
}
|
|
3974
4205
|
else {
|
|
3975
|
-
log('
|
|
4206
|
+
log('%s: dial of bootstrap peer %s failed (continuing): %o', context, peerId, error);
|
|
3976
4207
|
}
|
|
3977
4208
|
return false;
|
|
3978
4209
|
}
|
|
@@ -3982,31 +4213,33 @@ export class CadreNode {
|
|
|
3982
4213
|
//
|
|
3983
4214
|
// Optimystic's coordinator commits a control write WITHOUT broadcasting when the
|
|
3984
4215
|
// block's cluster has ≤1 member. An owner that authorizes/removes a peer or
|
|
3985
|
-
// (re)publishes its own record while no sibling is
|
|
4216
|
+
// (re)publishes its own record while no sibling is reachable therefore writes a
|
|
3986
4217
|
// row that exists ONLY in its local control DB — a sibling that connects later
|
|
3987
4218
|
// never observes it (pull-on-read routes to the block's cluster, which never
|
|
3988
4219
|
// learned of the write). reconcileControlCohort shrinks the alone window but
|
|
3989
|
-
// cannot close it (a write in the instant before any sibling
|
|
3990
|
-
// local-only). The remedy: detect "wrote while alone" (
|
|
3991
|
-
//
|
|
3992
|
-
//
|
|
4220
|
+
// cannot close it (a write in the instant before any sibling identifies is still
|
|
4221
|
+
// local-only). The remedy: detect "wrote while alone" (no identified sibling the
|
|
4222
|
+
// cohort would admit: committedAlone), queue the affected row, and RE-ISSUE the write
|
|
4223
|
+
// once one is reachable, as an idempotent monotonic update that now broadcasts.
|
|
3993
4224
|
// See docs/architecture.md (Control Network Convergence → "Writes made while alone").
|
|
3994
4225
|
// ============================================================================
|
|
3995
4226
|
/**
|
|
3996
|
-
* Whether a control write happening now would commit local-only
|
|
3997
|
-
*
|
|
3998
|
-
*
|
|
3999
|
-
*
|
|
4000
|
-
*
|
|
4001
|
-
*
|
|
4002
|
-
*
|
|
4003
|
-
* is acceptable as the agreed first cut.
|
|
4227
|
+
* Whether a control write happening now would commit local-only: no sibling is reachable
|
|
4228
|
+
* ({@link hasReachableControlSibling}, the public form, so the two cannot drift). A relay, a
|
|
4229
|
+
* bootstrap node or a peer that has not identified yet holds a connection without being in the
|
|
4230
|
+
* write's cohort, so a write made with only those connected is queued too. One made alone at
|
|
4231
|
+
* the revision the cohort reached meanwhile is a fork no later consult sees, because a consult
|
|
4232
|
+
* compares revision numbers only; the re-issue on the growth edge is what conflicts with the
|
|
4233
|
+
* cohort's copy and pulls it in.
|
|
4004
4234
|
*
|
|
4005
|
-
*
|
|
4006
|
-
* the
|
|
4235
|
+
* Still a proxy for the precise signal (the block's own cohort): a reachable sibling outside
|
|
4236
|
+
* this block's cohort leaves the write local-only, caught by the cohort's pull-on-read. On the
|
|
4237
|
+
* control network every block's cohort is the whole party, so that gap is a party larger than
|
|
4238
|
+
* `CONTROL_REPLICATION_BREADTH`. Re-issuing a replicated row is harmless (an idempotent
|
|
4239
|
+
* monotonic bump).
|
|
4007
4240
|
*/
|
|
4008
4241
|
committedAlone() {
|
|
4009
|
-
return this.
|
|
4242
|
+
return !this.hasReachableControlSibling();
|
|
4010
4243
|
}
|
|
4011
4244
|
/**
|
|
4012
4245
|
* Record (or clear) a just-committed owner membership write in the
|
|
@@ -4060,7 +4293,7 @@ export class CadreNode {
|
|
|
4060
4293
|
/**
|
|
4061
4294
|
* Drain the write-while-alone re-replication queue: re-issue the writes that
|
|
4062
4295
|
* committed local-only now that the cohort can broadcast them. Public so the
|
|
4063
|
-
*
|
|
4296
|
+
* growth trigger ({@link updateControlSiblingReachability}) and tests can drive
|
|
4064
4297
|
* it; concurrent calls collapse into one in-flight drain
|
|
4065
4298
|
* ({@link drainControlReplicationInFlight}) so two growth signals never
|
|
4066
4299
|
* double-issue. Best-effort throughout — a per-row failure leaves that entry
|
|
@@ -4098,13 +4331,15 @@ export class CadreNode {
|
|
|
4098
4331
|
// and receiving-node staleness is the pre-existing pull-on-read /
|
|
4099
4332
|
// periodic-refresh property.
|
|
4100
4333
|
await this.drainPendingRevocations();
|
|
4101
|
-
// 2. Self rows.
|
|
4102
|
-
//
|
|
4103
|
-
//
|
|
4334
|
+
// 2. Self rows. A self write made while alone is always re-touched: the re-touch
|
|
4335
|
+
// is what carries it to the cohort. Otherwise the first growth re-touches a row
|
|
4336
|
+
// a prior process may have committed local-only, unless the row already holds a
|
|
4337
|
+
// current record this process signed (a redeemed invitation's admission).
|
|
4104
4338
|
// registerSelf is idempotent + single-flight, so this never races a
|
|
4105
4339
|
// duplicate publish with the heartbeat republish.
|
|
4106
4340
|
if (firstGrowth || this.pendingSelfPeerWrite) {
|
|
4107
|
-
|
|
4341
|
+
const publish = this.pendingSelfPeerWrite ? this.registerSelf() : this.publishSelfRecordUnlessCurrent();
|
|
4342
|
+
await publish.catch((error) => log('drain: registerSelf failed: %o', error));
|
|
4108
4343
|
this.pendingSelfPeerWrite = false;
|
|
4109
4344
|
}
|
|
4110
4345
|
if (firstGrowth || this.pendingSelfDeviceWrite) {
|
|
@@ -4254,8 +4489,11 @@ export class CadreNode {
|
|
|
4254
4489
|
}
|
|
4255
4490
|
/**
|
|
4256
4491
|
* Body of {@link reconstructAuthoredMembership}'s sweep: re-issue each candidate
|
|
4257
|
-
* owner-authored row, returning how many were re-touched. A
|
|
4258
|
-
*
|
|
4492
|
+
* owner-authored row, returning how many were re-touched. A row the sweep already holds as
|
|
4493
|
+
* self-published is skipped without a call; the re-touch itself decides again over the row
|
|
4494
|
+
* it reads under the lock, so a row that vanished or was self-published since the sweep's
|
|
4495
|
+
* read is reported, not written. A per-row failure is logged and skipped; a teardown
|
|
4496
|
+
* mid-sweep stops it.
|
|
4259
4497
|
*/
|
|
4260
4498
|
async reissueAuthoredMembershipRows(selfPeerId, members) {
|
|
4261
4499
|
let touched = 0;
|
|
@@ -4263,17 +4501,15 @@ export class CadreNode {
|
|
|
4263
4501
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
4264
4502
|
return touched;
|
|
4265
4503
|
}
|
|
4266
|
-
|
|
4504
|
+
// Self, queued for the drain (step 4 re-touches it once), or peer-owned (self-Sig
|
|
4505
|
+
// present) → not this sweep's to bump.
|
|
4506
|
+
if (member.peerId === selfPeerId || this.pendingPeerWrites.has(member.peerId) || member.sig) {
|
|
4267
4507
|
continue;
|
|
4268
4508
|
}
|
|
4269
4509
|
try {
|
|
4270
|
-
|
|
4271
|
-
|
|
4272
|
-
if (!record || record.sig) {
|
|
4273
|
-
continue;
|
|
4510
|
+
if (await this.reissuePeerAuthorize(member.peerId) === 'retouched') {
|
|
4511
|
+
touched++;
|
|
4274
4512
|
}
|
|
4275
|
-
await this.reissuePeerAuthorize(member.peerId, record.updatedAt);
|
|
4276
|
-
touched++;
|
|
4277
4513
|
}
|
|
4278
4514
|
catch (error) {
|
|
4279
4515
|
log('drain: reconstruction re-touch of %s failed (continuing): %o', member.peerId, error);
|
|
@@ -4307,9 +4543,10 @@ export class CadreNode {
|
|
|
4307
4543
|
await this.deferMembershipGateRefresh('drain-reissue', () => this.reissuePendingPeerWrites());
|
|
4308
4544
|
}
|
|
4309
4545
|
/**
|
|
4310
|
-
* Body of {@link drainPendingPeerWrites}' loop. An entry is cleared
|
|
4311
|
-
*
|
|
4312
|
-
* the
|
|
4546
|
+
* Body of {@link drainPendingPeerWrites}' loop. An entry is cleared on every outcome the
|
|
4547
|
+
* re-touch reports — re-touched, or nothing left to re-issue because the row was removed
|
|
4548
|
+
* since or the peer self-published; the re-touch decides which over the row it reads under
|
|
4549
|
+
* the lock. A failure is logged and the entry stays queued for the next cohort growth.
|
|
4313
4550
|
*/
|
|
4314
4551
|
async reissuePendingPeerWrites() {
|
|
4315
4552
|
for (const peerId of [...this.pendingPeerWrites.keys()]) {
|
|
@@ -4317,13 +4554,8 @@ export class CadreNode {
|
|
|
4317
4554
|
return;
|
|
4318
4555
|
}
|
|
4319
4556
|
try {
|
|
4320
|
-
const
|
|
4321
|
-
|
|
4322
|
-
// Removed since, or peer self-published — nothing for the owner to re-issue.
|
|
4323
|
-
this.pendingPeerWrites.delete(peerId);
|
|
4324
|
-
continue;
|
|
4325
|
-
}
|
|
4326
|
-
await this.reissuePeerAuthorize(peerId, record.updatedAt);
|
|
4557
|
+
const outcome = await this.reissuePeerAuthorize(peerId);
|
|
4558
|
+
log('drain: re-issue of %s (authorize): %s', peerId, outcome);
|
|
4327
4559
|
this.pendingPeerWrites.delete(peerId);
|
|
4328
4560
|
}
|
|
4329
4561
|
catch (error) {
|
|
@@ -4336,57 +4568,196 @@ export class CadreNode {
|
|
|
4336
4568
|
* `UpdatedAt` strictly above the stored value (and the wall clock) and re-sign via
|
|
4337
4569
|
* the owner branch of `CadrePeer.AuthorizedUpdate`. The row already exists
|
|
4338
4570
|
* locally (it committed there, just local-only), so this is an UPDATE, not the
|
|
4339
|
-
* original INSERT.
|
|
4571
|
+
* original INSERT. The bump, and whether the row is still the owner's to re-issue, are
|
|
4572
|
+
* decided over the row as read inside the write ({@link SeedBootstrapService.reauthorizePeer}'s
|
|
4573
|
+
* re-broadcast mode).
|
|
4340
4574
|
*/
|
|
4341
|
-
async reissuePeerAuthorize(peerId
|
|
4575
|
+
async reissuePeerAuthorize(peerId) {
|
|
4342
4576
|
if (!this.seedBootstrapService) {
|
|
4343
|
-
|
|
4577
|
+
throw new Error('Seed bootstrap service not initialized. Call initializeSeedBootstrap() first.');
|
|
4344
4578
|
}
|
|
4345
|
-
|
|
4346
|
-
await this.seedBootstrapService.reauthorizePeer(peerId, updatedAt);
|
|
4579
|
+
return await this.seedBootstrapService.reauthorizePeer(peerId, 'rebroadcast');
|
|
4347
4580
|
}
|
|
4348
4581
|
/**
|
|
4349
|
-
*
|
|
4350
|
-
* write-while-alone re-replication queue (single-flight;
|
|
4351
|
-
* not on
|
|
4352
|
-
*
|
|
4582
|
+
* Re-evaluate {@link hasReachableControlSibling} and act on its edges: on false→true, drain
|
|
4583
|
+
* the write-while-alone re-replication queue (single-flight; best-effort, a drain failure is
|
|
4584
|
+
* logged, not thrown); on true→false, re-arm, so the next reachable sibling drains again.
|
|
4585
|
+
* Called when a peer identifies, when a connection closes, and after every membership refresh,
|
|
4586
|
+
* where a sibling that connected before this node authorized it (the production join order)
|
|
4587
|
+
* becomes reachable. A device counted in for its admission ({@link admittingDevices}) is not
|
|
4588
|
+
* a trigger: the admission's own writes are in flight, and its row landing refreshes
|
|
4589
|
+
* membership.
|
|
4353
4590
|
*/
|
|
4354
|
-
|
|
4591
|
+
updateControlSiblingReachability() {
|
|
4355
4592
|
if (!this._running || !this.controlNode) {
|
|
4356
4593
|
return;
|
|
4357
4594
|
}
|
|
4358
|
-
const
|
|
4359
|
-
if (
|
|
4595
|
+
const reachable = this.hasReachableControlSibling();
|
|
4596
|
+
if (reachable === this.controlSiblingReachable) {
|
|
4360
4597
|
return;
|
|
4361
4598
|
}
|
|
4362
|
-
this.
|
|
4363
|
-
|
|
4364
|
-
|
|
4365
|
-
|
|
4599
|
+
this.controlSiblingReachable = reachable;
|
|
4600
|
+
if (!reachable) {
|
|
4601
|
+
return;
|
|
4602
|
+
}
|
|
4603
|
+
log('Control cohort grew (a sibling is reachable); draining write-while-alone re-replication queue');
|
|
4604
|
+
void this.drainPendingControlReplication('sibling-reachable')
|
|
4605
|
+
.catch((error) => log('Control re-replication drain (sibling-reachable) failed: %o', error));
|
|
4366
4606
|
}
|
|
4367
|
-
/**
|
|
4368
|
-
|
|
4369
|
-
|
|
4607
|
+
/**
|
|
4608
|
+
* A connected peer identified as serving the control database: count it
|
|
4609
|
+
* ({@link identifiedControlPeers}), re-evaluate the growth edge, and run the sibling-open
|
|
4610
|
+
* reactions, once per connected period. libp2p re-emits `peer:identify` on every
|
|
4611
|
+
* identify-push from a connected peer, and those repeats start nothing.
|
|
4612
|
+
*/
|
|
4613
|
+
onControlPeerIdentified(peerId) {
|
|
4614
|
+
const id = peerId.toString();
|
|
4615
|
+
// An identify that reports after its connection closed counts nobody.
|
|
4616
|
+
if (this.identifiedControlPeers.has(id) || !this.controlNode || this.controlNode.getConnections(peerId).length === 0) {
|
|
4370
4617
|
return;
|
|
4371
4618
|
}
|
|
4372
|
-
|
|
4373
|
-
|
|
4619
|
+
this.identifiedControlPeers.add(id);
|
|
4620
|
+
this.updateControlSiblingReachability();
|
|
4621
|
+
this.onControlSiblingConnectionChange(id, 'open');
|
|
4622
|
+
}
|
|
4623
|
+
/**
|
|
4624
|
+
* A control connection closed: `remotePeer` stops counting once it has none left, and the
|
|
4625
|
+
* growth edge re-arms if no reachable sibling remains. The sibling-close reaction runs on every
|
|
4626
|
+
* close. libp2p's connection manager drops the closed connection from `getConnections` before
|
|
4627
|
+
* this listener runs: libp2p dispatches each event on its internal bus, where the connection
|
|
4628
|
+
* manager listens, before re-dispatching it to the node's own listeners.
|
|
4629
|
+
*/
|
|
4630
|
+
onControlConnectionClosed(remotePeer) {
|
|
4631
|
+
const id = remotePeer.toString();
|
|
4632
|
+
if (!this.controlNode || this.controlNode.getConnections(remotePeer).length === 0) {
|
|
4633
|
+
this.identifiedControlPeers.delete(id);
|
|
4374
4634
|
}
|
|
4635
|
+
this.updateControlSiblingReachability();
|
|
4636
|
+
this.onControlSiblingConnectionChange(id, 'close');
|
|
4375
4637
|
}
|
|
4376
4638
|
/**
|
|
4377
|
-
*
|
|
4378
|
-
*
|
|
4379
|
-
*
|
|
4380
|
-
*
|
|
4639
|
+
* Count the peers that identified before {@link wireControlConnectionListeners} ran, whose
|
|
4640
|
+
* `peer:identify` has already fired, from their peer-store records. Updates the growth edge
|
|
4641
|
+
* but runs no sibling-open reaction, as a connection that predates the listeners never did. A
|
|
4642
|
+
* peer not yet in the peer store is still identifying, and its own event counts it.
|
|
4643
|
+
*/
|
|
4644
|
+
async noteIdentifiedControlPeers() {
|
|
4645
|
+
const node = this.controlNode;
|
|
4646
|
+
if (!node) {
|
|
4647
|
+
return;
|
|
4648
|
+
}
|
|
4649
|
+
const connected = new Map();
|
|
4650
|
+
for (const connection of node.getConnections()) {
|
|
4651
|
+
connected.set(connection.remotePeer.toString(), connection.remotePeer);
|
|
4652
|
+
}
|
|
4653
|
+
const partyId = this.config.controlNetwork.partyId;
|
|
4654
|
+
await Promise.all([...connected.values()].map(async (peerId) => {
|
|
4655
|
+
try {
|
|
4656
|
+
const { protocols } = await node.peerStore.get(peerId);
|
|
4657
|
+
// Still wired to this node: a stop while the read was out cleared the set.
|
|
4658
|
+
const wired = this.controlPeerIdentifyHandler !== null && this.controlNode === node;
|
|
4659
|
+
if (wired && servesControlDatabase(partyId, protocols) && node.getConnections(peerId).length > 0) {
|
|
4660
|
+
this.identifiedControlPeers.add(peerId.toString());
|
|
4661
|
+
}
|
|
4662
|
+
}
|
|
4663
|
+
catch (error) {
|
|
4664
|
+
if (error.name !== 'NotFoundError') {
|
|
4665
|
+
log('Peer store read for connected control peer %s failed — left to its identify: %o', peerId.toString(), error);
|
|
4666
|
+
}
|
|
4667
|
+
}
|
|
4668
|
+
}));
|
|
4669
|
+
this.updateControlSiblingReachability();
|
|
4670
|
+
}
|
|
4671
|
+
/**
|
|
4672
|
+
* An AUTHORIZED sibling became reachable (its identify, once per connected period:
|
|
4673
|
+
* {@link onControlPeerIdentified}) or one of its connections closed: start a reconcile
|
|
4674
|
+
* pass. On open the pass asks the new sibling for strand addresses and re-arms the
|
|
4675
|
+
* delegate grants; on close it re-dials. First, on open, drop the membership rows
|
|
4676
|
+
* views: a machine that was offline while a sibling committed learns of the commit
|
|
4677
|
+
* only when it next reads the block from storage, and dropping the views makes the
|
|
4678
|
+
* pass's reads do that now rather than after the views' maximum age. Those reads can
|
|
4679
|
+
* fall inside the read-repair window, so the open also arms a second drop past it
|
|
4680
|
+
* ({@link armMembershipViewReload}). Any other
|
|
4681
|
+
* peer — a relay, a bootstrap node, another party's machine — starts nothing.
|
|
4682
|
+
*
|
|
4683
|
+
* On identify rather than `connection:open` because, until the sibling identifies,
|
|
4684
|
+
* Optimystic's cohort for every control block is this node alone: a pass started at the
|
|
4685
|
+
* open writes (the first self-record publish) and reads on a cohort of one.
|
|
4686
|
+
*/
|
|
4687
|
+
onControlSiblingConnectionChange(remotePeer, change) {
|
|
4688
|
+
if (!this.authorizedControlPeers.has(remotePeer)) {
|
|
4689
|
+
return;
|
|
4690
|
+
}
|
|
4691
|
+
if (change === 'open') {
|
|
4692
|
+
// NOTE: a machine removed from the cadre that has not learned of it keeps redialing
|
|
4693
|
+
// its siblings, each of which admits it provisionally and hangs up at the deadline
|
|
4694
|
+
// (28.5 s at the default link), and every such open drops the views here and once
|
|
4695
|
+
// more past the read-repair window: two storage reloads, with their cohort consults
|
|
4696
|
+
// refused, per cycle until it learns. If a removed machine's churn ever shows up,
|
|
4697
|
+
// skip both drops for a peer whose previous connection the remote closed inside the
|
|
4698
|
+
// provisional deadline.
|
|
4699
|
+
this.controlDatabase?.invalidateMembershipViews();
|
|
4700
|
+
this.armMembershipViewReload();
|
|
4701
|
+
}
|
|
4702
|
+
// NOTE: a close starts a pass at once, and a sibling in its bring-up quiet period lets the
|
|
4703
|
+
// dial resolve and then drops the connection (`createMembershipConnectionGater`'s deny
|
|
4704
|
+
// timing), so a restarting sibling costs one dial-connect-close pass after another for its
|
|
4705
|
+
// whole bring-up (5-6 passes in ~200 ms against a claimed node restarting locally). If a
|
|
4706
|
+
// slow device's bring-up ever makes that churn show up, delay the pass a close starts when
|
|
4707
|
+
// the previous connection to that sibling lived only a few milliseconds.
|
|
4708
|
+
log('Control connection to sibling %s %s; reconcile pass triggered', remotePeer, change);
|
|
4709
|
+
this.controlReconcileRuns?.trigger(0);
|
|
4710
|
+
}
|
|
4711
|
+
/**
|
|
4712
|
+
* Drop the membership views again {@link MEMBERSHIP_VIEW_RECONNECT_RELOAD_MS} after a
|
|
4713
|
+
* sibling connects, and start a pass whose reads reload them. This node's reads before
|
|
4714
|
+
* the connection stamped Optimystic's read-repair window on the control blocks, so the
|
|
4715
|
+
* reload the open itself starts is served from the local copy with no cohort consult,
|
|
4716
|
+
* and the view would keep rows that predate the sibling's commits until its maximum age.
|
|
4717
|
+
* The reload past the window starts the consults; one that brings a newer revision
|
|
4718
|
+
* raises the table change that drops the view and refreshes the gate. Re-armed by every
|
|
4719
|
+
* open, so it runs one window after the latest.
|
|
4720
|
+
*
|
|
4721
|
+
* NOTE: a consult Optimystic defers (16 already in flight) starts nothing and leaves the block
|
|
4722
|
+
* unstamped, so the view keeps its rows until the first read past its maximum age, which on an
|
|
4723
|
+
* idle node is the reconcile backstop. If `cluster-tx:read-repair-deferred` lines show at a
|
|
4724
|
+
* reconnect, re-arm this reload once more instead of waiting for the backstop.
|
|
4725
|
+
*/
|
|
4726
|
+
armMembershipViewReload() {
|
|
4727
|
+
if (this.membershipViewReloadTimer) {
|
|
4728
|
+
clearTimeout(this.membershipViewReloadTimer);
|
|
4729
|
+
}
|
|
4730
|
+
this.membershipViewReloadTimer = setTimeout(() => {
|
|
4731
|
+
this.membershipViewReloadTimer = null;
|
|
4732
|
+
log('Read-repair window passed since a sibling connected; reloading the membership views');
|
|
4733
|
+
this.controlDatabase?.invalidateMembershipViews();
|
|
4734
|
+
this.controlReconcileRuns?.trigger(0);
|
|
4735
|
+
}, MEMBERSHIP_VIEW_RECONNECT_RELOAD_MS);
|
|
4736
|
+
// Node-only: don't keep the event loop alive solely for this timer.
|
|
4737
|
+
this.membershipViewReloadTimer?.unref?.();
|
|
4738
|
+
}
|
|
4739
|
+
/**
|
|
4740
|
+
* Wire the control-connection listeners, which keep {@link identifiedControlPeers}: the
|
|
4741
|
+
* growth edge that drives write-while-alone re-replication, and the sibling reconcile
|
|
4742
|
+
* trigger ({@link onControlSiblingConnectionChange}). Wired in {@link start} (not the
|
|
4743
|
+
* delayed refresh path) so no early identify is missed, and reads back the peers that
|
|
4744
|
+
* identified before it ran ({@link noteIdentifiedControlPeers}); torn down in
|
|
4745
|
+
* {@link stopRecordRefresh}. Idempotent.
|
|
4381
4746
|
*/
|
|
4382
4747
|
wireControlConnectionListeners() {
|
|
4383
|
-
if (!this.controlNode || this.
|
|
4748
|
+
if (!this.controlNode || this.controlPeerIdentifyHandler) {
|
|
4384
4749
|
return;
|
|
4385
4750
|
}
|
|
4386
|
-
|
|
4387
|
-
this.
|
|
4388
|
-
|
|
4751
|
+
const partyId = this.config.controlNetwork.partyId;
|
|
4752
|
+
this.controlPeerIdentifyHandler = (event) => {
|
|
4753
|
+
if (servesControlDatabase(partyId, event.detail.protocols)) {
|
|
4754
|
+
this.onControlPeerIdentified(event.detail.peerId);
|
|
4755
|
+
}
|
|
4756
|
+
};
|
|
4757
|
+
this.controlConnectionCloseHandler = (event) => this.onControlConnectionClosed(event.detail.remotePeer);
|
|
4758
|
+
this.controlNode.addEventListener('peer:identify', this.controlPeerIdentifyHandler);
|
|
4389
4759
|
this.controlNode.addEventListener('connection:close', this.controlConnectionCloseHandler);
|
|
4760
|
+
void this.noteIdentifiedControlPeers();
|
|
4390
4761
|
}
|
|
4391
4762
|
// ============================================================================
|
|
4392
4763
|
// Device-token registry (control-network push-token publish + resolve)
|
|
@@ -4692,6 +5063,7 @@ export class CadreNode {
|
|
|
4692
5063
|
if (sAppId !== undefined) {
|
|
4693
5064
|
this.replicaSAppIds.set(strandId, sAppId);
|
|
4694
5065
|
log('Storage replica %s serves sApp %s', strandId, sAppId);
|
|
5066
|
+
this.strandWatcher?.sAppIdLearned(strandId);
|
|
4695
5067
|
}
|
|
4696
5068
|
}
|
|
4697
5069
|
catch (error) {
|
|
@@ -4800,6 +5172,7 @@ export class CadreNode {
|
|
|
4800
5172
|
this.strandLaunchRefusals.clear();
|
|
4801
5173
|
this.discoveredStrands.clear();
|
|
4802
5174
|
this.replicaSAppIds.clear();
|
|
5175
|
+
this.formedPartyKeys.clear();
|
|
4803
5176
|
// Drop delegate-admission state: the grants are scoped to the session that
|
|
4804
5177
|
// recorded them, so a stop()/start() cycle on this object must not keep
|
|
4805
5178
|
// admitting delegates announced under the previous one, nor carry stale
|
|
@@ -4820,6 +5193,10 @@ export class CadreNode {
|
|
|
4820
5193
|
// Detach the membership hub first: nothing this teardown does should drive a
|
|
4821
5194
|
// gate refresh, and a notification arriving after this point is a no-op.
|
|
4822
5195
|
if (this.controlDatabase) {
|
|
5196
|
+
// The database's registry would dispose the gate watch too; disposing it here, before
|
|
5197
|
+
// the listener goes, keeps a closing commit from scheduling a gate refresh.
|
|
5198
|
+
this.membershipGateWatch?.dispose();
|
|
5199
|
+
this.membershipGateWatch = null;
|
|
4823
5200
|
this.controlDatabase.setMembershipChangeListener(null);
|
|
4824
5201
|
this.controlDatabase.setGuardedDeleteListener(null);
|
|
4825
5202
|
this.controlDatabase.setControlWriteAbandonedListener(null);
|
|
@@ -5157,8 +5534,9 @@ export class CadreNode {
|
|
|
5157
5534
|
* registered, deliberately: both an explicit retry and the {@link StrandWatcher}'s
|
|
5158
5535
|
* automatic relaunch need it. A failed launch here hands the strand back to that
|
|
5159
5536
|
* relaunch — `StrandWatcher.forgetStrand` drops the id from the watcher's
|
|
5160
|
-
* `knownStrands
|
|
5161
|
-
*
|
|
5537
|
+
* `knownStrands`, records the backoff and wakes the watcher when it expires, so the
|
|
5538
|
+
* strand is re-attempted on the same ladder a watcher-driven failure gets
|
|
5539
|
+
* (`min(strandWatchInterval, 5 s) * 2^(failures-1)`, capped at 5 minutes). Because the
|
|
5162
5540
|
* config stays registered, the retry takes `handleStrandAdded`'s auto-launch branch:
|
|
5163
5541
|
* what the app sees is `strand:error` per failed retry and `strand:started` when one
|
|
5164
5542
|
* succeeds, never a second `strand:discovered`. {@link detachStrand} (reached via
|
|
@@ -5196,6 +5574,7 @@ export class CadreNode {
|
|
|
5196
5574
|
// A claim also overrides any earlier `stopStrand` of this id: the watcher must track
|
|
5197
5575
|
// the strand again, or a party-wide removal would never stop it here.
|
|
5198
5576
|
this.strandWatcher?.unsuppressStrand(strandRow.Id);
|
|
5577
|
+
this.strandWatcher?.sAppIdLearned(strandRow.Id);
|
|
5199
5578
|
log('Registered sAppConfig for strand %s (sApp: %s, founder: %s)', strandRow.Id, sAppConfig.id, founder ?? 'derived');
|
|
5200
5579
|
// Before the launch, so a foreign strand whose launch fails, or whose app is killed
|
|
5201
5580
|
// mid-launch, is still offered again: by the watcher's retry and by the next start.
|
|
@@ -5219,7 +5598,7 @@ export class CadreNode {
|
|
|
5219
5598
|
// Hand the strand back to the watcher's retry ladder. Harmless on the founder path
|
|
5220
5599
|
// (`foundStrand` → `publishStrand` + this call), where the watcher may never have
|
|
5221
5600
|
// offered the strand: the `knownStrands` delete is a no-op and the recorded backoff
|
|
5222
|
-
// only delays the watcher's own first attempt by one
|
|
5601
|
+
// only delays the watcher's own first attempt by one ladder step, which is what
|
|
5223
5602
|
// should happen right after an attempt that just failed.
|
|
5224
5603
|
//
|
|
5225
5604
|
// NOTE: the retry relaunches from the CONTROL row (or, for a strand joined from
|
|
@@ -5300,7 +5679,8 @@ export class CadreNode {
|
|
|
5300
5679
|
* (the key exists nowhere else). No app-data wipe is needed for either.
|
|
5301
5680
|
*
|
|
5302
5681
|
* The insert is signed through this node's owner signer ({@link getOwnerSigner}), in one
|
|
5303
|
-
* approval with a closed strand's party
|
|
5682
|
+
* approval and one control transaction with a closed strand's party key
|
|
5683
|
+
* ({@link ControlDatabase.insertFoundingRows}). The signer's key is persisted on the row
|
|
5304
5684
|
* as `FounderOwnerKey` (the schema pins the column to the verified signer), and this node's
|
|
5305
5685
|
* peer id as `FounderPeerId` (bound into the signature), which is what later lets any
|
|
5306
5686
|
* launch derive "this machine is the founder" from the row alone.
|
|
@@ -5311,7 +5691,8 @@ export class CadreNode {
|
|
|
5311
5691
|
*
|
|
5312
5692
|
* A CLOSED strand's publish also seats this party's own membership identity key —
|
|
5313
5693
|
* the `StrandPartyKey` row {@link ensureStrandPartyKey} documents — which is what the
|
|
5314
|
-
* founder bootstrap derives `Member.Key`/`Manager.MemberKey` from
|
|
5694
|
+
* founder bootstrap derives `Member.Key`/`Manager.MemberKey` from; a key the party already
|
|
5695
|
+
* holds for the id is kept. The
|
|
5315
5696
|
* `memberPrivateKey` ARGUMENT stays the strand-wide read secret formation hands to
|
|
5316
5697
|
* joiners; it derives nobody's identity.
|
|
5317
5698
|
*
|
|
@@ -5331,23 +5712,8 @@ export class CadreNode {
|
|
|
5331
5712
|
*/
|
|
5332
5713
|
async publishStrand(strandId, type = 'o', memberPrivateKey, sign) {
|
|
5333
5714
|
const signer = this.requireOwnerSigner(`publish strand ${strandId}`);
|
|
5334
|
-
|
|
5335
|
-
|
|
5336
|
-
const trimmed = requireNonBlank(strandId, 'strand id');
|
|
5337
|
-
// Refuse an unusable id HERE, not only at launch. `addStrand` already asserts, but it
|
|
5338
|
-
// runs after this row is written, and this row replicates: an id no node can turn into
|
|
5339
|
-
// a storage scope key would otherwise reach the whole party's control database and be
|
|
5340
|
-
// declined once per member. Asserted after the trim so the id checked is the id stored.
|
|
5341
|
-
assertStrandScopeKey(trimmed);
|
|
5342
|
-
// FounderPeerId records THIS machine as the founder, which is what later lets a relaunch
|
|
5343
|
-
// derive founder-ness; ControlDatabase.insertStrand writes the same id (its own node's).
|
|
5344
|
-
const desired = {
|
|
5345
|
-
Id: trimmed,
|
|
5346
|
-
Type: type,
|
|
5347
|
-
MemberPrivateKey: memberPrivateKey ?? null,
|
|
5348
|
-
FounderOwnerKey: signer.publicKey,
|
|
5349
|
-
FounderPeerId: this.controlNode.peerId.toString(),
|
|
5350
|
-
};
|
|
5715
|
+
const desired = this.founderStrandRow(strandId, type, memberPrivateKey, signer);
|
|
5716
|
+
const trimmed = desired.Id;
|
|
5351
5717
|
// Read first rather than leaning on the insert's collision for BOTH readings. The
|
|
5352
5718
|
// ordinary resume (an app killed after its own publish committed) then needs no error
|
|
5353
5719
|
// matching at all: the collision path can only recognise itself by the engine's message
|
|
@@ -5367,24 +5733,57 @@ export class CadreNode {
|
|
|
5367
5733
|
log('publishStrand(%s): already published (type %s) with identical content — no-op', trimmed, type);
|
|
5368
5734
|
// A closed strand THIS machine published must hold a party key (its own membership
|
|
5369
5735
|
// identity — see ensureStrandPartyKey). Insert-if-absent, so the ordinary repeat
|
|
5370
|
-
// is a read-only no-op and a
|
|
5371
|
-
//
|
|
5736
|
+
// is a read-only no-op and a closed row stored without its key heals here. A row
|
|
5737
|
+
// another machine published is that machine's to mint for.
|
|
5372
5738
|
if (type === 'c' && this.isSelfFoundedRow(matched)) {
|
|
5373
5739
|
await this.ensureStrandPartyKey(trimmed, undefined, sign);
|
|
5374
5740
|
}
|
|
5375
5741
|
return matched;
|
|
5376
5742
|
}
|
|
5377
|
-
return await
|
|
5743
|
+
return (await this.writeFoundingRows(desired, undefined, signer, sign)).strandRow;
|
|
5744
|
+
}
|
|
5745
|
+
/**
|
|
5746
|
+
* The `Strand` row this machine publishes for `strandId`, as {@link publishStrand} and
|
|
5747
|
+
* {@link foundStrand} write it: the id trimmed and checked, this node's owner key and peer id
|
|
5748
|
+
* as its founder.
|
|
5749
|
+
*/
|
|
5750
|
+
founderStrandRow(strandId, type, memberPrivateKey, signer) {
|
|
5751
|
+
// Trim/reject here so the id that lands matches the one unpublishStrand looks up: it
|
|
5752
|
+
// trims too, and an untrimmed row would be unreachable by the same string.
|
|
5753
|
+
const trimmed = requireNonBlank(strandId, 'strand id');
|
|
5754
|
+
// Refuse an unusable id HERE, not only at launch. `addStrand` already asserts, but it
|
|
5755
|
+
// runs after this row is written, and this row replicates: an id no node can turn into
|
|
5756
|
+
// a storage scope key would otherwise reach the whole party's control database and be
|
|
5757
|
+
// declined once per member. Asserted after the trim so the id checked is the id stored.
|
|
5758
|
+
assertStrandScopeKey(trimmed);
|
|
5759
|
+
// FounderPeerId records THIS machine as the founder, which is what later lets a relaunch
|
|
5760
|
+
// derive founder-ness; ControlDatabase.insertFoundingRows writes the same id (its own node's).
|
|
5761
|
+
return {
|
|
5762
|
+
Id: trimmed,
|
|
5763
|
+
Type: type,
|
|
5764
|
+
MemberPrivateKey: memberPrivateKey ?? null,
|
|
5765
|
+
FounderOwnerKey: signer.publicKey,
|
|
5766
|
+
FounderPeerId: this.controlNode.peerId.toString(),
|
|
5767
|
+
};
|
|
5378
5768
|
}
|
|
5379
5769
|
/**
|
|
5380
|
-
* The write half of {@link publishStrand}
|
|
5381
|
-
*
|
|
5382
|
-
*
|
|
5770
|
+
* The write half of {@link publishStrand} and {@link foundStrand}: under one owner approval,
|
|
5771
|
+
* insert `desired`, a closed strand's party key and the requested invitation in one control
|
|
5772
|
+
* transaction ({@link ControlDatabase.insertFoundingRows}), adopting a matching row a
|
|
5773
|
+
* concurrent founder landed first. Reads nothing first: the caller has just read the
|
|
5774
|
+
* `Strand` row, and a stored key or row is resolved from the insert's own rejection.
|
|
5383
5775
|
*/
|
|
5384
|
-
async
|
|
5385
|
-
const
|
|
5776
|
+
async writeFoundingRows(desired, invite, signer, sign) {
|
|
5777
|
+
const action = invite
|
|
5778
|
+
? `Publish the strand ${desired.Id} to the cadre, with an invitation to the app ${invite.sAppId}`
|
|
5779
|
+
: `Publish the strand ${desired.Id} to the cadre`;
|
|
5780
|
+
return await withinOwnerApproval(signer, { partyId: this.partyId, action }, sign, (approved) => this.insertOrAdoptFounding(desired, invite, signer, approved));
|
|
5781
|
+
}
|
|
5782
|
+
/** {@link writeFoundingRows} inside its approval. */
|
|
5783
|
+
async insertOrAdoptFounding(desired, invite, signer, sign) {
|
|
5784
|
+
let partyKey;
|
|
5386
5785
|
try {
|
|
5387
|
-
await this.
|
|
5786
|
+
partyKey = await this.seatFoundingRows(desired, invite, signer, sign);
|
|
5388
5787
|
}
|
|
5389
5788
|
catch (error) {
|
|
5390
5789
|
// Only the uniqueness collision is a candidate for the idempotent reading; every
|
|
@@ -5392,26 +5791,79 @@ export class CadreNode {
|
|
|
5392
5791
|
if (!isStrandIdConflict(error, 'Strand')) {
|
|
5393
5792
|
throw error;
|
|
5394
5793
|
}
|
|
5395
|
-
const
|
|
5396
|
-
|
|
5397
|
-
|
|
5398
|
-
|
|
5399
|
-
|
|
5794
|
+
const strandRow = await this.adoptConcurrentFounding(desired, error);
|
|
5795
|
+
// The rejection rolled the invitation back with the row; a caller asking for one gets one.
|
|
5796
|
+
if (invite) {
|
|
5797
|
+
await this.publishFoundingInviteIfAbsent(strandRow.Id, invite, sign);
|
|
5798
|
+
}
|
|
5799
|
+
return { strandRow };
|
|
5800
|
+
}
|
|
5801
|
+
log('Published strand %s (type %s) to control DB under owner %s', desired.Id, desired.Type, signer.publicKey);
|
|
5802
|
+
if (invite) {
|
|
5803
|
+
this.registerPublishedInvitation(invite.token, invite.expiresAtMs);
|
|
5804
|
+
}
|
|
5805
|
+
return { strandRow: desired, partyKey };
|
|
5806
|
+
}
|
|
5807
|
+
/**
|
|
5808
|
+
* Insert `desired`'s founding rows, minting a closed strand's party key — this party's own
|
|
5809
|
+
* membership identity, which the founder bootstrap derives `Member`/`Manager` from, distinct
|
|
5810
|
+
* from the row's shared `MemberPrivateKey` read secret. A key already stored for the id with
|
|
5811
|
+
* no strand row (one {@link ensureStrandPartyKey} seated, which it would adopt too) refuses
|
|
5812
|
+
* that transaction whole; the rows are then written again without a key.
|
|
5813
|
+
*
|
|
5814
|
+
* @returns The party key this seated; undefined when it seated none, so the launch reads the stored one.
|
|
5815
|
+
*/
|
|
5816
|
+
async seatFoundingRows(desired, invite, signer, sign) {
|
|
5817
|
+
const controlDatabase = this.controlDatabase;
|
|
5818
|
+
const strand = { id: desired.Id, type: desired.Type, memberPrivateKey: desired.MemberPrivateKey ?? undefined };
|
|
5819
|
+
if (desired.Type !== 'c') {
|
|
5820
|
+
await controlDatabase.insertFoundingRows({ strand, formationInvite: invite }, signer.publicKey, sign);
|
|
5821
|
+
return undefined;
|
|
5822
|
+
}
|
|
5823
|
+
const partyKey = await generateStrandMemberKey();
|
|
5824
|
+
try {
|
|
5825
|
+
await controlDatabase.insertFoundingRows({ strand, partyKey, formationInvite: invite }, signer.publicKey, sign);
|
|
5826
|
+
return partyKey;
|
|
5827
|
+
}
|
|
5828
|
+
catch (error) {
|
|
5829
|
+
if (!isStrandIdConflict(error, 'StrandPartyKey')) {
|
|
5400
5830
|
throw error;
|
|
5401
5831
|
}
|
|
5402
|
-
log('publishStrand(%s):
|
|
5403
|
-
|
|
5832
|
+
log('publishStrand(%s): this party already holds a key for the id; publishing the row and adopting that key', desired.Id);
|
|
5833
|
+
await controlDatabase.insertFoundingRows({ strand, formationInvite: invite }, signer.publicKey, sign);
|
|
5834
|
+
return undefined;
|
|
5404
5835
|
}
|
|
5405
|
-
|
|
5406
|
-
|
|
5407
|
-
|
|
5408
|
-
|
|
5409
|
-
|
|
5410
|
-
|
|
5411
|
-
|
|
5412
|
-
|
|
5836
|
+
}
|
|
5837
|
+
/**
|
|
5838
|
+
* Resolve a founding refused on `Strand.Id`: re-read the row that landed and adopt it when it
|
|
5839
|
+
* matches `desired` — a concurrent founder, such as two machines of one party on one id —
|
|
5840
|
+
* or throw naming the columns that differ.
|
|
5841
|
+
*/
|
|
5842
|
+
async adoptConcurrentFounding(desired, error) {
|
|
5843
|
+
const landed = await this.controlDatabase.queryStrand(desired.Id);
|
|
5844
|
+
if (!landed) {
|
|
5845
|
+
// Collided, yet no row is readable: not the race this branch handles (a row
|
|
5846
|
+
// deleted between the rejection and the re-read, or a stale local view). Surface
|
|
5847
|
+
// the original rather than guessing.
|
|
5848
|
+
throw error;
|
|
5413
5849
|
}
|
|
5414
|
-
|
|
5850
|
+
log('publishStrand(%s): lost a concurrent founding race; re-read the landed row', desired.Id);
|
|
5851
|
+
return requireMatchingStrandRow(landed, desired, 'landed concurrently from another founder');
|
|
5852
|
+
}
|
|
5853
|
+
/**
|
|
5854
|
+
* Publish a founding's invitation on its own, bound to `strandId`, for a founding that adopted
|
|
5855
|
+
* a stored row rather than writing its own — unless the invitation is already stored, bound to
|
|
5856
|
+
* this strand, as a repeat of a founding whose transaction committed leaves it.
|
|
5857
|
+
*
|
|
5858
|
+
* @param sign - The open approval's `sign`; omitted, the publish asks for its own.
|
|
5859
|
+
*/
|
|
5860
|
+
async publishFoundingInviteIfAbsent(strandId, invite, sign) {
|
|
5861
|
+
if ((await this.controlDatabase.queryFormationInvite(invite.token))?.strandId === strandId) {
|
|
5862
|
+
log('foundStrand(%s): the invitation is already published', strandId);
|
|
5863
|
+
return;
|
|
5864
|
+
}
|
|
5865
|
+
const { token, sAppId, ...options } = invite;
|
|
5866
|
+
await this.publishFormationInvite(token, sAppId, { ...options, strandId }, sign);
|
|
5415
5867
|
}
|
|
5416
5868
|
async foundStrand(config, options) {
|
|
5417
5869
|
const tTotal = performance.now();
|
|
@@ -5438,17 +5890,18 @@ export class CadreNode {
|
|
|
5438
5890
|
* so both forms of `foundStrand` reject on it before returning.
|
|
5439
5891
|
*/
|
|
5440
5892
|
async publishFoundingRow(config) {
|
|
5441
|
-
const { strandId, type = 'o', memberPrivateKey } = config;
|
|
5893
|
+
const { strandId, type = 'o', memberPrivateKey, formationInvite, sAppConfig } = config;
|
|
5442
5894
|
if (!this._running || !this.controlDatabase) {
|
|
5443
5895
|
throw new Error(`CadreNode must be started before attempting to found strand ${strandId}`);
|
|
5444
5896
|
}
|
|
5445
5897
|
const controlDatabase = this.controlDatabase;
|
|
5446
5898
|
const trimmed = requireNonBlank(strandId, 'strand id');
|
|
5899
|
+
const invite = formationInvite && { ...formationInvite, sAppId: sAppConfig.id };
|
|
5447
5900
|
const timed = (step, op) => timedStep('foundStrand', trimmed, step, op);
|
|
5448
5901
|
const published = await timed('queryStrand', () => controlDatabase.queryStrand(trimmed));
|
|
5449
|
-
const strandRow = published
|
|
5450
|
-
? this.
|
|
5451
|
-
: await timed('publishStrand', () => this.
|
|
5902
|
+
const { strandRow, partyKey } = published
|
|
5903
|
+
? await this.resumeFounding(published, type, memberPrivateKey, invite)
|
|
5904
|
+
: await timed('publishStrand', () => this.publishFoundingStrand(trimmed, type, memberPrivateKey, invite));
|
|
5452
5905
|
// Derived, not hardcoded `true`: adopting a row another machine published means that
|
|
5453
5906
|
// machine runs the bootstrap — this one must attach or it would write a second Header.
|
|
5454
5907
|
const founded = this.isSelfFoundedRow(strandRow);
|
|
@@ -5457,15 +5910,34 @@ export class CadreNode {
|
|
|
5457
5910
|
'(FounderPeerId is not this node\'s peer id) — attaching as a joiner; ' +
|
|
5458
5911
|
'the founder bootstrap runs on that machine', trimmed);
|
|
5459
5912
|
}
|
|
5460
|
-
return { strandRow, founded };
|
|
5913
|
+
return { strandRow, founded, partyKey };
|
|
5914
|
+
}
|
|
5915
|
+
/** {@link publishFoundingRow} when no row is stored: write the founding rows under this node's owner. */
|
|
5916
|
+
publishFoundingStrand(strandId, type, memberPrivateKey, invite) {
|
|
5917
|
+
const signer = this.requireOwnerSigner(`publish strand ${strandId}`);
|
|
5918
|
+
return this.writeFoundingRows(this.founderStrandRow(strandId, type, memberPrivateKey, signer), invite, signer);
|
|
5919
|
+
}
|
|
5920
|
+
/**
|
|
5921
|
+
* {@link publishFoundingRow} over a stored row: adopt it ({@link adoptPublishedStrand}), then
|
|
5922
|
+
* publish the requested invitation on its own, under its own approval.
|
|
5923
|
+
*/
|
|
5924
|
+
async resumeFounding(published, type, memberPrivateKey, invite) {
|
|
5925
|
+
const strandRow = this.adoptPublishedStrand(published, type, memberPrivateKey);
|
|
5926
|
+
if (invite) {
|
|
5927
|
+
await timedStep('foundStrand', strandRow.Id, 'publishFormationInvite', () => this.publishFoundingInviteIfAbsent(strandRow.Id, invite));
|
|
5928
|
+
}
|
|
5929
|
+
return { strandRow };
|
|
5461
5930
|
}
|
|
5462
5931
|
/**
|
|
5463
5932
|
* The start half of {@link foundStrand}. Its timing lines (`addStrand: start`, then the
|
|
5464
5933
|
* duration or `failed after`) are logged when the start settles, also in the background
|
|
5465
5934
|
* form, so a start that hangs still shows as a start line with no end line.
|
|
5935
|
+
*
|
|
5936
|
+
* A party key the publish half just seated is handed to the launch, which would otherwise
|
|
5937
|
+
* read back the row the same founding wrote.
|
|
5466
5938
|
*/
|
|
5467
|
-
attachFoundingRow({ strandRow, founded }, { sAppConfig, awaitFirstSync }) {
|
|
5468
|
-
return timedStep('foundStrand', strandRow.Id, 'addStrand', () => this.addStrand({ strandRow, sAppConfig, founder: founded, awaitFirstSync }));
|
|
5939
|
+
attachFoundingRow({ strandRow, founded, partyKey }, { sAppConfig, awaitFirstSync }) {
|
|
5940
|
+
return timedStep('foundStrand', strandRow.Id, 'addStrand', () => this.addStrand({ strandRow, sAppConfig, founder: founded, awaitFirstSync, partyMemberPrivateKey: partyKey }));
|
|
5469
5941
|
}
|
|
5470
5942
|
/**
|
|
5471
5943
|
* Resume onto an already-published `Strand` row: keep its stored content, reject a type
|
|
@@ -5494,9 +5966,10 @@ export class CadreNode {
|
|
|
5494
5966
|
* inverse of {@link publishStrand}.
|
|
5495
5967
|
*
|
|
5496
5968
|
* Party-wide, unlike {@link stopStrand}, which only stops the strand on THIS node: every
|
|
5497
|
-
* cadre node watching the table
|
|
5498
|
-
*
|
|
5499
|
-
*
|
|
5969
|
+
* cadre node watching the table hears the row's removal and stops its own instance (a
|
|
5970
|
+
* machine outside the `Strand` block's cohort, on a cadre larger than 16, only at its
|
|
5971
|
+
* watcher's backstop read, default 5 minutes). This node converges immediately — the
|
|
5972
|
+
* method forces a watcher run and stops any still-running local instance before resolving. Removing the row
|
|
5500
5973
|
* removes OUR party's participation only: other parties in the strand keep their own
|
|
5501
5974
|
* rows and the strand network carries on, so no cross-party sign-off is involved — this
|
|
5502
5975
|
* is an owner-signed control-plane write like every other, not "destroy the network".
|
|
@@ -5582,16 +6055,16 @@ export class CadreNode {
|
|
|
5582
6055
|
// deadline rather than dropping it (a binding cleared late is still worth more than
|
|
5583
6056
|
// one never cleared).
|
|
5584
6057
|
await this.strandManager.clearOwnMemberPeerBinding(trimmed);
|
|
5585
|
-
// Converge locally
|
|
5586
|
-
// onStrandRemoved for a strand it tracked; the explicit teardown below
|
|
5587
|
-
// whose strandFilter never admitted this strand (the watcher never knew
|
|
5588
|
-
// never fire). StrandInstanceManager.stopStrand no-ops on an unknown id, so the two
|
|
6058
|
+
// Converge locally before resolving rather than when the write wakes the watcher. The
|
|
6059
|
+
// watcher fires onStrandRemoved for a strand it tracked; the explicit teardown below
|
|
6060
|
+
// covers a node whose strandFilter never admitted this strand (the watcher never knew
|
|
6061
|
+
// it, so it will never fire). StrandInstanceManager.stopStrand no-ops on an unknown id, so the two
|
|
5589
6062
|
// paths cannot double-stop into an error.
|
|
5590
6063
|
//
|
|
5591
6064
|
// detachStrand, NOT stopStrand: this is a party-wide removal, not a local abandonment
|
|
5592
6065
|
// of a strand that still exists, so it must not SUPPRESS the id in the watcher. The
|
|
5593
6066
|
// row is already gone — there is nothing to suppress — and an id re-published before
|
|
5594
|
-
// the next
|
|
6067
|
+
// the next run's suppression cleanup would then never be offered again this session.
|
|
5595
6068
|
// (Same reasoning as handleStrandRemoved, which also avoids stopStrand's `_running`
|
|
5596
6069
|
// guard; here the guard is already satisfied by requireOwnerSigner above.)
|
|
5597
6070
|
await this.strandWatcher?.forcePoll();
|
|
@@ -5614,9 +6087,11 @@ export class CadreNode {
|
|
|
5614
6087
|
* as `MemberPrivateKey`; docs/strands.md → "Closed-Strand Member Key Handling") and is
|
|
5615
6088
|
* never put on the formation wire.
|
|
5616
6089
|
*
|
|
5617
|
-
*
|
|
5618
|
-
*
|
|
5619
|
-
*
|
|
6090
|
+
* A founding seats the key in the same transaction as the strand row, not through here.
|
|
6091
|
+
* Called internally by {@link publishStrand}'s repeat of a published closed strand and by a
|
|
6092
|
+
* founder launch that finds no row, which heal a closed strand row stored without its key
|
|
6093
|
+
* (one seated by `ControlDatabase.insertStrand` directly, or whose key was removed); public
|
|
6094
|
+
* so a test harness — or a joiner flow that persists a formation-issued identity — can seat
|
|
5620
6095
|
* a specific key deliberately.
|
|
5621
6096
|
*
|
|
5622
6097
|
* @param strandId - The strand the key is this party's identity for.
|
|
@@ -5664,8 +6139,8 @@ export class CadreNode {
|
|
|
5664
6139
|
* Resolve the party membership key a closed strand's launch threads into the founder
|
|
5665
6140
|
* bootstrap: the explicit attach-time key when given, else the party's persisted
|
|
5666
6141
|
* `StrandPartyKey` row, else — on the one machine the row names as founder (`FounderPeerId`)
|
|
5667
|
-
* — a freshly minted-and-persisted key, which seats the identity for a
|
|
5668
|
-
*
|
|
6142
|
+
* — a freshly minted-and-persisted key, which seats the identity for a closed row stored
|
|
6143
|
+
* without one (see {@link ensureStrandPartyKey}). Everyone else resolves undefined: non-founding
|
|
5669
6144
|
* machines never mint (no mint race between a party's machines), and only a founder
|
|
5670
6145
|
* bootstrap needs the key at all.
|
|
5671
6146
|
*
|
|
@@ -5685,6 +6160,11 @@ export class CadreNode {
|
|
|
5685
6160
|
if (explicitKey !== undefined) {
|
|
5686
6161
|
return explicitKey;
|
|
5687
6162
|
}
|
|
6163
|
+
const formed = this.formedPartyKeys.get(strand.Id);
|
|
6164
|
+
if (formed !== undefined) {
|
|
6165
|
+
this.formedPartyKeys.delete(strand.Id);
|
|
6166
|
+
return formed;
|
|
6167
|
+
}
|
|
5688
6168
|
if (!this.controlDatabase) {
|
|
5689
6169
|
return undefined;
|
|
5690
6170
|
}
|
|
@@ -5708,8 +6188,8 @@ export class CadreNode {
|
|
|
5708
6188
|
* Counterpart to {@link createOpenInvitation}, which only mints the
|
|
5709
6189
|
* out-of-band {@link OpenInvitation} envelope: persisting the matching
|
|
5710
6190
|
* `FormationInvite` row is what makes the token *redeemable* — the host's
|
|
5711
|
-
* {@link ControlFormationUsageRecorder}
|
|
5712
|
-
*
|
|
6191
|
+
* {@link ControlFormationUsageRecorder} reads it (`readToken`) to check a
|
|
6192
|
+
* redemption. A host minting a closed-strand invite does both (mint +
|
|
5713
6193
|
* publish), exactly as the integration harness's `createInvitation` does.
|
|
5714
6194
|
*
|
|
5715
6195
|
* Signs through the same owner signer as {@link publishStrand}, whose key must be an
|
|
@@ -5726,12 +6206,19 @@ export class CadreNode {
|
|
|
5726
6206
|
async publishFormationInvite(token, sAppId, options = {}, sign) {
|
|
5727
6207
|
const signer = this.requireOwnerSigner(`publish formation invite ${token}`);
|
|
5728
6208
|
await withinOwnerApproval(signer, { partyId: this.partyId, action: `Publish an invitation to the app ${sAppId}` }, sign, (approved) => this.controlDatabase.insertFormationInvite(token, sAppId, signer.publicKey, approved, options));
|
|
5729
|
-
|
|
5730
|
-
// it locally opens the connection gate's formation exemption immediately
|
|
5731
|
-
// rather than waiting for the durable row to become readable.
|
|
5732
|
-
this.strandSolicitationService?.registerMintedInvitation(token, options.expiresAtMs ?? Number.POSITIVE_INFINITY);
|
|
6209
|
+
this.registerPublishedInvitation(token, options.expiresAtMs);
|
|
5733
6210
|
log('Published formation invite %s (sApp %s) under owner %s', token, sAppId, signer.publicKey);
|
|
5734
6211
|
}
|
|
6212
|
+
/**
|
|
6213
|
+
* Register an invitation this node just published with the strand solicitation service. A
|
|
6214
|
+
* host may publish one whose token was minted elsewhere; registering it locally opens the
|
|
6215
|
+
* connection gate's formation exemption immediately rather than waiting for the durable row
|
|
6216
|
+
* to become readable. A token {@link createOpenInvitation} minted here is already registered,
|
|
6217
|
+
* and this records the same expiry again.
|
|
6218
|
+
*/
|
|
6219
|
+
registerPublishedInvitation(token, expiresAtMs) {
|
|
6220
|
+
this.strandSolicitationService?.registerMintedInvitation(token, expiresAtMs ?? Number.POSITIVE_INFINITY);
|
|
6221
|
+
}
|
|
5735
6222
|
/**
|
|
5736
6223
|
* Enroll an approver public key allowed to sign off on `ValidationUrl` redemptions.
|
|
5737
6224
|
*
|
|
@@ -5839,7 +6326,7 @@ export class CadreNode {
|
|
|
5839
6326
|
* fan-out, so a rediscovery costs nothing beyond the map lookup plus the (pure)
|
|
5840
6327
|
* founder derivation. NOT a silent no-op any more when the launch resolves as a
|
|
5841
6328
|
* FOUNDER: the tracked instance may have been launched first as a joiner (an app
|
|
5842
|
-
* attach, or this node's own watcher
|
|
6329
|
+
* attach, or this node's own watcher run winning the `resolveCohortSeed` window
|
|
5843
6330
|
* below), which used to drop the founder request and leave the strand headerless —
|
|
5844
6331
|
* now the tracked instance is founded in place
|
|
5845
6332
|
* ({@link StrandInstanceManager.foundExistingStrand}), waking it first if quiesced
|
|
@@ -5851,9 +6338,9 @@ export class CadreNode {
|
|
|
5851
6338
|
*
|
|
5852
6339
|
* An absent `sAppConfig` launches a storage replica ({@link CadreNodeConfig.hostUnclaimedStrands}),
|
|
5853
6340
|
* which is ALWAYS a joiner whatever the row says: the founder bootstrap writes the sApp
|
|
5854
|
-
* into `Strand.Header`, and a replica has none. So a self-founded row whose watcher
|
|
6341
|
+
* into `Strand.Header`, and a replica has none. So a self-founded row whose watcher run
|
|
5855
6342
|
* wins the race against its app's `addStrand` after a restart comes up as a replica — and
|
|
5856
|
-
* so does a `foundStrand` whose publish a watcher
|
|
6343
|
+
* so does a `foundStrand` whose publish a watcher run saw before its attach. A claim
|
|
5857
6344
|
* that finds a tracked replica upgrades it in place, over the same node and store
|
|
5858
6345
|
* ({@link StrandInstanceManager.attachSApp}), BEFORE any founding: the founder bootstrap
|
|
5859
6346
|
* needs the sApp the attach supplies.
|
|
@@ -5885,8 +6372,8 @@ export class CadreNode {
|
|
|
5885
6372
|
}
|
|
5886
6373
|
// A closed strand's launch carries the party's OWN membership identity key: the
|
|
5887
6374
|
// explicit attach-time key, else the control-layer StrandPartyKey row — minted here
|
|
5888
|
-
// when this machine is the row-derived founder and no row exists yet (a
|
|
5889
|
-
//
|
|
6375
|
+
// when this machine is the row-derived founder and no row exists yet (a closed row
|
|
6376
|
+
// stored without its key). A joiner with no persisted key threads undefined;
|
|
5890
6377
|
// only a FOUNDER bootstrap needs the key, and that path throws loudly without one
|
|
5891
6378
|
// (StrandDatabase).
|
|
5892
6379
|
const partyMemberPrivateKey = strand.Type === 'c'
|
|
@@ -6208,7 +6695,7 @@ export class CadreNode {
|
|
|
6208
6695
|
*
|
|
6209
6696
|
* Timestamps are recorded OPTIMISTICALLY, whatever `collectStrandAddrs` reports
|
|
6210
6697
|
* per peer: recording only on success would re-announce to an unreachable relay
|
|
6211
|
-
* on every reconcile
|
|
6698
|
+
* on every reconcile pass, one dial timeout each. A failed INITIAL announce costs
|
|
6212
6699
|
* the strand supervisor its first attempt only (the relay denies the
|
|
6213
6700
|
* reservation; every re-drive re-announces first through
|
|
6214
6701
|
* {@link announceDelegateToRelay}); a failed REFRESH retries within
|
|
@@ -6331,9 +6818,11 @@ export class CadreNode {
|
|
|
6331
6818
|
*
|
|
6332
6819
|
* Due times are per (sibling, strand) and set from each sibling's own outcome, so
|
|
6333
6820
|
* a sibling that connects late — a phone joining after the party's always-on
|
|
6334
|
-
* machines — is asked on the
|
|
6335
|
-
* (its view of the membership may not include us yet) is retried
|
|
6336
|
-
* {@link STRAND_PEER_ADDR_RETRY_MS} rather than
|
|
6821
|
+
* machines — is asked on the pass its connection starts, and one that could not
|
|
6822
|
+
* answer or refused (its view of the membership may not include us yet) is retried
|
|
6823
|
+
* within {@link STRAND_PEER_ADDR_RETRY_MS} rather than
|
|
6824
|
+
* {@link STRAND_PEER_ADDR_REFRESH_MS}; the pass that stamped the retry arms a wake
|
|
6825
|
+
* for it ({@link armReconcileWake}).
|
|
6337
6826
|
*
|
|
6338
6827
|
* Distinct from {@link refreshDelegateGrants}, deliberately: that pass covers
|
|
6339
6828
|
* RELAYS on a `DELEGATE_GRANT_TTL_MS / 2` throttle to keep circuit-relay
|
|
@@ -6382,15 +6871,17 @@ export class CadreNode {
|
|
|
6382
6871
|
*
|
|
6383
6872
|
* `connectedSiblingTargets`' membership read is unbounded, and with zero
|
|
6384
6873
|
* connections its answer is empty whatever the table holds, so decide it from the
|
|
6385
|
-
* connection list instead of paying for the read once a
|
|
6874
|
+
* connection list instead of paying for the read once a pass.
|
|
6386
6875
|
*
|
|
6387
|
-
* NOTE: otherwise the read runs on EVERY
|
|
6876
|
+
* NOTE: otherwise the read runs on EVERY pass with a running strand, not only when
|
|
6388
6877
|
* some (sibling, strand) is due: pruning a departed sibling needs the current
|
|
6389
6878
|
* target set, and whether anyone is due cannot be told without it (a connected
|
|
6390
|
-
* non-member never gets a due time to compare). One more `CadrePeer` read per
|
|
6391
|
-
*
|
|
6392
|
-
*
|
|
6393
|
-
*
|
|
6879
|
+
* non-member never gets a due time to compare). One more `CadrePeer` read per
|
|
6880
|
+
* pass, on top of the two `runReconcileControlCohort` already makes, all served
|
|
6881
|
+
* from the membership rows view: an in-memory scan while the view is current,
|
|
6882
|
+
* storage only after the table changed or the view's maximum age passed. Sharing
|
|
6883
|
+
* one row set across the pass would save no storage read and was declined (see
|
|
6884
|
+
* the NOTE there).
|
|
6394
6885
|
*
|
|
6395
6886
|
* NOTE: one strand-addr RPC per (running strand × due sibling) — each a tiny
|
|
6396
6887
|
* request/response on an already-open control connection. If a node ever runs
|
|
@@ -6473,8 +6964,8 @@ export class CadreNode {
|
|
|
6473
6964
|
*
|
|
6474
6965
|
* NOTE: every pass serialises the whole FRET table (`exportTable()`), verifies one
|
|
6475
6966
|
* signature per record and makes one `peerStore.merge` per peer, per running strand
|
|
6476
|
-
* per
|
|
6477
|
-
* nothing in libp2p's persistent peer store. If many strands or large tables make it
|
|
6967
|
+
* per pass — every change-driven pass and the 2-minute backstop. Fine at strand table
|
|
6968
|
+
* sizes, and an unchanged address set writes nothing in libp2p's persistent peer store. If many strands or large tables make it
|
|
6478
6969
|
* show up, read only the entries whose address stamp is near the one-hour mark.
|
|
6479
6970
|
*
|
|
6480
6971
|
* @returns the ids of the peers the table holds a usable record for.
|
|
@@ -6757,8 +7248,8 @@ export class CadreNode {
|
|
|
6757
7248
|
* The stop + emit are skipped when the strand manager holds no instance for `strandId` —
|
|
6758
7249
|
* e.g. a party owner that published a strand's row but never ran it locally, or an
|
|
6759
7250
|
* explicit {@link stopStrand} for an id this node never started. `hibernationManager`
|
|
6760
|
-
* untrack and the `sAppConfigs` / {@link strandLaunchRefusals} / {@link discoveredStrands}
|
|
6761
|
-
* deletes stay unconditional (all no-ops when there is nothing to remove), so a launch
|
|
7251
|
+
* untrack and the `sAppConfigs` / {@link strandLaunchRefusals} / {@link discoveredStrands} /
|
|
7252
|
+
* {@link formedPartyKeys} deletes stay unconditional (all no-ops when there is nothing to remove), so a launch
|
|
6762
7253
|
* that failed before an instance was ever tracked still gets its stray entries cleared.
|
|
6763
7254
|
*
|
|
6764
7255
|
* Dropping the {@link discoveredStrands} entry here covers both callers, and both
|
|
@@ -6775,6 +7266,7 @@ export class CadreNode {
|
|
|
6775
7266
|
this.sAppConfigs.delete(strandId);
|
|
6776
7267
|
this.strandLaunchRefusals.delete(strandId);
|
|
6777
7268
|
this.discoveredStrands.delete(strandId);
|
|
7269
|
+
this.formedPartyKeys.delete(strandId);
|
|
6778
7270
|
if (!this.strandManager.hasStrand(strandId)) {
|
|
6779
7271
|
log('detachStrand: strand %s not tracked locally — no-op (no stop, no strand:stopped)', strandId);
|
|
6780
7272
|
return;
|
|
@@ -7141,20 +7633,35 @@ export class CadreNode {
|
|
|
7141
7633
|
return this.controlDatabase;
|
|
7142
7634
|
}
|
|
7143
7635
|
/**
|
|
7144
|
-
* Open control-network connections right now
|
|
7145
|
-
*
|
|
7146
|
-
*
|
|
7147
|
-
*
|
|
7148
|
-
* It approximates the precise signal (the block's cluster size), and a caller that
|
|
7149
|
-
* samples it AFTER a write samples a slightly wider window than `committedAlone`
|
|
7150
|
-
* does inside the write itself. It exists so an embedder can warn an owner before a
|
|
7151
|
-
* control write that this machine currently sees none of its siblings, and report
|
|
7152
|
-
* after one that the write may not have travelled — a warning shown unconditionally
|
|
7153
|
-
* is a warning people learn to ignore.
|
|
7636
|
+
* Open control-network connections right now, of every kind: siblings, relays, bootstrap
|
|
7637
|
+
* nodes and peers still identifying. A report of connectivity, not of replication reach: a
|
|
7638
|
+
* control write can commit local-only with connections open, and whether one would is
|
|
7639
|
+
* {@link hasReachableControlSibling}.
|
|
7154
7640
|
*/
|
|
7155
7641
|
getControlConnectionCount() {
|
|
7156
7642
|
return this.controlNode?.getConnections().length ?? 0;
|
|
7157
7643
|
}
|
|
7644
|
+
/**
|
|
7645
|
+
* Whether a control write made now reaches another machine: a connected peer has identified
|
|
7646
|
+
* as serving the control database and is one this node sends control writes to (an
|
|
7647
|
+
* authorized member, or a device whose admission it is writing). Optimystic counts a peer
|
|
7648
|
+
* into a control block's cohort on those same terms; with none, a write commits on this node
|
|
7649
|
+
* alone and is queued for re-issue once one is reachable (docs/architecture.md → "Writes made
|
|
7650
|
+
* while alone").
|
|
7651
|
+
*
|
|
7652
|
+
* It exists so an embedder can warn an owner before a control write that this machine
|
|
7653
|
+
* currently reaches none of its siblings, and report after one that the write may not have
|
|
7654
|
+
* travelled; a warning shown unconditionally is a warning people learn to ignore. A caller
|
|
7655
|
+
* that samples it AFTER a write samples a slightly wider window than the write did.
|
|
7656
|
+
*/
|
|
7657
|
+
hasReachableControlSibling() {
|
|
7658
|
+
for (const peerId of this.identifiedControlPeers) {
|
|
7659
|
+
if (this.isControlCohortMember(peerId)) {
|
|
7660
|
+
return true;
|
|
7661
|
+
}
|
|
7662
|
+
}
|
|
7663
|
+
return false;
|
|
7664
|
+
}
|
|
7158
7665
|
/**
|
|
7159
7666
|
* Force a poll of the strand watcher (for testing)
|
|
7160
7667
|
*/
|
|
@@ -7231,7 +7738,8 @@ export class CadreNode {
|
|
|
7231
7738
|
}), this.controlNode, this.controlDatabase);
|
|
7232
7739
|
log('Owner signer wired (owner key %s)', signer.publicKey);
|
|
7233
7740
|
// Embedders wire the owner key after start(), so the runner's first pass usually found
|
|
7234
|
-
// this machine not yet an owner;
|
|
7741
|
+
// this machine not yet an owner; wiring a signer writes no table it watches, so without
|
|
7742
|
+
// this it would wait for its backstop.
|
|
7235
7743
|
this.pendingJoinRunner?.kick();
|
|
7236
7744
|
}
|
|
7237
7745
|
/**
|
|
@@ -7455,6 +7963,10 @@ export class CadreNode {
|
|
|
7455
7963
|
if (usage === undefined || invite === undefined) {
|
|
7456
7964
|
return false;
|
|
7457
7965
|
}
|
|
7966
|
+
// NOTE: accepted tradeoff — three ed25519 verifies per invitation-admitted row on every
|
|
7967
|
+
// refresh, CPU only since the rows come from the membership views; a memo of verdicts would
|
|
7968
|
+
// also have to follow the trusted-owner anchor, which syncOwnerAnchor rewrites. Revisit if
|
|
7969
|
+
// the gate's admission deadline ever shows it.
|
|
7458
7970
|
return verifyInvitationAdmission(this.partyId, row, usage, invite, key => this.trustedOwnerStore?.has(key) ?? false);
|
|
7459
7971
|
}
|
|
7460
7972
|
/**
|
|
@@ -7477,7 +7989,7 @@ export class CadreNode {
|
|
|
7477
7989
|
* builds its snapshot against that anchor, which is also what its previous snapshot was
|
|
7478
7990
|
* judged against. Caught here rather than failing the refresh, because the membership read
|
|
7479
7991
|
* tolerates an unreachable `Revocation` block on an isolated node
|
|
7480
|
-
* (`ControlDatabase.
|
|
7992
|
+
* (`ControlDatabase.readRevocationRowsOrNone`) and `queryRevocations` does not: failing the
|
|
7481
7993
|
* whole refresh on that would freeze such a node's gate snapshot.
|
|
7482
7994
|
* NOTE: on such a node the sync is skipped at every refresh until the block is held; if an
|
|
7483
7995
|
* isolated node ever has to derive owner additions while cut off, give `queryRevocations`
|
|
@@ -7486,9 +7998,12 @@ export class CadreNode {
|
|
|
7486
7998
|
* NOTE: cost per membership refresh is one `OwnerKey` read and one `Revocation` read (the
|
|
7487
7999
|
* tombstone list, whose `OwnerKey` stamps are handed to `queryOwnerKeyRows` so it does not
|
|
7488
8000
|
* read the retired set again, and which the hang-up of removed peers shares through
|
|
7489
|
-
* `loadTombstones`), plus the owner-grant chain when some owner row is
|
|
7490
|
-
*
|
|
7491
|
-
*
|
|
8001
|
+
* `loadTombstones`), plus the owner-grant chain's two reads when some owner row is
|
|
8002
|
+
* grant-seated. All are served from the control database's membership views, so they reach
|
|
8003
|
+
* storage only after their table changed or the view's maximum age passed
|
|
8004
|
+
* (`control-founding-consult-budget.spec.ts` pins the counts). On a party founded before
|
|
8005
|
+
* genesis created every control collection, whose `Revocation` table has never been
|
|
8006
|
+
* written, each `Revocation` load is a cohort consult until the first tombstone.
|
|
7492
8007
|
*
|
|
7493
8008
|
* @param retry - forwarded to every read; `false` on the refresh path, for the reason on
|
|
7494
8009
|
* {@link refreshAuthorizedControlPeers}.
|
|
@@ -7509,9 +8024,6 @@ export class CadreNode {
|
|
|
7509
8024
|
const tombstones = await loadTombstones();
|
|
7510
8025
|
const retired = new Set(tombstones.filter(tombstone => tombstone.tableName === 'OwnerKey').map(tombstone => tombstone.stampId));
|
|
7511
8026
|
const rows = await controlDatabase.queryOwnerKeyRows(retry, retired);
|
|
7512
|
-
// NOTE: the grant chain is two whole-table reads per membership refresh while any owner
|
|
7513
|
-
// row is grant-seated, over tables that are append-only; the memo suggested on
|
|
7514
|
-
// loadInvitationChain would cover it too.
|
|
7515
8027
|
const grantChain = rows.some(isConsentSeated) ? await this.loadOwnerGrantChain(controlDatabase, retry) : null;
|
|
7516
8028
|
const derivation = deriveOwnerAnchor({ partyId: this.partyId, anchor: before, rows, tombstones, grantChain });
|
|
7517
8029
|
if (derivation.refusedRemovals.length > 0) {
|
|
@@ -7569,15 +8081,8 @@ export class CadreNode {
|
|
|
7569
8081
|
/**
|
|
7570
8082
|
* The usage and invitation rows {@link hasAnchoredProof} resolves an invitation-admitted
|
|
7571
8083
|
* `CadrePeer` row through, keyed for lookup. Both reads honour `retry` for
|
|
7572
|
-
* {@link listAuthorizedMembers}' reason.
|
|
7573
|
-
*
|
|
7574
|
-
* NOTE: once invitations are the only admission route (ticket
|
|
7575
|
-
* `cadre-invitations-redeemable-by-any-member`), every cadre with a member not yet
|
|
7576
|
-
* re-vouched pays these two whole-table reads and three ed25519 verifies per row on every
|
|
7577
|
-
* membership refresh. Both tables are append-only and a verified chain never changes (expiry,
|
|
7578
|
-
* use count and withdrawal are not re-checked), so if the gate's admission deadline ever
|
|
7579
|
-
* shows this, memoise verified (row stamp → usage stamp) pairs per node and skip the reads
|
|
7580
|
-
* when every admitted row is already memoised.
|
|
8084
|
+
* {@link listAuthorizedMembers}' reason. The rows come from the membership views, so they
|
|
8085
|
+
* reach storage only after the table changed or the view's maximum age passed.
|
|
7581
8086
|
*/
|
|
7582
8087
|
async loadInvitationChain(controlDatabase, retry) {
|
|
7583
8088
|
const [usages, invites] = await Promise.all([
|
|
@@ -7731,16 +8236,16 @@ export class CadreNode {
|
|
|
7731
8236
|
* write-driven refresh never fires for it — yet applying it can ANCHOR a new
|
|
7732
8237
|
* owner key, which flips rows already present from unauthorized to authorized.
|
|
7733
8238
|
* Hence the explicit refresh here; without it a peer the freshly anchored owner
|
|
7734
|
-
* vouched for stays denied until the next
|
|
8239
|
+
* vouched for stays denied until the next membership change or the backstop pass.
|
|
7735
8240
|
*/
|
|
7736
8241
|
seedEventCallbacks() {
|
|
7737
8242
|
return {
|
|
7738
8243
|
onSeedReceived: (partyId, peerId) => this.emit('seed:received', { partyId, peerId }),
|
|
7739
8244
|
onSeedApplied: (partyId, peersAdded, seed) => {
|
|
7740
|
-
this.emit('seed:applied', { partyId, peersAdded });
|
|
7741
8245
|
// A wire-delivered seed never passes through the `applySeed` wrapper, so
|
|
7742
8246
|
// this is where its preferred peers become cold-start bootstrap targets.
|
|
7743
|
-
this.recordSeedBootstrapPeers(seed);
|
|
8247
|
+
const dialTargets = this.recordSeedBootstrapPeers(seed);
|
|
8248
|
+
this.emit('seed:applied', { partyId, peersAdded, dialTargets });
|
|
7744
8249
|
void this.refreshMembershipGate('seed-applied');
|
|
7745
8250
|
},
|
|
7746
8251
|
onSeedError: (partyId, error) => this.emit('seed:error', { partyId, error }),
|
|
@@ -7764,13 +8269,14 @@ export class CadreNode {
|
|
|
7764
8269
|
* NO `CadrePeer` writer needs to call this: the control database notifies it
|
|
7765
8270
|
* after every committed member-row write (`ControlDatabase.mutateCadrePeer`,
|
|
7766
8271
|
* wired in {@link start}), which is exactly what makes the refresh automatic
|
|
7767
|
-
* rather than a caller obligation.
|
|
7768
|
-
*
|
|
7769
|
-
*
|
|
7770
|
-
*
|
|
7771
|
-
*
|
|
7772
|
-
*
|
|
7773
|
-
*
|
|
8272
|
+
* rather than a caller obligation. A membership row that arrived by REPLICATION
|
|
8273
|
+
* refreshes it too, through the gate watch {@link start} takes on the membership
|
|
8274
|
+
* tables (within its 50 ms window; a change this node's storage never hears of
|
|
8275
|
+
* waits for the reconcile pass's backstop). It stays public for the one change
|
|
8276
|
+
* that writes no row and raises no table event: a newly anchored trusted owner
|
|
8277
|
+
* key ({@link applySeed}), which flips rows ALREADY present from unauthorized to
|
|
8278
|
+
* authorized without touching them. A caller may also await it to observe a
|
|
8279
|
+
* refresh that began after its own change.
|
|
7774
8280
|
*
|
|
7775
8281
|
* Coalescing: marks the snapshot stale and resolves once a refresh that began
|
|
7776
8282
|
* after this call has completed, so a caller that awaits it always observes
|
|
@@ -7795,13 +8301,10 @@ export class CadreNode {
|
|
|
7795
8301
|
* connected peer it newly covers — the strand-side counterpart to
|
|
7796
8302
|
* {@link refreshMembershipGate}.
|
|
7797
8303
|
*
|
|
7798
|
-
*
|
|
7799
|
-
*
|
|
7800
|
-
*
|
|
7801
|
-
*
|
|
7802
|
-
* The gate therefore polls (default 30 s). An app that has just called
|
|
7803
|
-
* `revokeMember` or `leaveStrand` should follow the write with this call so
|
|
7804
|
-
* the cut is immediate rather than up to one poll interval late.
|
|
8304
|
+
* The gate already refreshes on its own when the strand's `Member` or
|
|
8305
|
+
* `MemberPeer` table changes: at once for a `revokeMember` or `leaveStrand`
|
|
8306
|
+
* written on this machine, and when a replicated removal reaches this node's
|
|
8307
|
+
* storage. Call this to await the cut — it resolves after the sweep.
|
|
7805
8308
|
*
|
|
7806
8309
|
* Quiet no-op for a strand that is not running, is quiesced, is open, or has
|
|
7807
8310
|
* the gate disabled. Never rejects; resolves once the sweep has finished.
|
|
@@ -7852,8 +8355,8 @@ export class CadreNode {
|
|
|
7852
8355
|
* CONCURRENT external write's refresh for the life of the scope: that writer's
|
|
7853
8356
|
* promise resolves before its peer is admitted, and admission lands at scope
|
|
7854
8357
|
* exit instead. Acceptable — scope exit is milliseconds away (bounded by the
|
|
7855
|
-
* drain), versus the
|
|
7856
|
-
*
|
|
8358
|
+
* drain), versus the reconcile pass that was the alternative before any of this
|
|
8359
|
+
* existed — but it is the reason these scopes stay short and rare.
|
|
7857
8360
|
*/
|
|
7858
8361
|
async deferMembershipGateRefresh(reason, body) {
|
|
7859
8362
|
this.membershipGateDeferDepth++;
|
|
@@ -7989,7 +8492,7 @@ export class CadreNode {
|
|
|
7989
8492
|
throw new Error('CadreNode must be started before planning an owner key removal');
|
|
7990
8493
|
}
|
|
7991
8494
|
const key = requireEd25519PublicKeyB64(ownerPublicKeyB64, 'owner key');
|
|
7992
|
-
return
|
|
8495
|
+
return await this.planOwnerKeyRemovalWith(this.controlDatabase, key);
|
|
7993
8496
|
}
|
|
7994
8497
|
/**
|
|
7995
8498
|
* Remove an owner key, signed by this node's owner key (which must be a different owner): the
|
|
@@ -8022,7 +8525,7 @@ export class CadreNode {
|
|
|
8022
8525
|
throw new Error('Seed bootstrap service not initialized. Call initializeSeedBootstrap() first.');
|
|
8023
8526
|
}
|
|
8024
8527
|
const key = requireEd25519PublicKeyB64(ownerPublicKeyB64, 'owner key');
|
|
8025
|
-
const
|
|
8528
|
+
const plan = await this.planOwnerKeyRemovalWith(this.controlDatabase, key);
|
|
8026
8529
|
if (plan.refusal !== null) {
|
|
8027
8530
|
throw new OwnerKeyRemovalRefusedError(key, plan.refusal);
|
|
8028
8531
|
}
|
|
@@ -8037,18 +8540,18 @@ export class CadreNode {
|
|
|
8037
8540
|
}
|
|
8038
8541
|
const action = `Remove the owner key ${namedOwnerKey(key, plan.ownerKey?.label ?? null)} from the cadre`;
|
|
8039
8542
|
const removed = await signer.approve({ partyId: this.partyId, action }, async (sign) => {
|
|
8040
|
-
await this.revouchDependents(service, revouch,
|
|
8543
|
+
await this.revouchDependents(service, revouch, sign);
|
|
8041
8544
|
return await service.removeOwner(key, sign);
|
|
8042
8545
|
});
|
|
8043
8546
|
log('removeOwner(%s): removed=%s revouched owners=%d peers=%d', key, removed, revouch.owners.length, revouch.peers.length);
|
|
8044
8547
|
return { removed, revouched: revouch };
|
|
8045
8548
|
}
|
|
8046
|
-
/** {@link planOwnerKeyRemoval}
|
|
8549
|
+
/** {@link planOwnerKeyRemoval} over `controlDatabase`, for a caller that already holds it. */
|
|
8047
8550
|
async planOwnerKeyRemovalWith(controlDatabase, key) {
|
|
8048
8551
|
const [rows, owners, grants] = await Promise.all([
|
|
8049
8552
|
controlDatabase.queryCadrePeers(), controlDatabase.queryOwnerKeyRows(), controlDatabase.listOwnerGrantStatuses()
|
|
8050
8553
|
]);
|
|
8051
|
-
|
|
8554
|
+
return planOwnerKeyRemoval({
|
|
8052
8555
|
key,
|
|
8053
8556
|
rows,
|
|
8054
8557
|
owners,
|
|
@@ -8057,7 +8560,6 @@ export class CadreNode {
|
|
|
8057
8560
|
selfOwnerKey: this.ownerSignerEnrolledIn(new Set(owners.map((owner) => owner.key)))?.publicKey ?? null,
|
|
8058
8561
|
anchor: this.trustedOwnerStore?.all() ?? new Set(),
|
|
8059
8562
|
});
|
|
8060
|
-
return { plan, rows };
|
|
8061
8563
|
}
|
|
8062
8564
|
/**
|
|
8063
8565
|
* The re-vouch step of {@link removeOwner}: every chosen dependent row is rebound to this
|
|
@@ -8066,12 +8568,14 @@ export class CadreNode {
|
|
|
8066
8568
|
* the only rebinding `OwnerKey.NoUpdate` allows); peers through the owner re-touch
|
|
8067
8569
|
* (`reauthorizePeer`), which rewrites `VouchOwner`/`VouchSig` and clears `VouchUsage`.
|
|
8068
8570
|
*
|
|
8069
|
-
* The peer re-touch
|
|
8070
|
-
* `
|
|
8071
|
-
*
|
|
8072
|
-
*
|
|
8073
|
-
*
|
|
8074
|
-
*
|
|
8571
|
+
* The peer re-touch is the re-vouch mode of `reauthorizePeer`, which leaves the row's
|
|
8572
|
+
* `UpdatedAt` untouched and writes whether or not the machine has self-published. The owner
|
|
8573
|
+
* branch of `CadrePeer.AuthorizedUpdate` does not require a bump, and bumping a self-published
|
|
8574
|
+
* row would leave its `Sig` unverifiable, so unresolvable and unlisted as self-published, until
|
|
8575
|
+
* the machine's next heartbeat (the NOTE on that constraint); the write-while-alone drain's
|
|
8576
|
+
* re-broadcast mode bumps because it re-issues an otherwise unchanged row, whereas here the
|
|
8577
|
+
* voucher itself changes. A dependent whose row vanished since the plan is reported absent and
|
|
8578
|
+
* skipped.
|
|
8075
8579
|
*
|
|
8076
8580
|
* A failure rethrows and stops the call before the removal; re-vouches already made stay
|
|
8077
8581
|
* (the row is only re-signed by a current owner) and the caller retries. An owner re-vouch
|
|
@@ -8082,7 +8586,7 @@ export class CadreNode {
|
|
|
8082
8586
|
* rows anyway, so the other machines may get the removal's tombstone without the new voucher
|
|
8083
8587
|
* (`tickets/backlog/bug-owner-key-removal-revouch-made-alone-is-not-reissued`).
|
|
8084
8588
|
*/
|
|
8085
|
-
async revouchDependents(service, revouch,
|
|
8589
|
+
async revouchDependents(service, revouch, sign) {
|
|
8086
8590
|
for (const key of revouch.owners) {
|
|
8087
8591
|
await service.revouchOwner(key, sign);
|
|
8088
8592
|
if (this.committedAlone()) {
|
|
@@ -8091,9 +8595,7 @@ export class CadreNode {
|
|
|
8091
8595
|
}
|
|
8092
8596
|
}
|
|
8093
8597
|
for (const dependent of revouch.peers) {
|
|
8094
|
-
|
|
8095
|
-
const updatedAt = row !== undefined && row.updatedAt > 0 ? row.updatedAt : Date.now();
|
|
8096
|
-
await service.reauthorizePeer(dependent, updatedAt, sign);
|
|
8598
|
+
await service.reauthorizePeer(dependent, 'revouch', sign);
|
|
8097
8599
|
}
|
|
8098
8600
|
}
|
|
8099
8601
|
/**
|
|
@@ -8118,6 +8620,10 @@ export class CadreNode {
|
|
|
8118
8620
|
throw new Error('CadreNode must be started before planning a machine removal');
|
|
8119
8621
|
}
|
|
8120
8622
|
const machines = await this.readMachines(this.controlDatabase);
|
|
8623
|
+
// NOTE: the invitation and usage lists come from the membership views, so on a node that
|
|
8624
|
+
// heard of a row with no storage event (outside the block's cohort) an invitation issued
|
|
8625
|
+
// elsewhere in the last MEMBERSHIP_VIEW_MAX_AGE_MS can be missing and left unwithdrawn. If
|
|
8626
|
+
// removal must close that window, read both tables fresh here.
|
|
8121
8627
|
const [invites, usages, owners] = await Promise.all([
|
|
8122
8628
|
this.controlDatabase.listCadreInviteStatuses(),
|
|
8123
8629
|
this.controlDatabase.queryCadreInviteUsages(),
|
|
@@ -8328,12 +8834,13 @@ export class CadreNode {
|
|
|
8328
8834
|
* that connection from the addresses it was handed, on this launch or a later
|
|
8329
8835
|
* one.
|
|
8330
8836
|
*
|
|
8331
|
-
* Nothing is dialed here: the drone has not received the seed yet.
|
|
8332
|
-
*
|
|
8333
|
-
*
|
|
8334
|
-
*
|
|
8335
|
-
*
|
|
8336
|
-
*
|
|
8837
|
+
* Nothing is dialed here: the drone has not received the seed yet. Its `CadrePeer`
|
|
8838
|
+
* insert is a membership change, so a reconcile pass dials the retained addresses
|
|
8839
|
+
* within about 50 ms of this call returning, whether or not the caller runs one, and
|
|
8840
|
+
* a failed dial is retried on the redial ladder (15 s, 30 s, 60 s, then every
|
|
8841
|
+
* backstop). A caller that wants to await the connection calls
|
|
8842
|
+
* {@link reconcileControlCohort} itself, twice in a row: a call made while a pass is
|
|
8843
|
+
* in flight joins it, and that pass may have listed siblings before this add.
|
|
8337
8844
|
*/
|
|
8338
8845
|
async addDrone(options) {
|
|
8339
8846
|
if (!this.seedBootstrapService) {
|
|
@@ -8378,12 +8885,19 @@ export class CadreNode {
|
|
|
8378
8885
|
* by the node (its anchored branch), the row insert is a no-op on an existing row
|
|
8379
8886
|
* (`ControlDatabase.insertCadrePeer`), and the dial target is replaced, not duplicated.
|
|
8380
8887
|
*
|
|
8381
|
-
*
|
|
8382
|
-
*
|
|
8383
|
-
*
|
|
8384
|
-
*
|
|
8385
|
-
*
|
|
8386
|
-
*
|
|
8888
|
+
* Once it has acknowledged, the node restarts into this party (cadre-cli's
|
|
8889
|
+
* `restartIntoClaimedParty`), closing the connection the claim used. The dial target puts
|
|
8890
|
+
* the address that connection was dialed on first, so every later dial tries the address
|
|
8891
|
+
* that just worked before the others (a QR code lists public addresses before LAN ones, and
|
|
8892
|
+
* a home router often drops a connection to its own public address from inside). A
|
|
8893
|
+
* reconcile pass is kicked (not awaited) now, and the vouch's `CadrePeer` insert starts
|
|
8894
|
+
* another about 50 ms later; for the first {@link CONTROL_COHORT_REDIAL_MS} after the
|
|
8895
|
+
* acknowledgement this node also retries the node every second while it restarts
|
|
8896
|
+
* ({@link redialClaimedNode}). A caller that wants to wait for the connection awaits
|
|
8897
|
+
* {@link reconcileControlCohort} itself, with `addDrone`'s caveat that a pass already in
|
|
8898
|
+
* flight is joined rather than restarted. The claimed node dials this one back too, as the
|
|
8899
|
+
* seed's minter, when this node has a listen or relay address; a phone with neither is
|
|
8900
|
+
* reached by its own dials alone.
|
|
8387
8901
|
*
|
|
8388
8902
|
* @throws {ClaimRefusedError} when the node was reached and refused the claim.
|
|
8389
8903
|
* @throws {PeerUnreachableError} when the node was never reached.
|
|
@@ -8398,6 +8912,7 @@ export class CadreNode {
|
|
|
8398
8912
|
if (!signer) {
|
|
8399
8913
|
throw new Error('Owner key required to create seeds');
|
|
8400
8914
|
}
|
|
8915
|
+
let dialAddrs = target.multiaddrs;
|
|
8401
8916
|
await signer.approve({ partyId: this.partyId, action: `Claim the node ${target.peerId} for the cadre` }, async (sign) => {
|
|
8402
8917
|
const seed = await service.createSeed(sign);
|
|
8403
8918
|
const proof = claimProof(secret, target.peerId, seed.signerKey, seedDigest(seed));
|
|
@@ -8405,13 +8920,87 @@ export class CadreNode {
|
|
|
8405
8920
|
if (!ack.accepted) {
|
|
8406
8921
|
throw new ClaimRefusedError(target.peerId, ack.reason ?? 'no reason given', ack.code);
|
|
8407
8922
|
}
|
|
8923
|
+
// Read before anything else is awaited: the node's restart closes the connection.
|
|
8924
|
+
dialAddrs = this.connectedAddrFirst(target.peerId, target.multiaddrs);
|
|
8408
8925
|
log('claimNode: %s accepted the claim; vouching it into the cadre', target.peerId);
|
|
8409
8926
|
// The wrapper queues the write for re-replication: a phone claiming its first
|
|
8410
8927
|
// always-on node has no control connection yet, so the insert commits local-only.
|
|
8928
|
+
// NOTE: accepted tradeoff — the node's own signed record is not carried in the
|
|
8929
|
+
// acknowledgement and written with this vouch: it would be signed before the node
|
|
8930
|
+
// restarts, so its addresses may not be the restarted node's (a new relay reservation),
|
|
8931
|
+
// and the write-while-alone drain re-sends only rows whose `Sig` is null
|
|
8932
|
+
// (drainPendingPeerWrites, reconstructAuthoredMembership), so a vouch carrying it would
|
|
8933
|
+
// never reach this cadre's other members. The node publishes its record once it has
|
|
8934
|
+
// restarted. Revisit if the drain ever re-sends signed rows.
|
|
8411
8935
|
await this.vouchPeer(target.peerId, target.multiaddrs, sign);
|
|
8412
8936
|
});
|
|
8413
|
-
this.retainDialTarget(target.peerId,
|
|
8937
|
+
this.retainDialTarget(target.peerId, dialAddrs, 'claimNode');
|
|
8414
8938
|
void this.reconcileControlCohort().catch((error) => log('claimNode: reconcile kick after claiming %s failed: %o', target.peerId, error));
|
|
8939
|
+
void this.redialClaimedNode(target.peerId).catch((error) => log('claimNode: redialing %s while it restarts failed: %o', target.peerId, error));
|
|
8940
|
+
}
|
|
8941
|
+
/**
|
|
8942
|
+
* `addrs` with the address this node's open connection to `peerId` was dialed on moved to
|
|
8943
|
+
* the front, when it is one of them; `addrs` unchanged otherwise. Compared without a
|
|
8944
|
+
* trailing `/p2p/<peerId>`, which a QR code's address may or may not carry ({@link addrKey}).
|
|
8945
|
+
*/
|
|
8946
|
+
connectedAddrFirst(peerId, addrs) {
|
|
8947
|
+
const controlNode = this.controlNode;
|
|
8948
|
+
if (!controlNode) {
|
|
8949
|
+
return addrs;
|
|
8950
|
+
}
|
|
8951
|
+
const remote = peerIdFromString(peerId);
|
|
8952
|
+
const connected = new Set(controlNode.getConnections(remote).map((c) => addrKey(c.remoteAddr.toString(), remote)));
|
|
8953
|
+
const index = addrs.findIndex((addr) => connected.has(addrKey(addr, remote)));
|
|
8954
|
+
return index <= 0 ? addrs : [addrs[index], ...addrs.slice(0, index), ...addrs.slice(index + 1)];
|
|
8955
|
+
}
|
|
8956
|
+
/**
|
|
8957
|
+
* Retry a node this node has just claimed ({@link claimNode}) once every
|
|
8958
|
+
* {@link CLAIM_RESTART_REDIAL_STEP_MS} for {@link CONTROL_COHORT_REDIAL_MS} after it accepted.
|
|
8959
|
+
* The node restarts into this party right after it acknowledges, and while it does, a
|
|
8960
|
+
* reconcile pass's dial either fails or opens a connection the restarting node closes at
|
|
8961
|
+
* once. The first failure puts the pass's next redial 15 s out ({@link armReconcileWake}),
|
|
8962
|
+
* and an owner that cannot be dialed has no other way back to the node.
|
|
8963
|
+
*
|
|
8964
|
+
* Each step dials once from the retained dial target, unless a connection to the node is
|
|
8965
|
+
* open: the one the claim used until the restart closes it, or one to the restarted node
|
|
8966
|
+
* that has not identified yet. A dial that connects does not end the loop, because the
|
|
8967
|
+
* restarting node may still close it. The loop ends once the node has identified as serving
|
|
8968
|
+
* this party's control database ({@link identifiedControlPeers}, which the placeholder node
|
|
8969
|
+
* it ran before the claim never joins), at the window's end, when this node stops, or when
|
|
8970
|
+
* the target is forgotten ({@link removePeer}). Steps run one after another, so a dial
|
|
8971
|
+
* finishes on its own budget first. The redial streak is left alone: the ladder carries on
|
|
8972
|
+
* after the window. One loop per peer: a repeated claim while one runs starts none.
|
|
8973
|
+
*/
|
|
8974
|
+
async redialClaimedNode(peerId) {
|
|
8975
|
+
if (this.claimRedialPeers.has(peerId)) {
|
|
8976
|
+
return;
|
|
8977
|
+
}
|
|
8978
|
+
this.claimRedialPeers.add(peerId);
|
|
8979
|
+
try {
|
|
8980
|
+
const endsAt = Date.now() + CONTROL_COHORT_REDIAL_MS;
|
|
8981
|
+
while (Date.now() < endsAt && await this.redialClaimedNodeStep(peerId)) {
|
|
8982
|
+
await new Promise((resolve) => {
|
|
8983
|
+
const timer = setTimeout(resolve, CLAIM_RESTART_REDIAL_STEP_MS);
|
|
8984
|
+
// Node-only: a stop mid-window must not keep the event loop alive for the step.
|
|
8985
|
+
timer.unref?.();
|
|
8986
|
+
});
|
|
8987
|
+
}
|
|
8988
|
+
}
|
|
8989
|
+
finally {
|
|
8990
|
+
this.claimRedialPeers.delete(peerId);
|
|
8991
|
+
}
|
|
8992
|
+
}
|
|
8993
|
+
/** One step of {@link redialClaimedNode}; false ends the loop. */
|
|
8994
|
+
async redialClaimedNodeStep(peerId) {
|
|
8995
|
+
const controlNode = this.controlNode;
|
|
8996
|
+
const target = this.bootstrapPeerStore?.all().get(peerId);
|
|
8997
|
+
if (!this._running || !controlNode || !target || this.identifiedControlPeers.has(peerId)) {
|
|
8998
|
+
return false;
|
|
8999
|
+
}
|
|
9000
|
+
if (controlNode.getConnections(peerIdFromString(peerId)).length === 0) {
|
|
9001
|
+
await this.dialBootstrapPeer(peerId, target.addrs, 'claimNode(redial)');
|
|
9002
|
+
}
|
|
9003
|
+
return true;
|
|
8415
9004
|
}
|
|
8416
9005
|
/**
|
|
8417
9006
|
* Add a phone to the cadre with relay support.
|
|
@@ -8569,11 +9158,13 @@ export class CadreNode {
|
|
|
8569
9158
|
* Steps: refuse an invitation for another party; pin its `ownerKeys` into the node-local
|
|
8570
9159
|
* anchor (`'invite'` provenance — the pin sticks even if the redemption then fails, as it
|
|
8571
9160
|
* does for a seed invite, so a later attempt or seed from the same owner is anchored);
|
|
8572
|
-
* sign the request
|
|
8573
|
-
* acceptance verify the reply against the pinned
|
|
8574
|
-
* store, retain the answering member as a
|
|
8575
|
-
*
|
|
8576
|
-
* the connection it already holds, the same unified behaviour as after a seed.
|
|
9161
|
+
* sign the request, which carries this node's own signed address record; dial the listed
|
|
9162
|
+
* members in order (`redeemAtMembers`); on acceptance verify the reply against the pinned
|
|
9163
|
+
* keys, merge its dial hints into the peer store, retain the answering member as a
|
|
9164
|
+
* cold-start dial target and start a cohort reconcile pass. This node then syncs the control
|
|
9165
|
+
* database over the connection it already holds, the same unified behaviour as after a seed.
|
|
9166
|
+
* The member writes the record on this node's `CadrePeer` row when its signature verifies, so
|
|
9167
|
+
* the row resolves to this node's addresses from its admission.
|
|
8577
9168
|
*
|
|
8578
9169
|
* A device that is already a member (a restart with the invitation still configured) is
|
|
8579
9170
|
* answered as accepted by the member's idempotent redemption, with no second usage row.
|
|
@@ -8617,7 +9208,15 @@ export class CadreNode {
|
|
|
8617
9208
|
}
|
|
8618
9209
|
await this.trustOwnerKeys(invitation.ownerKeys, 'invite');
|
|
8619
9210
|
const pinned = new Set(invitation.ownerKeys);
|
|
8620
|
-
const
|
|
9211
|
+
const selfPeerId = controlNode.peerId.toString();
|
|
9212
|
+
// The addresses a publish would carry, sanitized before signing: the member stores the
|
|
9213
|
+
// signature only over the exact list it stores.
|
|
9214
|
+
const record = this.signSelfRecord(selfPeerId, signer, sanitizeAddrs(await this.collectSelfAddrs()), null);
|
|
9215
|
+
const request = signRedeemRequest(invitation, { peerKey: signer.publicKeyB64, peerPrivateKey: signer.privateKeyB64 }, generateStampId(selfPeerId), record);
|
|
9216
|
+
// Before dialing, not on acceptance: the member's commit can reach this node before its
|
|
9217
|
+
// reply does, and a reconcile pass that reads the row first must find it current. A record
|
|
9218
|
+
// no member wrote matches no row, so a failed redemption leaves nothing to undo.
|
|
9219
|
+
this.lastSignedSelfRecord = record;
|
|
8621
9220
|
const { reply, memberPeerId, memberAddr } = await redeemInPasses(controlNode, {
|
|
8622
9221
|
invitation,
|
|
8623
9222
|
request,
|
|
@@ -8628,7 +9227,6 @@ export class CadreNode {
|
|
|
8628
9227
|
...(options.onProgress ? { onProgress: options.onProgress } : {}),
|
|
8629
9228
|
});
|
|
8630
9229
|
log('Cadre invitation %s redeemed at %s', invitation.invite.key, memberAddr);
|
|
8631
|
-
const selfPeerId = controlNode.peerId.toString();
|
|
8632
9230
|
await mergeSeedPeers(controlNode, reply.peers.filter((peer) => peer.peerId !== selfPeerId));
|
|
8633
9231
|
if (memberPeerId !== null) {
|
|
8634
9232
|
// Every address the bundle gave the answering member, so a restart has the same choices.
|
|
@@ -8637,9 +9235,8 @@ export class CadreNode {
|
|
|
8637
9235
|
.map((addr) => addr.toString());
|
|
8638
9236
|
this.retainDialTarget(memberPeerId, memberAddrs, 'redeemCadreInvitation');
|
|
8639
9237
|
}
|
|
8640
|
-
//
|
|
8641
|
-
//
|
|
8642
|
-
await this.refreshMembershipGate('invitation-redeemed');
|
|
9238
|
+
// No gate refresh here: the rows about to replicate in are judged against the pins above
|
|
9239
|
+
// when they land, by the gate watch's refresh and the head refresh of this pass.
|
|
8643
9240
|
void this.reconcileControlCohort().catch((error) => log('redeemCadreInvitation: the post-redemption reconcile pass failed: %o', error));
|
|
8644
9241
|
return { peerId: memberPeerId, redeemedAt: new Date().toISOString() };
|
|
8645
9242
|
}
|
|
@@ -8811,25 +9408,31 @@ export class CadreNode {
|
|
|
8811
9408
|
return { ownerKey: signer.publicKey, alreadyOwner: reply.alreadyOwner, memberPeerId };
|
|
8812
9409
|
}
|
|
8813
9410
|
/**
|
|
8814
|
-
* {@link redeemOwnerGrant}'s local path: the alone refusal, the seat
|
|
8815
|
-
* then one approval around the consent signature and the
|
|
8816
|
-
* the approval, so a prompting signer never asks for a
|
|
9411
|
+
* {@link redeemOwnerGrant}'s local path: the alone refusal, the seat (only when this node holds
|
|
9412
|
+
* no grant row), the liveness refusal, then one approval around the consent signature and the
|
|
9413
|
+
* write. Every refusal comes before the approval, so a prompting signer never asks for a
|
|
9414
|
+
* redemption this node would refuse. The held row is read once and carried into the liveness
|
|
9415
|
+
* check and the write, as the member handler carries it (`MemberRedemptionHandler`).
|
|
8817
9416
|
*/
|
|
8818
9417
|
async redeemOwnerGrantLocally(controlDatabase, bundle, signer) {
|
|
8819
9418
|
if (await this.ownerKeyWriteAlone()) {
|
|
8820
9419
|
throw new OwnerKeyAddRefusedError(signer.publicKey, 'alone');
|
|
8821
9420
|
}
|
|
8822
|
-
|
|
9421
|
+
const held = await controlDatabase.queryOwnerGrant(bundle.grant.key);
|
|
9422
|
+
if (held === null) {
|
|
9423
|
+
await refusingUnsyncedGrant(() => controlDatabase.seatOwnerGrant(bundle.grant));
|
|
9424
|
+
}
|
|
8823
9425
|
// The write answers a key that is already an owner without reading the grant, so only a
|
|
8824
9426
|
// key that is not one needs the grant live (the same permissive pre-check the handler's
|
|
8825
9427
|
// admission uses; the write enforces every condition).
|
|
8826
|
-
if (!(await controlDatabase.getOwnerKeys()).has(signer.publicKey) && !await controlDatabase.isOwnerGrantLive(bundle.grant)) {
|
|
9428
|
+
if (!(await controlDatabase.getOwnerKeys()).has(signer.publicKey) && !await controlDatabase.isOwnerGrantLive(bundle.grant, held)) {
|
|
8827
9429
|
throw new OwnerGrantRejectedError('grant-spent', 'The grant is expired, withdrawn, exhausted, or its issuer is no longer an owner');
|
|
8828
9430
|
}
|
|
8829
9431
|
const { alreadyOwner } = await signer.approve(ownerGrantApproval(bundle), async (sign) => {
|
|
8830
9432
|
const request = await this.signOwnerGrantRequest(bundle, signer, sign);
|
|
8831
9433
|
return await refusingUnsyncedGrant(() => controlDatabase.redeemOwnerGrant({
|
|
8832
9434
|
grantKey: bundle.grant.key,
|
|
9435
|
+
...(held ? { held } : {}),
|
|
8833
9436
|
seatedKey: signer.publicKey,
|
|
8834
9437
|
usageStampId: request.usageStampId,
|
|
8835
9438
|
grantSig: request.grantSig,
|
|
@@ -9015,7 +9618,7 @@ export class CadreNode {
|
|
|
9015
9618
|
// Overridden on the same grounds: only this node holds the running strand
|
|
9016
9619
|
// instance's database and this party's `StrandPartyKey` identity, and a wrong
|
|
9017
9620
|
// issuer here would admit joiners under someone else's authority.
|
|
9018
|
-
issueMembershipInvite: (
|
|
9621
|
+
issueMembershipInvite: (host, signal) => this.issueStrandMembershipInvite(host, signal)
|
|
9019
9622
|
});
|
|
9020
9623
|
const previous = this.strandSolicitationService;
|
|
9021
9624
|
if (previous) {
|
|
@@ -9280,10 +9883,17 @@ export class CadreNode {
|
|
|
9280
9883
|
selfId: this.controlNode.peerId.toString(),
|
|
9281
9884
|
linkRoundTripMs: this.config.network?.linkRoundTripMs,
|
|
9282
9885
|
isOwner: () => this.isOwner(),
|
|
9283
|
-
// NOTE: one owner-key read and one pending-join read (three tables) per pass (30 s); for a
|
|
9284
|
-
// party that never asked for a join the second reads never-written blocks, which consults
|
|
9285
|
-
// the cohort. If it shows up in a device profile, poll more slowly while the tables read empty.
|
|
9286
9886
|
readRows: () => database().queryPendingJoins(),
|
|
9887
|
+
// What `isOwner` and `queryPendingJoins` read: the owner keys, the joins and their
|
|
9888
|
+
// outcomes, and the tombstones both filter by. A dismissal writes its tombstone with the
|
|
9889
|
+
// request's delete, but the two can replicate separately.
|
|
9890
|
+
watchChanges: (onChange, options) => database().watchTables([
|
|
9891
|
+
'CadreControl.JoinRequest',
|
|
9892
|
+
'CadreControl.JoinSuccess',
|
|
9893
|
+
'CadreControl.JoinFailure',
|
|
9894
|
+
'CadreControl.OwnerKey',
|
|
9895
|
+
'CadreControl.Revocation',
|
|
9896
|
+
], onChange, options),
|
|
9287
9897
|
readRow: (id) => database().queryPendingJoin(id),
|
|
9288
9898
|
attempt: (join) => this.formStrand(this.pendingJoinInvitation(join), parseStoredDisclosure(join)),
|
|
9289
9899
|
record: (expected, outcome) => ownerWrite(`Record the outcome of the pending join ${expected.Id}`, (ownerKey, sign) => database().recordJoinOutcome(expected, outcome, ownerKey, sign)),
|
|
@@ -9344,6 +9954,13 @@ export class CadreNode {
|
|
|
9344
9954
|
* token is spent, and a join no store names would vanish at the next restart.
|
|
9345
9955
|
*/
|
|
9346
9956
|
async rememberFormedStrand(result) {
|
|
9957
|
+
// NOTE: accepted tradeoff — the party-wide `JoinedStrand` row is not written here, in
|
|
9958
|
+
// formStrand's own approval, but later by an owner machine's connected reconcile pass
|
|
9959
|
+
// (`JoinedStrandSession.syncWithParty`). That publish is gated on a control connection so a
|
|
9960
|
+
// machine cut off from its own party never commits it alone, and during formStrand the
|
|
9961
|
+
// count always includes the connection to the other party's responder, so the gate would
|
|
9962
|
+
// always pass here. Non-owner and signer-less machines need this local path regardless.
|
|
9963
|
+
// Revisit if the gate can tell a connection to this party's own machines from any other.
|
|
9347
9964
|
try {
|
|
9348
9965
|
await this.joinedStrands.remember({
|
|
9349
9966
|
Id: result.strandId,
|
|
@@ -9383,7 +10000,7 @@ export class CadreNode {
|
|
|
9383
10000
|
*/
|
|
9384
10001
|
async adoptFormationMembershipInvite(strandId, invite) {
|
|
9385
10002
|
try {
|
|
9386
|
-
await this.ensureStrandPartyKey(strandId);
|
|
10003
|
+
this.formedPartyKeys.set(strandId, await this.ensureStrandPartyKey(strandId));
|
|
9387
10004
|
}
|
|
9388
10005
|
catch (error) {
|
|
9389
10006
|
throw new FormationPostApprovalError(strandId, `Formation for strand ${strandId} was approved (its one-time token is spent), but ` +
|
|
@@ -9428,6 +10045,10 @@ export class CadreNode {
|
|
|
9428
10045
|
* implements): mint a single-use `Strand.Invite` against the LIVE host strand so a
|
|
9429
10046
|
* validated joiner can seat its own `Strand.Member` row.
|
|
9430
10047
|
*
|
|
10048
|
+
* `host` is the strand as the formation session resolved it from its `Strand` row, read
|
|
10049
|
+
* once there; this method reads no `Strand` row of its own. A row removed in between is
|
|
10050
|
+
* refused when the consent row is written (the `StrandExists` CHECK).
|
|
10051
|
+
*
|
|
9431
10052
|
* - Open host strand → `null` (no members, nothing to invite into).
|
|
9432
10053
|
* - Closed host strand whose founder launch was refused as pre-split
|
|
9433
10054
|
* ({@link strandLaunchRefusals}) → rethrow the recorded `PreSplitStrandIdentityError`;
|
|
@@ -9475,17 +10096,12 @@ export class CadreNode {
|
|
|
9475
10096
|
* `MEMBERSHIP_INVITE_UNAVAILABLE_REASON` forever; that flow needs manager delegation,
|
|
9476
10097
|
* not a retry.
|
|
9477
10098
|
*/
|
|
9478
|
-
async issueStrandMembershipInvite(
|
|
10099
|
+
async issueStrandMembershipInvite(host, signal) {
|
|
10100
|
+
const { strandId } = host;
|
|
9479
10101
|
if (!this.controlDatabase) {
|
|
9480
10102
|
throw new Error(`Cannot issue a membership invitation for strand ${strandId}: control database unavailable`);
|
|
9481
10103
|
}
|
|
9482
|
-
|
|
9483
|
-
if (!row) {
|
|
9484
|
-
// The bound resolution saw this row moments ago; a vanished row is a concurrent
|
|
9485
|
-
// unpublish — reject rather than invite into a strand this party just removed.
|
|
9486
|
-
throw new Error(`Cannot issue a membership invitation for strand ${strandId}: its Strand row is gone`);
|
|
9487
|
-
}
|
|
9488
|
-
if (row.Type !== 'c') {
|
|
10104
|
+
if (host.strandType !== 'c') {
|
|
9489
10105
|
return null;
|
|
9490
10106
|
}
|
|
9491
10107
|
const refusal = this.strandLaunchRefusals.get(strandId);
|
|
@@ -9503,8 +10119,8 @@ export class CadreNode {
|
|
|
9503
10119
|
throw new Error(`Cannot issue a membership invitation for closed strand ${strandId}: its runtime is ` +
|
|
9504
10120
|
'not live on this responder (not launched, or still starting)');
|
|
9505
10121
|
}
|
|
9506
|
-
if (
|
|
9507
|
-
await assertNotPreSplitStrand(db, strandId, strandMemberKeyPair(
|
|
10122
|
+
if (host.memberPrivateKey) {
|
|
10123
|
+
await assertNotPreSplitStrand(db, strandId, strandMemberKeyPair(host.memberPrivateKey).publicKeyB64);
|
|
9508
10124
|
}
|
|
9509
10125
|
if (signal?.aborted) {
|
|
9510
10126
|
throw new Error(`Cannot issue a membership invitation for closed strand ${strandId}: the formation ` +
|