@serfab/cadre-core 1.17.0 → 1.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/dist/cadre-invite-protocol.d.ts +35 -13
- package/dist/cadre-invite-protocol.js +90 -25
- package/dist/cadre-invite-protocol.js.map +1 -1
- package/dist/cadre-node.d.ts +586 -255
- package/dist/cadre-node.js +1176 -564
- package/dist/cadre-node.js.map +1 -1
- package/dist/control-cohort.d.ts +66 -9
- package/dist/control-cohort.js +67 -9
- package/dist/control-cohort.js.map +1 -1
- package/dist/control-database.d.ts +451 -193
- package/dist/control-database.js +701 -359
- package/dist/control-database.js.map +1 -1
- package/dist/control-formation-recorder.d.ts +46 -43
- package/dist/control-formation-recorder.js +77 -78
- package/dist/control-formation-recorder.js.map +1 -1
- package/dist/control-protocol-guard.d.ts +6 -0
- package/dist/control-protocol-guard.js +14 -2
- package/dist/control-protocol-guard.js.map +1 -1
- package/dist/control-schema-version.d.ts +48 -31
- package/dist/control-schema-version.js +67 -45
- package/dist/control-schema-version.js.map +1 -1
- package/dist/control-schema.d.ts +8 -8
- package/dist/control-schema.js +16 -16
- package/dist/control-schema.js.map +1 -1
- package/dist/control-write-retry.d.ts +42 -7
- package/dist/control-write-retry.js +87 -19
- package/dist/control-write-retry.js.map +1 -1
- package/dist/enrolled-machine-store.d.ts +3 -3
- package/dist/enrolled-machine-store.js +3 -3
- package/dist/formation-token.d.ts +48 -0
- package/dist/formation-token.js +26 -0
- package/dist/formation-token.js.map +1 -0
- package/dist/index.d.ts +14 -13
- package/dist/index.js +16 -13
- package/dist/index.js.map +1 -1
- package/dist/joined-strand-store.d.ts +26 -9
- package/dist/joined-strand-store.js +63 -24
- package/dist/joined-strand-store.js.map +1 -1
- package/dist/link-budget.d.ts +9 -7
- package/dist/link-budget.js +9 -7
- package/dist/link-budget.js.map +1 -1
- package/dist/member-redemption.d.ts +15 -9
- package/dist/member-redemption.js +13 -8
- package/dist/member-redemption.js.map +1 -1
- package/dist/membership-view.d.ts +91 -0
- package/dist/membership-view.js +110 -0
- package/dist/membership-view.js.map +1 -0
- package/dist/owner-grant-protocol.d.ts +11 -7
- package/dist/owner-grant-protocol.js +31 -17
- package/dist/owner-grant-protocol.js.map +1 -1
- package/dist/peer-addr-book.d.ts +10 -1
- package/dist/peer-addr-book.js +4 -3
- package/dist/peer-addr-book.js.map +1 -1
- package/dist/peer-join-backfill.d.ts +89 -20
- package/dist/peer-join-backfill.js +265 -108
- package/dist/peer-join-backfill.js.map +1 -1
- package/dist/pending-join-runner.d.ts +48 -18
- package/dist/pending-join-runner.js +74 -56
- package/dist/pending-join-runner.js.map +1 -1
- package/dist/seed-bootstrap.d.ts +47 -40
- package/dist/seed-bootstrap.js +66 -61
- package/dist/seed-bootstrap.js.map +1 -1
- package/dist/strand-database.d.ts +23 -2
- package/dist/strand-database.js +26 -6
- package/dist/strand-database.js.map +1 -1
- package/dist/strand-first-sync-gate.d.ts +176 -89
- package/dist/strand-first-sync-gate.js +288 -122
- package/dist/strand-first-sync-gate.js.map +1 -1
- package/dist/strand-formation-deadlines.d.ts +9 -9
- package/dist/strand-formation-deadlines.js +9 -9
- package/dist/strand-formation-manager.d.ts +23 -17
- package/dist/strand-formation-manager.js +46 -44
- package/dist/strand-formation-manager.js.map +1 -1
- package/dist/strand-formation-protocol.d.ts +5 -8
- package/dist/strand-formation-protocol.js +101 -16
- package/dist/strand-formation-protocol.js.map +1 -1
- package/dist/strand-instance-manager.d.ts +26 -17
- package/dist/strand-instance-manager.js +119 -50
- package/dist/strand-instance-manager.js.map +1 -1
- package/dist/strand-membership-reconciler.d.ts +16 -13
- package/dist/strand-membership-reconciler.js +30 -22
- package/dist/strand-membership-reconciler.js.map +1 -1
- package/dist/strand-membership-writer.d.ts +40 -28
- package/dist/strand-membership-writer.js +212 -153
- package/dist/strand-membership-writer.js.map +1 -1
- package/dist/strand-move.js +5 -2
- package/dist/strand-move.js.map +1 -1
- package/dist/strand-revocation-enforcer.d.ts +134 -60
- package/dist/strand-revocation-enforcer.js +144 -71
- package/dist/strand-revocation-enforcer.js.map +1 -1
- package/dist/strand-solicitation.d.ts +50 -39
- package/dist/strand-solicitation.js +10 -3
- package/dist/strand-solicitation.js.map +1 -1
- package/dist/strand-watcher.d.ts +80 -13
- package/dist/strand-watcher.js +152 -48
- package/dist/strand-watcher.js.map +1 -1
- package/dist/table-change-watch.d.ts +63 -0
- package/dist/table-change-watch.js +209 -0
- package/dist/table-change-watch.js.map +1 -0
- package/dist/timeout-scheduler.d.ts +4 -3
- package/dist/timeout-scheduler.js +4 -3
- package/dist/timeout-scheduler.js.map +1 -1
- package/dist/types.d.ts +90 -49
- package/dist/types.js +4 -4
- package/dist/types.js.map +1 -1
- package/package.json +8 -8
- package/src/cadre-invite-protocol.ts +119 -31
- package/src/cadre-node.ts +1282 -584
- package/src/control-cohort.ts +86 -9
- package/src/control-database.ts +848 -403
- package/src/control-formation-recorder.ts +359 -353
- package/src/control-protocol-guard.ts +16 -2
- package/src/control-schema-version.ts +101 -51
- package/src/control-schema.ts +16 -16
- package/src/control-write-retry.ts +91 -19
- package/src/delegate-admission.ts +351 -351
- package/src/enrolled-machine-store.ts +3 -3
- package/src/formation-token.ts +63 -0
- package/src/index.ts +30 -17
- package/src/joined-strand-store.ts +69 -24
- package/src/link-budget.ts +9 -7
- package/src/member-redemption.ts +20 -13
- package/src/membership-view.ts +144 -0
- package/src/owner-grant-protocol.ts +33 -18
- package/src/peer-addr-book.ts +4 -3
- package/src/peer-join-backfill.ts +971 -761
- package/src/pending-join-runner.ts +76 -53
- package/src/seed-bootstrap.ts +72 -67
- package/src/strand-database.ts +352 -322
- package/src/strand-first-sync-gate.ts +334 -134
- package/src/strand-formation-deadlines.ts +227 -227
- package/src/strand-formation-manager.ts +731 -725
- package/src/strand-formation-protocol.ts +122 -24
- package/src/strand-instance-manager.ts +2092 -2026
- package/src/strand-membership-reconciler.ts +33 -25
- package/src/strand-membership-writer.ts +2093 -2019
- package/src/strand-move.ts +5 -2
- package/src/strand-revocation-enforcer.ts +740 -654
- package/src/strand-solicitation.ts +61 -34
- package/src/strand-watcher.ts +175 -56
- package/src/table-change-watch.ts +285 -0
- package/src/timeout-scheduler.ts +29 -28
- package/src/types.ts +89 -51
package/src/cadre-node.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import debug from 'debug';
|
|
2
2
|
import { toString as uint8ArrayToString, fromString as uint8ArrayFromString } from 'uint8arrays';
|
|
3
|
-
import type { Libp2p, PeerId, PrivateKey } from '@libp2p/interface';
|
|
3
|
+
import type { Connection, IdentifyResult, Libp2p, PeerId, PrivateKey } from '@libp2p/interface';
|
|
4
4
|
import { peerIdFromString, peerIdFromPrivateKey } from '@libp2p/peer-id';
|
|
5
5
|
import { generateKeyPair } from '@libp2p/crypto/keys';
|
|
6
6
|
import { createLibp2pNode, type IRawStorage } from '@optimystic/db-p2p';
|
|
@@ -19,6 +19,7 @@ import type {
|
|
|
19
19
|
SAppConfig,
|
|
20
20
|
CadreNodeEvents,
|
|
21
21
|
ControlNetworkSeed,
|
|
22
|
+
SeedDialTarget,
|
|
22
23
|
ApplySeedResult,
|
|
23
24
|
SeedAckMessage,
|
|
24
25
|
SeedDeliveryTarget,
|
|
@@ -94,7 +95,7 @@ import {
|
|
|
94
95
|
type JoinedStrandStore,
|
|
95
96
|
type PartyJoinedStrandLedger
|
|
96
97
|
} from './joined-strand-store.js';
|
|
97
|
-
import { mergePeerAddrs, groupAddrsByPeerId, type MergeAddrsResult } from './peer-addr-book.js';
|
|
98
|
+
import { addrKey, mergePeerAddrs, groupAddrsByPeerId, type MergeAddrsResult } from './peer-addr-book.js';
|
|
98
99
|
import { strandFretPeerAddrs } from './strand-fret-addrs.js';
|
|
99
100
|
import { verifyCadrePeerVoucher, verifyInvitationAdmission, verifyRevocationSigner } from './peer-authorization.js';
|
|
100
101
|
import { deriveOwnerAnchor } from './owner-anchor-sync.js';
|
|
@@ -116,17 +117,22 @@ import {
|
|
|
116
117
|
verifyDeviceTokenSignature,
|
|
117
118
|
isPushPlatform
|
|
118
119
|
} from './device-token.js';
|
|
119
|
-
import { StrandWatcher, type StrandQueryable, type SAppIdLookup } from './strand-watcher.js';
|
|
120
|
+
import { StrandWatcher, DEFAULT_STRAND_WATCH_BACKSTOP_MS, type StrandQueryable, type SAppIdLookup } from './strand-watcher.js';
|
|
120
121
|
import { StrandInstanceManager, liveStrandStatus } from './strand-instance-manager.js';
|
|
121
122
|
import { PeerJoinBackfill } from './peer-join-backfill.js';
|
|
122
123
|
import { deriveCohortMembers } from './strand-cohort.js';
|
|
123
124
|
import type { CohortPeerRow } from './strand-cohort.js';
|
|
124
125
|
import {
|
|
125
126
|
selectControlCohortDials,
|
|
127
|
+
nextReconcileWakeMs,
|
|
128
|
+
CLAIM_RESTART_REDIAL_STEP_MS,
|
|
129
|
+
CONTROL_COHORT_REDIAL_MS,
|
|
126
130
|
DEFAULT_CONTROL_COHORT_RECONCILE_MS,
|
|
127
131
|
DEFAULT_CONTROL_COHORT_TARGET_DEGREE,
|
|
128
132
|
type ControlCohortReconcileResult
|
|
129
133
|
} from './control-cohort.js';
|
|
134
|
+
import { scheduleRuns, type TableChangeWatch } from './table-change-watch.js';
|
|
135
|
+
import { MEMBERSHIP_VIEW_RECONNECT_RELOAD_MS } from './membership-view.js';
|
|
130
136
|
import {
|
|
131
137
|
dialPeerAddrs,
|
|
132
138
|
SelfRelayOnlyError,
|
|
@@ -138,11 +144,12 @@ import {
|
|
|
138
144
|
import { ADMISSION_DECISION_TIMEOUT_MS, declaredCohortReadDeadlineMs, optimysticDialLimits, peerJoinPushBudget, relayReservationBudgetMs, relayedDialBudgetMs, relayedRequestBudgetMs, resolveLinkRoundTripMs } from './link-budget.js';
|
|
139
145
|
import { EnrollmentService } from './enrollment.js';
|
|
140
146
|
import { HibernationManager, type HibernationCallbacks } from './hibernation-manager.js';
|
|
141
|
-
import { ControlDatabase, generateStampId, isPendingJoinConflict, isStrandIdConflict, IssuedRowNotHereYetError, pendingJoinId, type JoinRequestFields, type RevokedRowRef } from './control-database.js';
|
|
147
|
+
import { ControlDatabase, generateStampId, isPendingJoinConflict, isStrandIdConflict, IssuedRowNotHereYetError, pendingJoinId, type CadrePeerRetouchOutcome, type FoundingInvite, type JoinRequestFields, type RevokedRowRef } from './control-database.js';
|
|
142
148
|
import {
|
|
143
149
|
CadreInviteHandler,
|
|
144
150
|
encodeCadreInvitation,
|
|
145
151
|
redeemInPasses,
|
|
152
|
+
sanitizeAddrs,
|
|
146
153
|
signRedeemRequest,
|
|
147
154
|
type CadreInvitation
|
|
148
155
|
} from './cadre-invite-protocol.js';
|
|
@@ -164,11 +171,12 @@ import {
|
|
|
164
171
|
requestedPendingJoin
|
|
165
172
|
} from './pending-join-runner.js';
|
|
166
173
|
import type { ControlRetryAbandonment } from './control-retry.js';
|
|
167
|
-
import { SeedBootstrapService, mergeSeedPeers, seedDigest, type SeedEventCallbacks, type SeedBootstrapConfig } from './seed-bootstrap.js';
|
|
174
|
+
import { SeedBootstrapService, mergeSeedPeers, seedDialTargets, seedDigest, type SeedEventCallbacks, type SeedBootstrapConfig } from './seed-bootstrap.js';
|
|
168
175
|
import { claimSecretTrustPolicy, type SeedTrustPolicy } from './seed-trust-policy.js';
|
|
169
176
|
import { ClaimRefusedError, claimProof, parseClaimSecret } from './claim-proof.js';
|
|
170
177
|
import {
|
|
171
178
|
StrandSolicitationService,
|
|
179
|
+
type BoundHostStrand,
|
|
172
180
|
type StrandSolicitationServiceOptions
|
|
173
181
|
} from './strand-solicitation.js';
|
|
174
182
|
import { ControlFormationUsageRecorder } from './control-formation-recorder.js';
|
|
@@ -183,7 +191,7 @@ import {
|
|
|
183
191
|
UnauthorizedReservationBudget,
|
|
184
192
|
type InboundConnectionVerdict
|
|
185
193
|
} from './membership-connection-gater.js';
|
|
186
|
-
import { controlNetworkName, installControlProtocolGuard } from './control-protocol-guard.js';
|
|
194
|
+
import { controlNetworkName, installControlProtocolGuard, servesControlDatabase } from './control-protocol-guard.js';
|
|
187
195
|
import { StrandWakeService, dialWake } from './strand-wake-protocol.js';
|
|
188
196
|
import { StrandAddrService, collectStrandAddrs, type StrandAddrPeer, type StrandAddrOutcome, type StrandAddrCollection } from './strand-addr-protocol.js';
|
|
189
197
|
import {
|
|
@@ -226,8 +234,8 @@ const timing = debug('sereus:cadre:timing');
|
|
|
226
234
|
*
|
|
227
235
|
* Ten minutes sits comfortably inside the peerStore's one-hour address expiry
|
|
228
236
|
* (`MAX_ADDRESS_AGE`, see `peer-addr-book.ts`) with headroom for several missed
|
|
229
|
-
* passes, and far above the
|
|
230
|
-
* strand-addr RPC fan-out stays cheap.
|
|
237
|
+
* passes, and far above the reconcile pass's 2-minute backstop the refresh rides
|
|
238
|
+
* on, so the strand-addr RPC fan-out stays cheap.
|
|
231
239
|
*/
|
|
232
240
|
export const STRAND_PEER_ADDR_REFRESH_MS = 10 * 60 * 1000;
|
|
233
241
|
|
|
@@ -237,11 +245,29 @@ export const STRAND_PEER_ADDR_REFRESH_MS = 10 * 60 * 1000;
|
|
|
237
245
|
* phone asking before its `CadrePeer` row has replicated to the sibling is refused
|
|
238
246
|
* and its delegate grant goes unrecorded, and waiting the full
|
|
239
247
|
* {@link STRAND_PEER_ADDR_REFRESH_MS} for the next try is the delay this bounds.
|
|
240
|
-
*
|
|
241
|
-
*
|
|
248
|
+
* A pass arms its own wake for the earliest due time it leaves behind
|
|
249
|
+
* (`nextReconcileWakeMs`), so a sibling that keeps failing costs one timed-out
|
|
250
|
+
* RPC a minute, with no pass in between.
|
|
242
251
|
*/
|
|
243
252
|
export const STRAND_PEER_ADDR_RETRY_MS = 60 * 1000;
|
|
244
253
|
|
|
254
|
+
/**
|
|
255
|
+
* What one reconcile pass did, for the wrapper that arms the next wake; callers get only
|
|
256
|
+
* its `dialed` ({@link ControlCohortReconcileResult}).
|
|
257
|
+
*/
|
|
258
|
+
interface ControlCohortPassOutcome extends ControlCohortReconcileResult {
|
|
259
|
+
/**
|
|
260
|
+
* The pass dialed a selected sibling, or a cold-start bootstrap peer, and the dial failed.
|
|
261
|
+
* A sibling the pass could not dial at all — no dialable address, or every address relays
|
|
262
|
+
* through this node — does not count: a retry would dial nothing, and every way it gains
|
|
263
|
+
* an address (its record replicating in, a connection it opens) starts a pass of its own.
|
|
264
|
+
*/
|
|
265
|
+
unconnected: boolean;
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/** How {@link CadreNode.dialControlSibling} ended: connected, a failed dial, or nothing to dial. */
|
|
269
|
+
type SiblingDialOutcome = 'connected' | 'failed' | 'undialable';
|
|
270
|
+
|
|
245
271
|
/**
|
|
246
272
|
* Default lifetime of an owner grant ({@link CadreNode.createOwnerGrant}): whoever holds the
|
|
247
273
|
* bundle can become an owner, so it is a bearer credential for admin rights and lives only
|
|
@@ -441,6 +467,19 @@ function requireMatchingStrandRow(live: StrandRow, desired: StrandRow, situation
|
|
|
441
467
|
return live;
|
|
442
468
|
}
|
|
443
469
|
|
|
470
|
+
/**
|
|
471
|
+
* What a founding's control write resolved: the live `Strand` row, and the party key the write
|
|
472
|
+
* seated for a closed strand. No key when it adopted a stored row or a stored key; the launch
|
|
473
|
+
* then reads the stored one.
|
|
474
|
+
*/
|
|
475
|
+
interface FoundingWrite {
|
|
476
|
+
strandRow: StrandRow;
|
|
477
|
+
partyKey?: string;
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
/** The publish half of `foundStrand`, handed to its start half. */
|
|
481
|
+
type FoundingPublish = FoundingWrite & Pick<FoundStrandResult, 'founded'>;
|
|
482
|
+
|
|
444
483
|
/**
|
|
445
484
|
* Await `work` unless `signal` aborts first: `true` once `work` resolves, `false` when the
|
|
446
485
|
* signal aborted before it did (already at the call, or during the wait). A rejection of
|
|
@@ -663,8 +702,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
663
702
|
*
|
|
664
703
|
* Cleared by {@link cleanup} only, deliberately NOT by {@link detachStrand}: the watcher's
|
|
665
704
|
* filter rejection stops a replica through that very method, and forgetting the id there
|
|
666
|
-
* would have the next
|
|
667
|
-
* relaunch the replica —
|
|
705
|
+
* would have the next watcher run find it unknown again, re-admit the strand provisionally
|
|
706
|
+
* and relaunch the replica — on every other run, forever. A strand id's sApp never changes,
|
|
668
707
|
* and the map is bounded by the strands this party has published, like
|
|
669
708
|
* {@link discoveredStrands}.
|
|
670
709
|
*/
|
|
@@ -695,19 +734,24 @@ export class CadreNode implements SAppIdLookup {
|
|
|
695
734
|
* wait that ends in mutual denial (the upstream gate is fail-closed on
|
|
696
735
|
* timeout, unlike the fail-open connection gater). Optimystic's predicate is
|
|
697
736
|
* synchronous by contract. So the set is refreshed OUT OF BAND instead — after
|
|
698
|
-
* start,
|
|
699
|
-
*
|
|
700
|
-
*
|
|
701
|
-
*
|
|
737
|
+
* start, at the head of every {@link reconcileControlCohort} pass, whenever a
|
|
738
|
+
* membership table changes (the gate watch {@link start} takes through
|
|
739
|
+
* `ControlDatabase.watchMembershipTables`, which hears a local commit and a change
|
|
740
|
+
* replicated into this node's storage alike, within a 50 ms window), and immediately
|
|
741
|
+
* after every LOCAL membership mutation so a just-vouched peer is admitted
|
|
742
|
+
* before the writer's call returns. That last refresh is AUTOMATIC: the
|
|
743
|
+
* control database notifies {@link refreshMembershipGate} after every
|
|
702
744
|
* committed `CadrePeer` write (see `ControlDatabase.mutateCadrePeer`), so no
|
|
703
|
-
* writer has to remember. A change
|
|
704
|
-
*
|
|
705
|
-
*
|
|
706
|
-
*
|
|
707
|
-
*
|
|
708
|
-
*
|
|
709
|
-
*
|
|
710
|
-
*
|
|
745
|
+
* writer has to remember. A change this node's storage never hears of — a
|
|
746
|
+
* machine outside the block's cohort on a party larger than the replication
|
|
747
|
+
* breadth — is picked up at the pass's 2-minute backstop: bounded staleness,
|
|
748
|
+
* acceptable because this gate is defense in depth: rows an unadmitted peer
|
|
749
|
+
* manages to write are still disbelieved at read time by the voucher-anchored
|
|
750
|
+
* predicate. For cohort choice the same window means a member whose row
|
|
751
|
+
* arrived by replication is left out of this node's control-block cohorts
|
|
752
|
+
* until that refresh. Optimystic reads the predicate live on every lookup, so
|
|
753
|
+
* replacing this set is the whole re-evaluation; there is no per-peer
|
|
754
|
+
* reclassify call to make.
|
|
711
755
|
*/
|
|
712
756
|
private authorizedControlPeers: Set<string> = new Set();
|
|
713
757
|
|
|
@@ -771,9 +815,9 @@ export class CadreNode implements SAppIdLookup {
|
|
|
771
815
|
* When this node last ANNOUNCED a delegate, keyed `<targetPeerId>\n<strandId>`
|
|
772
816
|
* (the peer announced TO, not the delegate). Throttles
|
|
773
817
|
* {@link refreshDelegateGrants} to once per `DELEGATE_GRANT_TTL_MS / 2` per
|
|
774
|
-
* (relay, strand) so
|
|
775
|
-
* chatter; the launch/resume announce passes record here too.
|
|
776
|
-
* strand is no longer running are pruned on each reconcile pass.
|
|
818
|
+
* (relay, strand) so a run of change-driven reconcile passes never becomes
|
|
819
|
+
* per-pass RPC chatter; the launch/resume announce passes record here too.
|
|
820
|
+
* Keys whose strand is no longer running are pruned on each reconcile pass.
|
|
777
821
|
*/
|
|
778
822
|
private readonly delegateAnnounceAt = new Map<string, number>();
|
|
779
823
|
|
|
@@ -802,12 +846,13 @@ export class CadreNode implements SAppIdLookup {
|
|
|
802
846
|
* strand, keyed `peerStrandKey(siblingControlPeerId, strandId)` → epoch ms; a
|
|
803
847
|
* missing key is due. Bounds {@link refreshStrandPeerAddrs}'s fan-out per
|
|
804
848
|
* (sibling, strand) rather than per strand, so a sibling that connects after a
|
|
805
|
-
* pass is asked on the
|
|
806
|
-
* sibling's answer set. An answer (even an empty one) makes the sibling
|
|
807
|
-
* in {@link STRAND_PEER_ADDR_REFRESH_MS}; no answer, in
|
|
808
|
-
* {@link STRAND_PEER_ADDR_RETRY_MS}
|
|
809
|
-
*
|
|
810
|
-
*
|
|
849
|
+
* pass is asked on the pass its connection starts instead of waiting out a stamp
|
|
850
|
+
* another sibling's answer set. An answer (even an empty one) makes the sibling
|
|
851
|
+
* due again in {@link STRAND_PEER_ADDR_REFRESH_MS}; no answer, in
|
|
852
|
+
* {@link STRAND_PEER_ADDR_RETRY_MS}, and each pass arms its own wake for the
|
|
853
|
+
* earliest due time it leaves behind (`nextReconcileWakeMs`). Keys whose strand
|
|
854
|
+
* stopped running or whose sibling is no longer connected are pruned every pass,
|
|
855
|
+
* so a resumed strand and a reconnected sibling are both asked at once.
|
|
811
856
|
*/
|
|
812
857
|
private readonly strandAddrAskDueAt = new Map<string, number>();
|
|
813
858
|
|
|
@@ -893,6 +938,19 @@ export class CadreNode implements SAppIdLookup {
|
|
|
893
938
|
*/
|
|
894
939
|
private readonly pendingMembershipInvites = new Map<string, StrandMembershipInvite>();
|
|
895
940
|
|
|
941
|
+
/**
|
|
942
|
+
* The party key {@link adoptFormationMembershipInvite} just seated or read, by strand id,
|
|
943
|
+
* so the launch that follows a join does not read the `StrandPartyKey` row again. Taken
|
|
944
|
+
* (deleted) by the {@link resolveStrandPartyKey} that uses it, so any later launch reads
|
|
945
|
+
* the row; dropped by {@link detachStrand}. The joiner's counterpart of the founder's
|
|
946
|
+
* `StrandConfig.partyMemberPrivateKey`, without the app carrying the secret.
|
|
947
|
+
*
|
|
948
|
+
* NOTE: an entry is not dropped when the row is removed, because no `CadreNode` path removes
|
|
949
|
+
* it; an embedder calling `getControlDatabase().deleteStrandPartyKey` between a join and its
|
|
950
|
+
* launch would get the deleted key. If a node-level rotation is added, drop the entry there.
|
|
951
|
+
*/
|
|
952
|
+
private readonly formedPartyKeys = new Map<string, string>();
|
|
953
|
+
|
|
896
954
|
/**
|
|
897
955
|
* Invite keys of the membership invitations this process has staged, by
|
|
898
956
|
* {@link adoptFormationMembershipInvite} or from a `JoinSuccess` row
|
|
@@ -992,7 +1050,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
992
1050
|
|
|
993
1051
|
/**
|
|
994
1052
|
* This session's joined strands: {@link joinedStrandStore} and the party-wide
|
|
995
|
-
* `JoinedStrand` table, unioned with the control rows each watcher
|
|
1053
|
+
* `JoinedStrand` table, unioned with the control rows each watcher run, published and
|
|
996
1054
|
* removed by the reconcile pass. Per SESSION, unlike the store — rebuilt at every
|
|
997
1055
|
* {@link start}, which is what ends a revoked join's "keep offering until this session
|
|
998
1056
|
* ends" (see `JoinedStrandSession.forgetAfterThisSession`).
|
|
@@ -1025,17 +1083,41 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1025
1083
|
/** Listener that re-publishes the self record when reachable addresses change. */
|
|
1026
1084
|
private selfPeerUpdateHandler: (() => void) | null = null;
|
|
1027
1085
|
/**
|
|
1028
|
-
*
|
|
1029
|
-
*
|
|
1030
|
-
*
|
|
1031
|
-
* {@link
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
*
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1086
|
+
* The reconcile pass's schedule (see {@link reconcileControlCohort}): single-flight
|
|
1087
|
+
* runs on `table-change-watch.ts`'s rule, with no table subscription of its own — the
|
|
1088
|
+
* gate watch ({@link membershipGateWatch}), the control connection listeners,
|
|
1089
|
+
* `self:peer:update`, and the pass's own wake ({@link armReconcileWake}) `trigger` it,
|
|
1090
|
+
* and its backstop (`network.controlCohort.reconcileMs`, default
|
|
1091
|
+
* `DEFAULT_CONTROL_COHORT_RECONCILE_MS`) runs it when nothing has. Built alongside
|
|
1092
|
+
* {@link recordRefreshTimer} in {@link startRecordRefresh}, about 1 s after start, and
|
|
1093
|
+
* disposed in {@link stopRecordRefresh}; its timers never keep the loop alive.
|
|
1094
|
+
*/
|
|
1095
|
+
private controlReconcileRuns: TableChangeWatch | null = null;
|
|
1096
|
+
/**
|
|
1097
|
+
* The subscription on the membership tables (`ControlDatabase.watchMembershipTables`) that
|
|
1098
|
+
* re-materializes the membership gate and triggers {@link controlReconcileRuns} on every
|
|
1099
|
+
* change ({@link onMembershipGateTablesChanged}).
|
|
1100
|
+
* Taken in {@link start} right after the membership listener is wired, disposed in
|
|
1101
|
+
* {@link cleanup} before the control database closes.
|
|
1102
|
+
*/
|
|
1103
|
+
private membershipGateWatch: TableChangeWatch | null = null;
|
|
1104
|
+
/** The second membership-view drop after a sibling connects ({@link armMembershipViewReload}); cleared in {@link stopRecordRefresh}. */
|
|
1105
|
+
private membershipViewReloadTimer: ReturnType<typeof setTimeout> | null = null;
|
|
1106
|
+
/**
|
|
1107
|
+
* Consecutive passes whose dial of a selected sibling or a cold-start bootstrap peer
|
|
1108
|
+
* failed (`ControlCohortPassOutcome.unconnected`), for the redial ladder
|
|
1109
|
+
* ({@link armReconcileWake}); 0 after a pass none of whose dials did. Per session:
|
|
1110
|
+
* reset in {@link stopRecordRefresh}.
|
|
1111
|
+
*/
|
|
1112
|
+
private reconcileRedialStreak = 0;
|
|
1113
|
+
/** Peers a {@link redialClaimedNode} loop is running for, so a repeated claim starts no second one. */
|
|
1114
|
+
private readonly claimRedialPeers = new Set<string>();
|
|
1115
|
+
/**
|
|
1116
|
+
* Single-flight guard for {@link reconcileControlCohort}. An explicit call, the
|
|
1117
|
+
* `self:peer:update` trigger and a scheduled run can fire close together; collapsing
|
|
1118
|
+
* concurrent passes into one in-flight run prevents two passes from double-dialing the
|
|
1119
|
+
* same siblings (mirrors {@link registerSelfInFlight}). A scheduled run waits for this
|
|
1120
|
+
* pass and then starts its own rather than joining it.
|
|
1039
1121
|
*/
|
|
1040
1122
|
private reconcileControlCohortInFlight: Promise<ControlCohortReconcileResult> | null = null;
|
|
1041
1123
|
/**
|
|
@@ -1058,20 +1140,28 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1058
1140
|
* machine in the party is already discarding this node's address as stale.
|
|
1059
1141
|
*/
|
|
1060
1142
|
private lastSelfRecordPublishAt: number | null = null;
|
|
1143
|
+
/**
|
|
1144
|
+
* The newest record this process signed for its own `CadrePeer` row: written by either path
|
|
1145
|
+
* of {@link publishSelfRecord}, or handed to a member to write by {@link redeemCadreInvitation}.
|
|
1146
|
+
* What {@link publishSelfRecordUnlessCurrent} compares the row against. Per session, cleared
|
|
1147
|
+
* with {@link lastSelfRecordPublishAt}: a row carried over from an earlier process is never
|
|
1148
|
+
* taken as current.
|
|
1149
|
+
*/
|
|
1150
|
+
private lastSignedSelfRecord: PeerAddressRecord | null = null;
|
|
1061
1151
|
/** Say-once latch for {@link escalateIfSelfRecordStale}; re-armed by the next publish. */
|
|
1062
1152
|
private selfRecordStaleWarned = false;
|
|
1063
1153
|
|
|
1064
1154
|
// ── Write-while-alone re-replication (control-write-ensure-replicated) ──────
|
|
1065
1155
|
/**
|
|
1066
1156
|
* Re-replication queue for owner `authorizePeer` writes that committed while
|
|
1067
|
-
* this node was alone
|
|
1068
|
-
*
|
|
1069
|
-
*
|
|
1157
|
+
* this node was alone ({@link committedAlone}) — the Optimystic commit was
|
|
1158
|
+
* local-only (the block's cohort was this node) and never broadcast. Maps the
|
|
1159
|
+
* affected subject peerId → 'authorize', to
|
|
1070
1160
|
* re-issue once the cohort grows. Removals are NOT tracked here — a delete
|
|
1071
1161
|
* leaves a `Revocation` tombstone, tracked by {@link pendingRevocations}; a
|
|
1072
1162
|
* removePeer also clears any queued authorize for the same subject (the row is
|
|
1073
|
-
* gone, so re-issuing the insert would be wrong). Drained
|
|
1074
|
-
*
|
|
1163
|
+
* gone, so re-issuing the insert would be wrong). Drained when a sibling becomes
|
|
1164
|
+
* reachable ({@link updateControlSiblingReachability}) by {@link drainPendingControlReplication}.
|
|
1075
1165
|
*/
|
|
1076
1166
|
private pendingPeerWrites: Map<string, 'authorize'> = new Map();
|
|
1077
1167
|
/**
|
|
@@ -1095,14 +1185,6 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1095
1185
|
* exactly a "removals from before this lifetime" case).
|
|
1096
1186
|
*/
|
|
1097
1187
|
private reissuedHeldRevocations = false;
|
|
1098
|
-
/**
|
|
1099
|
-
* Whether this process has seen the singleton `Revocation` ledger marker filed — its own
|
|
1100
|
-
* `'opened'` or an `'already-open'` answer, in {@link openRevocationLedgerIfDue}. A
|
|
1101
|
-
* record of work done, not a cached authorization answer: the marker can never be
|
|
1102
|
-
* deleted (`NoDelete`), so this can only go stale if the database itself is replaced,
|
|
1103
|
-
* which is why stop() clears it with the other per-process replication state.
|
|
1104
|
-
*/
|
|
1105
|
-
private revocationLedgerOpened = false;
|
|
1106
1188
|
/** This node's own `CadrePeer` self-write committed local-only (re-touched on growth). */
|
|
1107
1189
|
private pendingSelfPeerWrite = false;
|
|
1108
1190
|
/** This node's own `DeviceToken` self-write committed local-only (re-touched on growth). */
|
|
@@ -1117,15 +1199,25 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1117
1199
|
/** Single-flight guard for the re-replication drain (mirrors {@link registerSelfInFlight}). */
|
|
1118
1200
|
private drainControlReplicationInFlight: Promise<void> | null = null;
|
|
1119
1201
|
/**
|
|
1120
|
-
*
|
|
1121
|
-
*
|
|
1122
|
-
*
|
|
1202
|
+
* Connected control peers whose identify says they serve the control database
|
|
1203
|
+
* ({@link onControlPeerIdentified}): the peers Optimystic can count into a control block's
|
|
1204
|
+
* cohort, before its membership check. A peer joins on its first `peer:identify` of a
|
|
1205
|
+
* connected period, not on `connection:open`: until it identifies, every control block's
|
|
1206
|
+
* cohort is this node alone. It leaves when its last connection closes. Read through
|
|
1207
|
+
* {@link hasReachableControlSibling}, which applies the membership check.
|
|
1123
1208
|
*/
|
|
1124
|
-
private
|
|
1125
|
-
/**
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1209
|
+
private readonly identifiedControlPeers = new Set<string>();
|
|
1210
|
+
/**
|
|
1211
|
+
* {@link hasReachableControlSibling} at its last evaluation, so the drain fires on the
|
|
1212
|
+
* false→true edge only (the earliest point a re-issue reaches the cohort), not on every later
|
|
1213
|
+
* identify. Re-armed to false when no reachable sibling remains
|
|
1214
|
+
* ({@link updateControlSiblingReachability}).
|
|
1215
|
+
*/
|
|
1216
|
+
private controlSiblingReachable = false;
|
|
1217
|
+
/** `peer:identify` listener: {@link onControlPeerIdentified} (teardown in {@link stopRecordRefresh}). */
|
|
1218
|
+
private controlPeerIdentifyHandler: ((event: CustomEvent<IdentifyResult>) => void) | null = null;
|
|
1219
|
+
/** `connection:close` listener: {@link onControlConnectionClosed} (teardown in {@link stopRecordRefresh}). */
|
|
1220
|
+
private controlConnectionCloseHandler: ((event: CustomEvent<Connection>) => void) | null = null;
|
|
1129
1221
|
|
|
1130
1222
|
constructor(config: CadreNodeConfig) {
|
|
1131
1223
|
this.config = config;
|
|
@@ -1318,7 +1410,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1318
1410
|
// loaded before the first reconcile pass could consult them.
|
|
1319
1411
|
this.initializeBootstrapPeerStore();
|
|
1320
1412
|
|
|
1321
|
-
// The remembered cross-party joins, which the strand watcher built below
|
|
1413
|
+
// The remembered cross-party joins, which the strand watcher built below reads
|
|
1322
1414
|
// beside the control rows. Before network bring-up, so a mis-scoped injected store
|
|
1323
1415
|
// fails closed like the two above.
|
|
1324
1416
|
this.initializeJoinedStrandStore();
|
|
@@ -1404,6 +1496,21 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1404
1496
|
// rest of start() and the refresh early-returns while it is — a write in that
|
|
1405
1497
|
// window is covered by start's own pass below.
|
|
1406
1498
|
this.controlDatabase.setMembershipChangeListener((reason) => this.refreshMembershipGate(reason));
|
|
1499
|
+
// The gate watch: a change to any membership table, replicated in or committed here,
|
|
1500
|
+
// re-materializes the gate and starts a reconcile pass. Separate from the pass so a
|
|
1501
|
+
// replicated revocation does not wait behind a pass spending its dial budget on an
|
|
1502
|
+
// unreachable sibling. The watch handler only arms a timer, so nothing runs under the
|
|
1503
|
+
// write lock the membership listener is called with; `onChange` runs from that timer.
|
|
1504
|
+
//
|
|
1505
|
+
// NOTE: a table this `Database` has not yet touched hears local commits only
|
|
1506
|
+
// (`table-change-watch.ts`). The start refresh reads `CadrePeer`, `OwnerKey` and
|
|
1507
|
+
// `Revocation`; the four invitation and grant tables are first read when some row is
|
|
1508
|
+
// admitted through one, and a load the committed-read path served is not kept and
|
|
1509
|
+
// arms no bridge, so a chain row that replicates in before then waits for the next
|
|
1510
|
+
// change on a touched table or the backstop. Nothing depends on a chain row until a
|
|
1511
|
+
// `CadrePeer` or `OwnerKey` row refers to it, and that row is itself such a change.
|
|
1512
|
+
this.membershipGateWatch = this.controlDatabase.watchMembershipTables(
|
|
1513
|
+
() => this.onMembershipGateTablesChanged(), { label: 'membership-gate' });
|
|
1407
1514
|
// Committed-delete seam → write-while-alone revocation queue. Same wiring
|
|
1408
1515
|
// window argument as the membership listener above.
|
|
1409
1516
|
this.controlDatabase.setGuardedDeleteListener((revocation) => this.noteGuardedDelete(revocation));
|
|
@@ -1423,7 +1530,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1423
1530
|
onStrandRemoved: async (strandId) => this.handleStrandRemoved(strandId)
|
|
1424
1531
|
},
|
|
1425
1532
|
this.config.strandFilter ?? { mode: 'all' },
|
|
1426
|
-
this.config.strandWatchInterval ??
|
|
1533
|
+
this.config.strandWatchInterval ?? DEFAULT_STRAND_WATCH_BACKSTOP_MS,
|
|
1427
1534
|
this // CadreNode implements SAppIdLookup
|
|
1428
1535
|
);
|
|
1429
1536
|
|
|
@@ -1506,8 +1613,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1506
1613
|
log('CadreNode started successfully');
|
|
1507
1614
|
|
|
1508
1615
|
// Wire the control-connection growth listeners immediately (not via the
|
|
1509
|
-
// delayed refresh path) so no early `
|
|
1510
|
-
// write-while-alone re-replication drain must fire
|
|
1616
|
+
// delayed refresh path) so no early `peer:identify` is missed — the
|
|
1617
|
+
// write-while-alone re-replication drain must fire when the first sibling is reachable.
|
|
1511
1618
|
this.wireControlConnectionListeners();
|
|
1512
1619
|
|
|
1513
1620
|
// Arm the control network's peer-join block catch-up now that the control
|
|
@@ -1898,7 +2005,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1898
2005
|
);
|
|
1899
2006
|
}
|
|
1900
2007
|
this.joinedStrandStore = store;
|
|
1901
|
-
this.joinedStrands = new JoinedStrandSession(store, this.createPartyJoinedStrandLedger()
|
|
2008
|
+
this.joinedStrands = new JoinedStrandSession(store, this.createPartyJoinedStrandLedger(),
|
|
2009
|
+
() => this.strandWatcher?.refresh());
|
|
1902
2010
|
}
|
|
1903
2011
|
|
|
1904
2012
|
/**
|
|
@@ -2054,8 +2162,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2054
2162
|
|
|
2055
2163
|
/**
|
|
2056
2164
|
* Arm the CONTROL network's peer-join block catch-up: push every block in the
|
|
2057
|
-
* control database's own raw store to each
|
|
2058
|
-
*
|
|
2165
|
+
* control database's own raw store to each AUTHORIZED member on each
|
|
2166
|
+
* (re)connection, skipping what it is known to hold. This is what physically replicates
|
|
2059
2167
|
* control blocks committed while the writer was alone — the named
|
|
2060
2168
|
* collection-header blocks written once at genesis above all, whose revision
|
|
2061
2169
|
* never moves again, so no later commit ever carries them to a member that
|
|
@@ -2064,7 +2172,9 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2064
2172
|
* for peers it knew about before the restart).
|
|
2065
2173
|
*
|
|
2066
2174
|
* Unlike the per-strand instances (see `strand-instance-manager.ts`), pushes
|
|
2067
|
-
* are gated on {@link
|
|
2175
|
+
* are gated on the authorized-member snapshot ({@link authorizedControlPeers},
|
|
2176
|
+
* which {@link refreshAuthorizedControlPeers} materializes from the same rows and
|
|
2177
|
+
* rule {@link isAuthorizedMember} queries), judged at push time: the control
|
|
2068
2178
|
* network's inbound connection gate deliberately admits non-members in
|
|
2069
2179
|
* several states (seed delivery to an un-enrolled node, an open enrollment
|
|
2070
2180
|
* window, an outstanding invitation, configured bootstrap/relay peers), and
|
|
@@ -2080,23 +2190,19 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2080
2190
|
* The one non-member the gate admits is a device whose invitation
|
|
2081
2191
|
* redemption this member is admitting ({@link admitRedeemingDevice}).
|
|
2082
2192
|
*
|
|
2083
|
-
* No-ops (logged) when
|
|
2084
|
-
*
|
|
2085
|
-
* no way to dial.
|
|
2193
|
+
* No-ops (logged) when {@link controlBackfillStorage} answers nothing (disabled,
|
|
2194
|
+
* or no control storage configured) or the node exposes no key network — the
|
|
2195
|
+
* backfill would have nothing to read or no way to dial.
|
|
2086
2196
|
*/
|
|
2087
2197
|
private startControlBackfill(): void {
|
|
2088
|
-
if (
|
|
2198
|
+
if (!this.controlNode) {
|
|
2089
2199
|
return;
|
|
2090
2200
|
}
|
|
2091
|
-
|
|
2092
|
-
// idempotent (it returns the same wrapped store `buildControlNodeOptions` already
|
|
2093
|
-
// resolved), so the catch-up cannot be silently disarmed by a future reordering that
|
|
2094
|
-
// moves this call ahead of the one that populated the field.
|
|
2095
|
-
const storage = this.resolveControlStorage();
|
|
2201
|
+
const storage = this.controlBackfillStorage();
|
|
2096
2202
|
const keyNetwork = (this.controlNode as Libp2pNodeWithRepo).keyNetwork;
|
|
2097
2203
|
if (!storage || !keyNetwork) {
|
|
2098
2204
|
log('Control peer-join block catch-up is inert: %s',
|
|
2099
|
-
!storage ? 'no control storage configured' : 'control node exposes no keyNetwork');
|
|
2205
|
+
!storage ? 'disabled, or no control storage configured' : 'control node exposes no keyNetwork');
|
|
2100
2206
|
return;
|
|
2101
2207
|
}
|
|
2102
2208
|
const networkName = this.controlNetworkName();
|
|
@@ -2109,7 +2215,11 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2109
2215
|
// under — derived from the same networkName binding the node options
|
|
2110
2216
|
// used, never re-spelled here.
|
|
2111
2217
|
protocolPrefix: `/optimystic/${networkName}`,
|
|
2112
|
-
|
|
2218
|
+
reportBlockHeld: (this.controlNode as Libp2pNodeWithRepo).reportBlockHeld,
|
|
2219
|
+
// The membership snapshot, not {@link isControlNetworkMember}: that predicate admits
|
|
2220
|
+
// everyone while the snapshot is empty and admits bootstrap/relay peers outright, and
|
|
2221
|
+
// neither may receive the control store.
|
|
2222
|
+
authorizePeer: async (peerId) => this.admittingDevices.has(peerId) || this.authorizedControlPeers.has(peerId)
|
|
2113
2223
|
}, {
|
|
2114
2224
|
// A shorter settle window than the strand default (1000 ms): the control
|
|
2115
2225
|
// store is small (a party's membership — dozens of blocks), a re-push is
|
|
@@ -2165,8 +2275,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2165
2275
|
const release = this.countAdmittingDevice(peerId);
|
|
2166
2276
|
try {
|
|
2167
2277
|
const result = await backfill.forceCatchUpPeer(peerIdFromString(peerId));
|
|
2168
|
-
log('Device %s caught up before its admission: offered=%d accepted=%d rejected=%d',
|
|
2169
|
-
peerId, result.offered, result.accepted, result.rejected.length);
|
|
2278
|
+
log('Device %s caught up before its admission: offered=%d alreadyHeld=%d accepted=%d rejected=%d',
|
|
2279
|
+
peerId, result.offered, result.alreadyHeld, result.accepted, result.rejected.length);
|
|
2170
2280
|
return release;
|
|
2171
2281
|
} catch (error) {
|
|
2172
2282
|
release();
|
|
@@ -2230,6 +2340,20 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2230
2340
|
return this.controlStorage;
|
|
2231
2341
|
}
|
|
2232
2342
|
|
|
2343
|
+
/**
|
|
2344
|
+
* The store the control network's peer-join block catch-up reads, or `undefined` when this
|
|
2345
|
+
* runtime does not arm it: `controlBackfill.enabled: false`, or no control storage. Both of
|
|
2346
|
+
* its readers ask here so they cannot disagree: {@link startControlBackfill} arms on it, and
|
|
2347
|
+
* {@link buildControlNodeOptions} turns Optimystic's rebalance reaction off on it.
|
|
2348
|
+
*
|
|
2349
|
+
* Through the resolver, not the {@link controlStorage} memo field: the resolver is
|
|
2350
|
+
* idempotent, so the catch-up cannot be silently disarmed by a future reordering that moves
|
|
2351
|
+
* a read ahead of the one that populated the field.
|
|
2352
|
+
*/
|
|
2353
|
+
private controlBackfillStorage(): IRawStorage | undefined {
|
|
2354
|
+
return this.config.controlBackfill?.enabled === false ? undefined : this.resolveControlStorage();
|
|
2355
|
+
}
|
|
2356
|
+
|
|
2233
2357
|
/**
|
|
2234
2358
|
* Warn the operator when `network.announceAddrs` will silently cost this node the
|
|
2235
2359
|
* relay reachability it also configured.
|
|
@@ -2337,6 +2461,19 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2337
2461
|
cohortQueryTimeoutMs: declaredCohortReadDeadlineMs(network)
|
|
2338
2462
|
}),
|
|
2339
2463
|
arachnode: { enableRingZulu: profile === 'storage' },
|
|
2464
|
+
// One join catch-up per node, for the strand node's reason (`strand-instance-manager.ts`,
|
|
2465
|
+
// at its `rebalance` option): Optimystic's rebalance growth arm would push a joining
|
|
2466
|
+
// member each control block again, one request per block. Every peer the growth arm
|
|
2467
|
+
// pushes to, the control catch-up pushes to as well, except the ones it deliberately
|
|
2468
|
+
// withholds the control store from: the cohort predicate (`isNetworkMember` below) admits
|
|
2469
|
+
// everyone while the membership snapshot is empty, and bootstrap/relay peers outright.
|
|
2470
|
+
// With the catch-up disabled the reaction stays on.
|
|
2471
|
+
//
|
|
2472
|
+
// NOTE: no ring shifts and no reclaimable shed blocks on this node either, unused today.
|
|
2473
|
+
// If storage nodes ever need them, turn the reaction back on: the catch-up already reports
|
|
2474
|
+
// its accepted pushes to the rebalance monitor (`reportBlockHeld`); measure whether the
|
|
2475
|
+
// growth arm still re-sends.
|
|
2476
|
+
...(this.controlBackfillStorage() && { rebalance: { enabled: false } }),
|
|
2340
2477
|
...(identityKey && { privateKey: identityKey }),
|
|
2341
2478
|
// With `network.relayAddrs` set, built with libp2p's relay discovery off (given a
|
|
2342
2479
|
// transport list to wrap — `node-transports.ts`), so the supervisor
|
|
@@ -2405,10 +2542,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2405
2542
|
// Who this node SENDS control writes to (cohort and coordinator choice), where
|
|
2406
2543
|
// `authorizeInboundStream` is who may send to it. Without it Optimystic picks cohort
|
|
2407
2544
|
// members from the protocols a peer's peer-store record lists, which any connected
|
|
2408
|
-
// peer can fill.
|
|
2409
|
-
|
|
2410
|
-
// whose verified redemption is being written, which must hold the rows that admit it.
|
|
2411
|
-
isNetworkMember: (peerId) => this.isControlNetworkMember(peerId) || this.admittingDevices.has(peerId)
|
|
2545
|
+
// peer can fill.
|
|
2546
|
+
isNetworkMember: (peerId) => this.isControlCohortMember(peerId)
|
|
2412
2547
|
};
|
|
2413
2548
|
|
|
2414
2549
|
return nodeOptions;
|
|
@@ -2492,8 +2627,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2492
2627
|
* (it redeems here by seating the row from its bundle, after which check 7
|
|
2493
2628
|
* admits) or from a sibling whose membership row is still replicating (check 5
|
|
2494
2629
|
* admits once it lands; if it has not landed by the deadline, the connection
|
|
2495
|
-
* is closed and
|
|
2496
|
-
* dials are never gated). The protocol guard refuses all of them every
|
|
2630
|
+
* is closed and the reconcile pass the close starts on either side re-forms it,
|
|
2631
|
+
* since outbound dials are never gated). The protocol guard refuses all of them every
|
|
2497
2632
|
* members-only protocol meanwhile. A circuit-relay reservation admitted by
|
|
2498
2633
|
* {@link admitControlRelayReservation} disarms the deadline (see
|
|
2499
2634
|
* `membership-connection-gater.ts` → "The relay-reservation seam").
|
|
@@ -2724,6 +2859,18 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2724
2859
|
return this.authorizedControlPeers.size === 0 || this.authorizedControlPeers.has(remotePeerId);
|
|
2725
2860
|
}
|
|
2726
2861
|
|
|
2862
|
+
/**
|
|
2863
|
+
* Whether this node sends control writes to `remotePeerId`: Optimystic's `isNetworkMember`
|
|
2864
|
+
* for the control node (cohort and coordinator choice), and the membership half of
|
|
2865
|
+
* {@link hasReachableControlSibling}. The stream gate's rule ({@link isControlNetworkMember}),
|
|
2866
|
+
* unlogged, because Optimystic asks several times per peer per cohort assembly and counts
|
|
2867
|
+
* refusals on its own debug line; plus a device whose verified redemption is being written
|
|
2868
|
+
* ({@link admittingDevices}), which must hold the rows that admit it.
|
|
2869
|
+
*/
|
|
2870
|
+
private isControlCohortMember(remotePeerId: string): boolean {
|
|
2871
|
+
return this.isControlNetworkMember(remotePeerId) || this.admittingDevices.has(remotePeerId);
|
|
2872
|
+
}
|
|
2873
|
+
|
|
2727
2874
|
/**
|
|
2728
2875
|
* Per-stream authorization for the protocol guard's `members-snapshot` class
|
|
2729
2876
|
* (`control-protocol-guard.ts`): the four Optimystic control-DB protocols
|
|
@@ -2765,7 +2912,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2765
2912
|
* this listener with its write lock HELD (`notifyMembershipChanged`), so a retrying
|
|
2766
2913
|
* read here would sleep its backoff holding the lock and stall every other local
|
|
2767
2914
|
* writer. Nothing is lost — this refresh already keeps the previous snapshot on
|
|
2768
|
-
* failure and is re-driven by the next membership
|
|
2915
|
+
* failure and is re-driven by the next membership change (local or replicated, through
|
|
2916
|
+
* the gate watch) and by the reconcile pass's backstop.
|
|
2769
2917
|
*
|
|
2770
2918
|
* The trusted-owner anchor sync ({@link syncOwnerAnchor}) runs FIRST, so the snapshot
|
|
2771
2919
|
* built here judges `CadrePeer` rows against the anchor as the `OwnerKey` table now
|
|
@@ -2840,15 +2988,20 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2840
2988
|
// commit is what would authorize it), and the gate re-judges at push time.
|
|
2841
2989
|
//
|
|
2842
2990
|
// NOTE: every refresh re-arms every connected peer that is not yet caught
|
|
2843
|
-
// up
|
|
2844
|
-
// connection that is admitted but never authorized — a configured
|
|
2991
|
+
// up. A connection that is admitted but never authorized — a configured
|
|
2845
2992
|
// bootstrap or relay peer, the steady state for a NAT'd node — is
|
|
2846
|
-
// therefore re-judged on every
|
|
2847
|
-
//
|
|
2848
|
-
//
|
|
2849
|
-
//
|
|
2850
|
-
//
|
|
2993
|
+
// therefore re-judged on every refresh, forever: every local membership
|
|
2994
|
+
// write, every replicated membership change (the gate watch), and every
|
|
2995
|
+
// reconcile pass, down to the 2-minute backstop on an idle party. Each
|
|
2996
|
+
// such pass is one timer and a lookup in the snapshot just set (the gate
|
|
2997
|
+
// reads no control database). If a node ever holds many such connections,
|
|
2998
|
+
// or refreshes get frequent, skip re-arming peers whose last pass was
|
|
2999
|
+
// denied under the SAME membership snapshot instead of re-arming
|
|
3000
|
+
// unconditionally.
|
|
2851
3001
|
void this.controlBackfill?.scheduleConnectedPeers();
|
|
3002
|
+
// Same join order for the growth edge: a sibling that identified before this commit
|
|
3003
|
+
// authorized it becomes reachable here.
|
|
3004
|
+
this.updateControlSiblingReachability();
|
|
2852
3005
|
await this.hangUpRevokedControlPeers(rows, loadTombstones);
|
|
2853
3006
|
await this.noteOwnRemoval(rows, loadTombstones);
|
|
2854
3007
|
} catch (error) {
|
|
@@ -3035,9 +3188,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3035
3188
|
}
|
|
3036
3189
|
|
|
3037
3190
|
/**
|
|
3038
|
-
* What the strand watcher
|
|
3191
|
+
* What the strand watcher reads: this party's control rows plus a row per cross-party
|
|
3039
3192
|
* join (see {@link joinedStrands}), so a joined strand is offered and relaunched on
|
|
3040
|
-
* the same path as the party's own.
|
|
3193
|
+
* the same path as the party's own. It runs on a change to either table; the
|
|
3194
|
+
* machine-local joins wake it through {@link JoinedStrandSession}'s change callback.
|
|
3041
3195
|
*/
|
|
3042
3196
|
private createStrandQueryable(): StrandQueryable {
|
|
3043
3197
|
return {
|
|
@@ -3049,7 +3203,14 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3049
3203
|
log('Querying strands from control database');
|
|
3050
3204
|
const control = await this.controlDatabase.queryStrands();
|
|
3051
3205
|
return this.joinedStrands ? await this.joinedStrands.withControlRows(control) : control;
|
|
3052
|
-
}
|
|
3206
|
+
},
|
|
3207
|
+
watchChanges: (onChange, options) => {
|
|
3208
|
+
if (!this.controlDatabase) {
|
|
3209
|
+
throw new Error('CadreNode must initialize its control database before watching its strand tables');
|
|
3210
|
+
}
|
|
3211
|
+
return this.controlDatabase.watchTables(['CadreControl.Strand', 'CadreControl.JoinedStrand'], onChange, options);
|
|
3212
|
+
},
|
|
3213
|
+
lastReadPartial: () => this.joinedStrands?.lastReadPartial() ?? false
|
|
3053
3214
|
};
|
|
3054
3215
|
}
|
|
3055
3216
|
|
|
@@ -3065,10 +3226,6 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3065
3226
|
return this.controlDatabase;
|
|
3066
3227
|
};
|
|
3067
3228
|
return {
|
|
3068
|
-
// NOTE: one more control read per watcher poll (5 s), and for a party that never
|
|
3069
|
-
// joined anything it reads a never-written block, which consults the cohort. If poll
|
|
3070
|
-
// cost shows up in a device profile, cache the list and re-read it on the reconcile
|
|
3071
|
-
// cadence.
|
|
3072
3229
|
list: () => database().queryJoinedStrands(),
|
|
3073
3230
|
names: async (strandId) => (await database().queryStrand(strandId)) !== null
|
|
3074
3231
|
|| (await database().queryJoinedStrand(strandId)) !== null,
|
|
@@ -3155,26 +3312,29 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3155
3312
|
* after start (non-blocking). {@link registerSelf} is idempotent and safely
|
|
3156
3313
|
* no-ops when it cannot yet sign/insert (e.g. owner key not installed),
|
|
3157
3314
|
* so the timer is harmless even when registration only becomes possible later.
|
|
3315
|
+
* A record this process already put in place that is still current (a founding owner's
|
|
3316
|
+
* explicit publish, a redeemed invitation's admission, a restart's first-growth re-touch) is
|
|
3317
|
+
* not written again ({@link publishSelfRecordUnlessCurrent}). Whether the node is connected
|
|
3318
|
+
* yet does not matter: a device that redeems within a second of starting is.
|
|
3158
3319
|
*/
|
|
3159
3320
|
private scheduleSelfRegistration(): void {
|
|
3160
3321
|
this.selfRegistrationTimer = setTimeout(() => {
|
|
3161
3322
|
void (async () => {
|
|
3162
3323
|
try {
|
|
3163
|
-
await this.
|
|
3324
|
+
await this.publishSelfRecordUnlessCurrent();
|
|
3164
3325
|
} catch (error) {
|
|
3165
|
-
// Background task — a failed publish must not crash the node. A
|
|
3166
|
-
//
|
|
3167
|
-
//
|
|
3168
|
-
//
|
|
3326
|
+
// Background task — a failed publish must not crash the node. A publish that loses
|
|
3327
|
+
// its row to another writer — the owner's re-touch, or a restarted machine's own row
|
|
3328
|
+
// replicating in while the update runs — is re-run by the write funnel over the row
|
|
3329
|
+
// it finds (`ControlDatabase.updateSelfPeerRecord`); one that lost past the retry
|
|
3330
|
+
// budget fails here, and the next connected reconcile pass is the fallback
|
|
3169
3331
|
// (publishFirstSelfRecord).
|
|
3170
3332
|
log('Self-registration failed: %o', error);
|
|
3171
3333
|
}
|
|
3172
|
-
// Wire the ongoing refresh
|
|
3173
|
-
// an eager reconcile pass
|
|
3174
|
-
// interval was just armed inside startRecordRefresh).
|
|
3334
|
+
// Wire the ongoing refresh and the control-cohort reconcile schedule, then
|
|
3335
|
+
// run an eager reconcile pass now that the node has settled.
|
|
3175
3336
|
this.startRecordRefresh();
|
|
3176
|
-
|
|
3177
|
-
log('Control-cohort reconcile (start) failed: %o', error));
|
|
3337
|
+
this.controlReconcileRuns?.trigger(0);
|
|
3178
3338
|
})();
|
|
3179
3339
|
}, 1000); // Small delay to ensure node is fully started
|
|
3180
3340
|
// Node-only: don't keep the event loop alive solely for this timer.
|
|
@@ -3249,16 +3409,16 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3249
3409
|
return 'skipped';
|
|
3250
3410
|
}
|
|
3251
3411
|
// Collected only past that skip, which a node waiting for its row replicated in takes on
|
|
3252
|
-
//
|
|
3412
|
+
// reconcile pass made while a sibling is reachable ({@link publishFirstSelfRecord}).
|
|
3253
3413
|
const addrs = await this.collectSelfAddrs();
|
|
3254
3414
|
|
|
3255
3415
|
if (!existing && ownerService) {
|
|
3256
3416
|
// First-time row: requires an owner signature (the node is its own
|
|
3257
3417
|
// owner). insertSelfPeerRecord throws if no owner key is present.
|
|
3258
3418
|
const record = this.signSelfRecord(peerId, signingKey, addrs, null);
|
|
3419
|
+
const aloneBefore = this.committedAlone();
|
|
3259
3420
|
if (await ownerService.insertSelfPeerRecord(record)) {
|
|
3260
|
-
|
|
3261
|
-
this.noteSelfRecordPublished();
|
|
3421
|
+
this.noteSelfPeerWritten(record, aloneBefore);
|
|
3262
3422
|
log('registerSelf: inserted own CadrePeer record (owner-signed, updatedAt=%d, %d addrs, sig=%s…)', record.updatedAt, addrs.length, record.sig.slice(0, 16));
|
|
3263
3423
|
return 'inserted';
|
|
3264
3424
|
}
|
|
@@ -3273,44 +3433,102 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3273
3433
|
log('registerSelf: own CadrePeer row appeared mid-publish (concurrent authorize); self-updating to carry the signature');
|
|
3274
3434
|
}
|
|
3275
3435
|
|
|
3276
|
-
//
|
|
3277
|
-
//
|
|
3278
|
-
// the
|
|
3279
|
-
//
|
|
3280
|
-
//
|
|
3281
|
-
//
|
|
3282
|
-
|
|
3283
|
-
|
|
3284
|
-
|
|
3285
|
-
|
|
3286
|
-
if (!current) {
|
|
3436
|
+
// The update body reads the row under the write lock and signs over what it finds, on
|
|
3437
|
+
// every attempt — so neither `existing` (a read that predates this publish) nor a first
|
|
3438
|
+
// attempt that lost the row to another writer decides the `UpdatedAt` the strictly-greater
|
|
3439
|
+
// self-update rule is measured against, and a removal landing mid-publish re-reads as
|
|
3440
|
+
// absent instead of matching no rows under a 'refreshed'. The record it returns is the
|
|
3441
|
+
// one in the row.
|
|
3442
|
+
const aloneBefore = this.committedAlone();
|
|
3443
|
+
const record = await this.controlDatabase.updateSelfPeerRecord(peerId,
|
|
3444
|
+
(current) => this.signSelfRecord(peerId, signingKey, addrs, current));
|
|
3445
|
+
if (record === null) {
|
|
3287
3446
|
// Two causes, and the second is a steady state rather than a race: the row really
|
|
3288
3447
|
// vanished mid-publish (next refresh re-inserts), or this node was REVOKED — its row
|
|
3289
3448
|
// is physically present, so insertCadrePeer's in-lock existence check no-op'd above,
|
|
3290
|
-
// but
|
|
3291
|
-
//
|
|
3292
|
-
//
|
|
3449
|
+
// but the body reads it as absent because its stamp is retired. The revoked case
|
|
3450
|
+
// repeats every heartbeat and never re-inserts; that is intended (a revoked node has
|
|
3451
|
+
// nothing to publish), so this stays a skip, not an error.
|
|
3293
3452
|
log('registerSelf: own CadrePeer row not readable (vanished mid-publish, or this node is revoked and its stamp retired); skipping');
|
|
3294
3453
|
return 'skipped';
|
|
3295
3454
|
}
|
|
3296
|
-
|
|
3297
|
-
await this.controlDatabase.updateSelfPeerRecord(record);
|
|
3298
|
-
if (this.committedAlone()) this.pendingSelfPeerWrite = true;
|
|
3299
|
-
this.noteSelfRecordPublished();
|
|
3455
|
+
this.noteSelfPeerWritten(record, aloneBefore);
|
|
3300
3456
|
log('registerSelf: refreshed own CadrePeer record (updatedAt=%d, %d addrs, sig=%s…)', record.updatedAt, addrs.length, record.sig.slice(0, 16));
|
|
3301
3457
|
return 'refreshed';
|
|
3302
3458
|
}
|
|
3303
3459
|
|
|
3304
3460
|
/**
|
|
3305
|
-
*
|
|
3306
|
-
*
|
|
3307
|
-
*
|
|
3461
|
+
* Bookkeeping for a committed write of this node's own `CadrePeer` row by {@link publishSelfRecord}:
|
|
3462
|
+
* `record` is the one the row holds (for the update path, the one its body signed on the
|
|
3463
|
+
* attempt that committed, not the first it signed). Queued for the drain's re-touch
|
|
3464
|
+
* ({@link pendingSelfPeerWrite}) when no sibling was reachable when the write started or when
|
|
3465
|
+
* it ended: a sibling that identifies while the write is out may do so after the write's cohort
|
|
3466
|
+
* was chosen, and the first-growth drain skips a row holding {@link lastSignedSelfRecord} unless
|
|
3467
|
+
* this flag is set. Both fields are set here together so no write can leave the record without
|
|
3468
|
+
* the flag.
|
|
3308
3469
|
*/
|
|
3309
|
-
private
|
|
3310
|
-
this.
|
|
3470
|
+
private noteSelfPeerWritten(record: PeerAddressRecord, aloneBefore: boolean): void {
|
|
3471
|
+
if (aloneBefore || this.committedAlone()) this.pendingSelfPeerWrite = true;
|
|
3472
|
+
this.lastSignedSelfRecord = record;
|
|
3473
|
+
this.noteSelfRecordPublished();
|
|
3474
|
+
}
|
|
3475
|
+
|
|
3476
|
+
/**
|
|
3477
|
+
* Record that this node's own address record is in place as of `at`, and re-arm the
|
|
3478
|
+
* stale-record warning. Called for BOTH publishing paths of {@link publishSelfRecord}
|
|
3479
|
+
* ({@link noteSelfPeerWritten}), and by {@link publishSelfRecordUnlessCurrent} with the stamp
|
|
3480
|
+
* of a record already in place — a `skipped` publish wrote nothing and found nothing, so it
|
|
3481
|
+
* must not refresh the stamp.
|
|
3482
|
+
*/
|
|
3483
|
+
private noteSelfRecordPublished(at = Date.now()): void {
|
|
3484
|
+
this.lastSelfRecordPublishAt = at;
|
|
3311
3485
|
this.selfRecordStaleWarned = false;
|
|
3312
3486
|
}
|
|
3313
3487
|
|
|
3488
|
+
/**
|
|
3489
|
+
* {@link registerSelf}, unless the own row already holds the record this process last signed
|
|
3490
|
+
* ({@link lastSignedSelfRecord}) and that record is current ({@link isCurrentSelfRecord});
|
|
3491
|
+
* then nothing is written and the record counts as published at its own stamp. For a publish
|
|
3492
|
+
* that puts a record in place (the boot timer, the reconcile pass's first publish, the
|
|
3493
|
+
* first-growth drain), never the heartbeat, which exists to re-stamp freshness. `'current'`
|
|
3494
|
+
* is this method's own answer; {@link registerSelf} always writes.
|
|
3495
|
+
*/
|
|
3496
|
+
private async publishSelfRecordUnlessCurrent(): Promise<SelfRegistrationOutcome | 'current'> {
|
|
3497
|
+
if (this.registerSelfInFlight) {
|
|
3498
|
+
return this.registerSelfInFlight;
|
|
3499
|
+
}
|
|
3500
|
+
const record = this.lastSignedSelfRecord;
|
|
3501
|
+
if (record !== null && await this.isCurrentSelfRecord(record)) {
|
|
3502
|
+
this.noteSelfRecordPublished(record.updatedAt);
|
|
3503
|
+
log('registerSelf: own CadrePeer record already current (updatedAt=%d); not written again', record.updatedAt);
|
|
3504
|
+
return 'current';
|
|
3505
|
+
}
|
|
3506
|
+
return this.registerSelf();
|
|
3507
|
+
}
|
|
3508
|
+
|
|
3509
|
+
/**
|
|
3510
|
+
* Is `record` still the one to publish: signed less than half a heartbeat ago, with this
|
|
3511
|
+
* node's profile, over the addresses it would publish now, and the signature the own row
|
|
3512
|
+
* holds? The age bound keeps a record that is not re-written fresh for resolvers: at most half
|
|
3513
|
+
* a heartbeat old here and re-stamped by the heartbeat within one more, so never older than
|
|
3514
|
+
* 1.5 heartbeats (11.25 min) against {@link DEFAULT_PEER_RECORD_MAX_AGE_MS}. Cheap checks
|
|
3515
|
+
* first: the row comes from the membership view, the addresses from the app's resolver.
|
|
3516
|
+
*/
|
|
3517
|
+
private async isCurrentSelfRecord(record: PeerAddressRecord): Promise<boolean> {
|
|
3518
|
+
if (Date.now() - record.updatedAt >= DEFAULT_PEER_RECORD_HEARTBEAT_MS / 2 || record.profile !== this.config.profile) {
|
|
3519
|
+
return false;
|
|
3520
|
+
}
|
|
3521
|
+
const row = await this.controlDatabase?.queryPeerRecord(record.peerId);
|
|
3522
|
+
if (row?.sig !== record.sig) {
|
|
3523
|
+
return false;
|
|
3524
|
+
}
|
|
3525
|
+
// Sanitized, as the record a redemption carries is (`signRedeemRequest`); a published
|
|
3526
|
+
// record is not, so one whose list sanitizing would change (over 16 addresses) is never
|
|
3527
|
+
// current and is written as before.
|
|
3528
|
+
const addrs = sanitizeAddrs(await this.collectSelfAddrs());
|
|
3529
|
+
return addrs.length === record.addrs.length && addrs.every((addr, i) => addr === record.addrs[i]);
|
|
3530
|
+
}
|
|
3531
|
+
|
|
3314
3532
|
/**
|
|
3315
3533
|
* Report an abandoned control write to the embedding app.
|
|
3316
3534
|
*
|
|
@@ -3455,19 +3673,24 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3455
3673
|
* process does not learn it was removed ({@link noteOwnRemoval}), and pays that read on every
|
|
3456
3674
|
* pass.
|
|
3457
3675
|
*
|
|
3458
|
-
*
|
|
3459
|
-
*
|
|
3460
|
-
*
|
|
3461
|
-
* then never see rows written while it
|
|
3462
|
-
*
|
|
3463
|
-
*
|
|
3676
|
+
* Only while a sibling is reachable ({@link hasReachableControlSibling}): a row written
|
|
3677
|
+
* elsewhere arrives from a sibling, and an owner's publish committed alone is local-only and
|
|
3678
|
+
* forks its own history — a restarted owner whose embedder runs a pass before it reconnects
|
|
3679
|
+
* would write its row on the stale side of the fork, and then never see rows written while it
|
|
3680
|
+
* was down (`cadre-invite-any-member` runs that pass and fails every time without this
|
|
3681
|
+
* condition). A connection is not enough: until the sibling identifies, the write's cohort is
|
|
3682
|
+
* this node alone. The boot-time publish and the first-growth drain already try at the
|
|
3683
|
+
* node's lone and first-sibling moments; this retries on the passes after.
|
|
3684
|
+
*
|
|
3685
|
+
* A machine admitted by invitation finds its row already holding the record its redemption
|
|
3686
|
+
* carried, and writes nothing ({@link publishSelfRecordUnlessCurrent}).
|
|
3464
3687
|
*/
|
|
3465
3688
|
private async publishFirstSelfRecord(): Promise<void> {
|
|
3466
3689
|
if (this.lastSelfRecordPublishAt !== null || this.isAwaitingClaim() || this.removal !== null
|
|
3467
|
-
|| this.
|
|
3690
|
+
|| !this.hasReachableControlSibling()) {
|
|
3468
3691
|
return;
|
|
3469
3692
|
}
|
|
3470
|
-
const outcome = await this.
|
|
3693
|
+
const outcome = await this.publishSelfRecordUnlessCurrent().catch((error: unknown) => {
|
|
3471
3694
|
log('reconcileControlCohort: first self-record publish failed (continuing): %o', error);
|
|
3472
3695
|
return 'skipped' as const;
|
|
3473
3696
|
});
|
|
@@ -3506,17 +3729,22 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3506
3729
|
this.escalateIfSelfRecordStale(reason, error);
|
|
3507
3730
|
});
|
|
3508
3731
|
};
|
|
3509
|
-
const
|
|
3510
|
-
|
|
3511
|
-
|
|
3512
|
-
|
|
3732
|
+
const backstopMs = this.controlReconcileBackstopMs();
|
|
3733
|
+
// The pass's schedule. A run never joins a pass already in flight: that pass may have
|
|
3734
|
+
// read its rows before the change that woke this run, so joining it would drop the
|
|
3735
|
+
// change until the backstop. It waits for the pass, then starts its own.
|
|
3736
|
+
this.controlReconcileRuns = scheduleRuns(async () => {
|
|
3737
|
+
await this.reconcileControlCohortInFlight?.catch(() => undefined);
|
|
3738
|
+
await this.reconcileControlCohort().catch((error) =>
|
|
3739
|
+
log('Control-cohort reconcile (scheduled) failed: %o', error));
|
|
3740
|
+
}, { label: 'control-reconcile', backstopMs });
|
|
3513
3741
|
|
|
3514
3742
|
// Address churn re-publishes the self record AND re-checks the control cohort:
|
|
3515
3743
|
// a relay-reservation rotation / NAT change can drop a sibling connection, so
|
|
3516
3744
|
// the next pass should re-observe and re-dial it.
|
|
3517
3745
|
this.selfPeerUpdateHandler = () => {
|
|
3518
3746
|
republish('self:peer:update');
|
|
3519
|
-
|
|
3747
|
+
this.controlReconcileRuns?.trigger(0);
|
|
3520
3748
|
};
|
|
3521
3749
|
this.controlNode.addEventListener('self:peer:update', this.selfPeerUpdateHandler);
|
|
3522
3750
|
|
|
@@ -3538,15 +3766,32 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3538
3766
|
this.recordRefreshTimer = setInterval(() => republish('heartbeat'), DEFAULT_PEER_RECORD_HEARTBEAT_MS);
|
|
3539
3767
|
(this.recordRefreshTimer as { unref?: () => void } | null)?.unref?.();
|
|
3540
3768
|
|
|
3541
|
-
|
|
3542
|
-
|
|
3543
|
-
|
|
3769
|
+
log('Record refresh wired (heartbeat=%dms, cohortReconcileBackstop=%dms)',
|
|
3770
|
+
DEFAULT_PEER_RECORD_HEARTBEAT_MS, backstopMs);
|
|
3771
|
+
}
|
|
3544
3772
|
|
|
3545
|
-
|
|
3546
|
-
|
|
3773
|
+
/** The reconcile schedule's backstop: `network.controlCohort.reconcileMs`, default `DEFAULT_CONTROL_COHORT_RECONCILE_MS`. */
|
|
3774
|
+
private controlReconcileBackstopMs(): number {
|
|
3775
|
+
return this.config.network?.controlCohort?.reconcileMs ?? DEFAULT_CONTROL_COHORT_RECONCILE_MS;
|
|
3547
3776
|
}
|
|
3548
3777
|
|
|
3549
|
-
/**
|
|
3778
|
+
/**
|
|
3779
|
+
* The gate watch's run ({@link membershipGateWatch}): re-materialize the gate, so a
|
|
3780
|
+
* replicated revocation hangs up the removed machine now (`hangUpRevokedControlPeers`
|
|
3781
|
+
* ends every refresh), then start a reconcile pass for whatever else the change means
|
|
3782
|
+
* (a new sibling to dial, a tombstoned row to reap). A local membership write refreshes
|
|
3783
|
+
* the gate twice — once from the awaited membership listener, so the writer sees its
|
|
3784
|
+
* change before returning, and once here, 50 ms later. The listener's refresh runs under
|
|
3785
|
+
* the write lock, so its reads take the committed-read path and the view keeps nothing
|
|
3786
|
+
* from them; this refresh is the read that refills the view after the write, which some
|
|
3787
|
+
* later caller would otherwise pay. Accepted: one reload per change, not two.
|
|
3788
|
+
*/
|
|
3789
|
+
private async onMembershipGateTablesChanged(): Promise<void> {
|
|
3790
|
+
await this.refreshMembershipGate('table-change');
|
|
3791
|
+
this.controlReconcileRuns?.trigger(0);
|
|
3792
|
+
}
|
|
3793
|
+
|
|
3794
|
+
/** Tear down the self-record refresh timers, the reconcile schedule and the listeners (see {@link cleanup}). */
|
|
3550
3795
|
private stopRecordRefresh(): void {
|
|
3551
3796
|
if (this.selfRegistrationTimer) {
|
|
3552
3797
|
clearTimeout(this.selfRegistrationTimer);
|
|
@@ -3556,24 +3801,30 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3556
3801
|
clearInterval(this.recordRefreshTimer);
|
|
3557
3802
|
this.recordRefreshTimer = null;
|
|
3558
3803
|
}
|
|
3559
|
-
if (this.
|
|
3560
|
-
|
|
3561
|
-
this.
|
|
3804
|
+
if (this.controlReconcileRuns) {
|
|
3805
|
+
this.controlReconcileRuns.dispose();
|
|
3806
|
+
this.controlReconcileRuns = null;
|
|
3807
|
+
}
|
|
3808
|
+
if (this.membershipViewReloadTimer) {
|
|
3809
|
+
clearTimeout(this.membershipViewReloadTimer);
|
|
3810
|
+
this.membershipViewReloadTimer = null;
|
|
3562
3811
|
}
|
|
3812
|
+
this.reconcileRedialStreak = 0;
|
|
3563
3813
|
if (this.controlNode) {
|
|
3564
3814
|
if (this.selfPeerUpdateHandler) {
|
|
3565
3815
|
this.controlNode.removeEventListener('self:peer:update', this.selfPeerUpdateHandler);
|
|
3566
3816
|
}
|
|
3567
|
-
if (this.
|
|
3568
|
-
this.controlNode.removeEventListener('
|
|
3817
|
+
if (this.controlPeerIdentifyHandler) {
|
|
3818
|
+
this.controlNode.removeEventListener('peer:identify', this.controlPeerIdentifyHandler);
|
|
3569
3819
|
}
|
|
3570
3820
|
if (this.controlConnectionCloseHandler) {
|
|
3571
3821
|
this.controlNode.removeEventListener('connection:close', this.controlConnectionCloseHandler);
|
|
3572
3822
|
}
|
|
3573
3823
|
}
|
|
3574
3824
|
this.selfPeerUpdateHandler = null;
|
|
3575
|
-
this.
|
|
3825
|
+
this.controlPeerIdentifyHandler = null;
|
|
3576
3826
|
this.controlConnectionCloseHandler = null;
|
|
3827
|
+
this.identifiedControlPeers.clear();
|
|
3577
3828
|
|
|
3578
3829
|
// Reset the write-while-alone re-replication state so a stop()→start() cycle
|
|
3579
3830
|
// re-arms the growth edge and re-runs BOTH one-shot passes. Any queued
|
|
@@ -3582,10 +3833,9 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3582
3833
|
// first-growth revocation sweep — which is why the sweep flag must reset too,
|
|
3583
3834
|
// or the second lifetime would skip the sweep and strand exactly the
|
|
3584
3835
|
// committed-alone tombstones it exists to carry.
|
|
3585
|
-
this.
|
|
3836
|
+
this.controlSiblingReachable = false;
|
|
3586
3837
|
this.reconstructedLocalOnlyWrites = false;
|
|
3587
3838
|
this.reissuedHeldRevocations = false;
|
|
3588
|
-
this.revocationLedgerOpened = false;
|
|
3589
3839
|
this.pendingPeerWrites.clear();
|
|
3590
3840
|
this.pendingRevocations.clear();
|
|
3591
3841
|
this.pendingSelfPeerWrite = false;
|
|
@@ -3594,6 +3844,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3594
3844
|
// from scratch, and carrying the previous lifetime's stamp forward would either
|
|
3595
3845
|
// suppress the stale-record warning or fire it on a node that just came up.
|
|
3596
3846
|
this.lastSelfRecordPublishAt = null;
|
|
3847
|
+
this.lastSignedSelfRecord = null;
|
|
3597
3848
|
this.selfRecordStaleWarned = false;
|
|
3598
3849
|
}
|
|
3599
3850
|
|
|
@@ -3783,26 +4034,37 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3783
4034
|
* Run one proactive control-cohort dial pass to keep this node connected to its
|
|
3784
4035
|
* cadre siblings (so the `CadreControl` collections form a replicating cohort).
|
|
3785
4036
|
* Public so the cohort-growth-driven re-replication path
|
|
3786
|
-
* (`control-write-ensure-replicated`)
|
|
3787
|
-
*
|
|
3788
|
-
*
|
|
4037
|
+
* (`control-write-ensure-replicated`), `addDrone`'s callers, `claimNode`,
|
|
4038
|
+
* `redeemCadreInvitation` and tests can run a pass on demand. On its own a pass
|
|
4039
|
+
* runs when something it acts on changes, and otherwise at a backstop
|
|
4040
|
+
* ({@link controlReconcileRuns}): once after start; on `self:peer:update`; when a
|
|
4041
|
+
* membership table changes, local commit or replicated ({@link membershipGateWatch});
|
|
4042
|
+
* when a control connection to an authorized sibling opens or closes
|
|
4043
|
+
* ({@link onControlSiblingConnectionChange}); on the redial and strand-address wakes
|
|
4044
|
+
* a pass arms for itself ({@link armReconcileWake}); and at the backstop,
|
|
4045
|
+
* `network.controlCohort.reconcileMs` (default `DEFAULT_CONTROL_COHORT_RECONCILE_MS`,
|
|
4046
|
+
* 2 minutes) after the last pass settled. A connection to a relay, a bootstrap node or
|
|
4047
|
+
* another party's machine starts no pass: the pass acts on none of them, beyond
|
|
4048
|
+
* counting them as "connected" for its write steps, which therefore run at the next
|
|
4049
|
+
* membership change or backstop on a one-machine party whose only connection is a
|
|
4050
|
+
* relay.
|
|
3789
4051
|
*
|
|
3790
4052
|
* Concurrent triggers collapse into a single in-flight pass
|
|
3791
4053
|
* (see {@link reconcileControlCohortInFlight}) so two passes never double-dial;
|
|
3792
|
-
* a call that joins an in-flight pass resolves to THAT pass's result
|
|
4054
|
+
* a call that joins an in-flight pass resolves to THAT pass's result (a scheduled
|
|
4055
|
+
* run waits for it instead, so the change that woke the run is seen).
|
|
3793
4056
|
* Best-effort throughout: a failure to resolve/dial any one sibling is logged
|
|
3794
4057
|
* and the pass continues; the whole pass is a no-op when the node is alone.
|
|
3795
4058
|
*
|
|
3796
4059
|
* Resolves to the peers the pass dialled ({@link ControlCohortReconcileResult}),
|
|
3797
4060
|
* so a caller can tell a link this pass opened from one something else opened —
|
|
3798
|
-
* the pass skips a peer that is already connected. The
|
|
3799
|
-
* discard it.
|
|
4061
|
+
* the pass skips a peer that is already connected. The scheduled runs discard it.
|
|
3800
4062
|
*/
|
|
3801
4063
|
async reconcileControlCohort(): Promise<ControlCohortReconcileResult> {
|
|
3802
4064
|
if (this.reconcileControlCohortInFlight) {
|
|
3803
4065
|
return this.reconcileControlCohortInFlight;
|
|
3804
4066
|
}
|
|
3805
|
-
const op = this.
|
|
4067
|
+
const op = this.runReconcileControlCohortAndArmWake();
|
|
3806
4068
|
this.reconcileControlCohortInFlight = op;
|
|
3807
4069
|
try {
|
|
3808
4070
|
return await op;
|
|
@@ -3811,38 +4073,77 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3811
4073
|
}
|
|
3812
4074
|
}
|
|
3813
4075
|
|
|
4076
|
+
/** One pass, then the wake it earns ({@link armReconcileWake}); the public result is the pass's `dialed`. */
|
|
4077
|
+
private async runReconcileControlCohortAndArmWake(): Promise<ControlCohortReconcileResult> {
|
|
4078
|
+
const startedAt = Date.now();
|
|
4079
|
+
const outcome = await this.runReconcileControlCohort();
|
|
4080
|
+
this.armReconcileWake(outcome, startedAt);
|
|
4081
|
+
return { dialed: outcome.dialed };
|
|
4082
|
+
}
|
|
4083
|
+
|
|
4084
|
+
/**
|
|
4085
|
+
* After every pass, explicit or scheduled: advance or reset the redial streak and
|
|
4086
|
+
* `trigger` the schedule for the earliest of the pass's own reasons to run again
|
|
4087
|
+
* (`nextReconcileWakeMs`): a redial for a sibling or bootstrap peer it left
|
|
4088
|
+
* unconnected, or a strand-address ask falling due before the backstop. Nothing else
|
|
4089
|
+
* the pass does needs a wake sooner than the backstop: delegate grants re-announce
|
|
4090
|
+
* every 15 minutes inside a 30-minute TTL, the FRET and formation address re-merges
|
|
4091
|
+
* guard a one-hour expiry, and the self record has its own heartbeat. A pass that
|
|
4092
|
+
* runs before the schedule exists (an explicit call during the first second after
|
|
4093
|
+
* start) still counts for the streak.
|
|
4094
|
+
*/
|
|
4095
|
+
private armReconcileWake(outcome: ControlCohortPassOutcome, startedAt: number): void {
|
|
4096
|
+
this.reconcileRedialStreak = outcome.unconnected ? this.reconcileRedialStreak + 1 : 0;
|
|
4097
|
+
const runs = this.controlReconcileRuns;
|
|
4098
|
+
if (!runs) {
|
|
4099
|
+
return;
|
|
4100
|
+
}
|
|
4101
|
+
const wakeMs = nextReconcileWakeMs({
|
|
4102
|
+
redialStreak: this.reconcileRedialStreak,
|
|
4103
|
+
passStartedAt: startedAt,
|
|
4104
|
+
strandAddrDueAt: this.strandAddrAskDueAt.values(),
|
|
4105
|
+
now: Date.now(),
|
|
4106
|
+
backstopMs: this.controlReconcileBackstopMs(),
|
|
4107
|
+
});
|
|
4108
|
+
if (wakeMs !== undefined) {
|
|
4109
|
+
log('reconcileControlCohort: next pass in %dms (redial streak %d)', wakeMs, this.reconcileRedialStreak);
|
|
4110
|
+
runs.trigger(wakeMs);
|
|
4111
|
+
}
|
|
4112
|
+
}
|
|
4113
|
+
|
|
3814
4114
|
/** Body of {@link reconcileControlCohort}; serialised by its single-flight guard. */
|
|
3815
|
-
private async runReconcileControlCohort(): Promise<
|
|
4115
|
+
private async runReconcileControlCohort(): Promise<ControlCohortPassOutcome> {
|
|
3816
4116
|
// Shutdown / not-yet-started guard (mirrors publishSelfRecord). A pass that
|
|
3817
4117
|
// fires after stop() began must early-return rather than touch a torn-down node.
|
|
3818
4118
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3819
|
-
return { dialed: [] };
|
|
3820
|
-
}
|
|
3821
|
-
//
|
|
3822
|
-
//
|
|
3823
|
-
//
|
|
3824
|
-
//
|
|
3825
|
-
// pass's length when that is longer (passes are single-flight, and one
|
|
3826
|
-
// unreachable sibling's dial alone can take `controlDialBudget().totalMs`).
|
|
4119
|
+
return { dialed: [], unconnected: false };
|
|
4120
|
+
}
|
|
4121
|
+
// Refresh the per-stream gate's materialized authorized set at the head of every
|
|
4122
|
+
// pass, so the sibling enumeration below judges rows the same way the gate does.
|
|
4123
|
+
// Changes reach the gate on their own through the gate watch; this refresh is the
|
|
4124
|
+
// backstop's, for a change this node's storage never heard of.
|
|
3827
4125
|
// NOTE: this refresh and the sibling enumeration below each run their own
|
|
3828
4126
|
// CadrePeer query (two reads per pass), plus a third from
|
|
3829
4127
|
// `refreshStrandPeerAddrs` on every pass where a strand is running AND this
|
|
3830
4128
|
// node holds a control connection (one read for the whole pass, not one per
|
|
3831
4129
|
// strand; it runs even when no sibling is due, because pruning departed
|
|
3832
4130
|
// siblings needs the current target set). Each also reads Revocation first.
|
|
3833
|
-
//
|
|
3834
|
-
//
|
|
3835
|
-
//
|
|
3836
|
-
//
|
|
3837
|
-
// control-founding-consult-budget.spec.ts).
|
|
3838
|
-
//
|
|
4131
|
+
// All of them are served from the control database's membership views
|
|
4132
|
+
// (`ControlDatabase.readMembershipRows`): an in-memory scan while the view is
|
|
4133
|
+
// current, storage only for the first read after the table changed or after
|
|
4134
|
+
// `MEMBERSHIP_VIEW_MAX_AGE_MS`, and an idle pass makes no cohort consult
|
|
4135
|
+
// (pinned in control-founding-consult-budget.spec.ts). Threading one row set
|
|
4136
|
+
// through the refresh, the reap, the enumeration and the strand-address targets
|
|
4137
|
+
// would save no storage read and was declined
|
|
4138
|
+
// (`control-reconcile-woken-by-changes-reads-once`); rows going stale during a
|
|
4139
|
+
// pass is harmless, since a change during the pass starts another one.
|
|
3839
4140
|
await this.refreshMembershipGate('reconcile');
|
|
3840
4141
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3841
|
-
return { dialed: [] };
|
|
4142
|
+
return { dialed: [], unconnected: false };
|
|
3842
4143
|
}
|
|
3843
4144
|
await this.publishFirstSelfRecord();
|
|
3844
4145
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3845
|
-
return { dialed: [] };
|
|
4146
|
+
return { dialed: [], unconnected: false };
|
|
3846
4147
|
}
|
|
3847
4148
|
// Refresh relay delegate grants BEFORE the sibling enumeration below: a
|
|
3848
4149
|
// solo cadre with a party relay has no siblings to dial but must still keep
|
|
@@ -3850,7 +4151,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3850
4151
|
// the reservation and faces the connection gate again).
|
|
3851
4152
|
await this.refreshDelegateGrants();
|
|
3852
4153
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3853
|
-
return { dialed: [] };
|
|
4154
|
+
return { dialed: [], unconnected: false };
|
|
3854
4155
|
}
|
|
3855
4156
|
// Then re-warm each running strand's own address book from its FRET address
|
|
3856
4157
|
// records and from whichever siblings are due an ask. Same reasoning as
|
|
@@ -3858,15 +4159,16 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3858
4159
|
// network, and every layer under cadre-core dials strand peers by bare peer id.
|
|
3859
4160
|
await this.refreshStrandPeerAddrs();
|
|
3860
4161
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3861
|
-
return { dialed: [] };
|
|
4162
|
+
return { dialed: [], unconnected: false };
|
|
3862
4163
|
}
|
|
3863
4164
|
const selfPeerId = this.controlNode.peerId.toString();
|
|
3864
4165
|
|
|
3865
4166
|
// Reap guarded rows this node still holds for incarnations an already-committed
|
|
3866
4167
|
// Revocation tombstone retires — the half of a removal replication cannot carry
|
|
3867
|
-
// (a delete of an already-absent row replays as nothing).
|
|
3868
|
-
//
|
|
3869
|
-
//
|
|
4168
|
+
// (a delete of an already-absent row replays as nothing). Only while a sibling is
|
|
4169
|
+
// reachable (hasReachableControlSibling, not merely connected): a reap is a write, and a
|
|
4170
|
+
// write committed alone is local-only and forks this node's own history, which is the
|
|
4171
|
+
// condition this work exists to stop creating
|
|
3870
4172
|
// (tickets/blocked/forked-control-collection-sync-livelocks.md). Nothing is urgent
|
|
3871
4173
|
// here — the stale row already reads as absent everywhere — so waiting for
|
|
3872
4174
|
// connectivity costs nothing and keeps the reap fork-CLOSING rather than
|
|
@@ -3888,7 +4190,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3888
4190
|
// avoid. Bounded by how long a sweep runs, which is bounded by the backlog of
|
|
3889
4191
|
// unreaped rows (0 in steady state). If that backlog can ever be large, move the
|
|
3890
4192
|
// check inside reapRevokedRows' loop via an injected predicate.
|
|
3891
|
-
if (this.
|
|
4193
|
+
if (this.hasReachableControlSibling()) {
|
|
3892
4194
|
try {
|
|
3893
4195
|
const reaped = await this.controlDatabase.reapRevokedRows(selfPeerId);
|
|
3894
4196
|
if (reaped > 0) {
|
|
@@ -3900,41 +4202,22 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3900
4202
|
log('reconcileControlCohort: reap pass failed (continuing): %o', error);
|
|
3901
4203
|
}
|
|
3902
4204
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3903
|
-
return { dialed: [] };
|
|
3904
|
-
}
|
|
3905
|
-
|
|
3906
|
-
// File the Revocation ledger marker once, so that table stops being a never-written
|
|
3907
|
-
// block, which the storage layer re-checks with the cohort on every read (and every
|
|
3908
|
-
// membership lookup and guarded insert reads it). Connected-only for the reap's
|
|
3909
|
-
// reason above: a marker committed alone is local-only, and one filed by a
|
|
3910
|
-
// disconnected owner while another machine creates the same collection is exactly
|
|
3911
|
-
// that fork. Owner-only, and at most once per process — see openRevocationLedgerIfDue.
|
|
3912
|
-
//
|
|
3913
|
-
// NOTE: accepted tradeoff — a solo founder does not file the marker until it holds a
|
|
3914
|
-
// control connection (normally its first sibling; a relay or bootstrap connection also
|
|
3915
|
-
// counts — see getControlConnectionCount — and is the reap's same residual), so until
|
|
3916
|
-
// then it keeps paying one local findCluster per read of
|
|
3917
|
-
// the missing Revocation block (0.009 ms each, measured upstream; no network work on
|
|
3918
|
-
// a cohort of one). Fork safety weighed over that and kept; revisit if findCluster
|
|
3919
|
-
// ever shows up as material in a device profile, or if a solo-founding marker can be
|
|
3920
|
-
// made fork-safe (e.g. filed inside the genesis transaction, before any other machine
|
|
3921
|
-
// can hold the party's collections).
|
|
3922
|
-
await this.openRevocationLedgerIfDue();
|
|
3923
|
-
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3924
|
-
return { dialed: [] };
|
|
4205
|
+
return { dialed: [], unconnected: false };
|
|
3925
4206
|
}
|
|
3926
4207
|
|
|
3927
4208
|
// Publish this machine's unpublished cross-party joins party-wide and remove the rows a
|
|
3928
|
-
// self-revocation queued (JoinedStrandSession.syncWithParty).
|
|
3929
|
-
//
|
|
3930
|
-
// a join
|
|
4209
|
+
// self-revocation queued (JoinedStrandSession.syncWithParty). Behind the reap's gate,
|
|
4210
|
+
// for its reason; owner-only by the session's own signer check. Nothing is urgent:
|
|
4211
|
+
// a join recorded locally starts no pass, so it reaches the party on the next pass
|
|
4212
|
+
// while a sibling is reachable — a membership change, a sibling connecting, or the
|
|
4213
|
+
// backstop, 2 minutes at most.
|
|
3931
4214
|
try {
|
|
3932
4215
|
await this.joinedStrands?.syncWithParty();
|
|
3933
4216
|
} catch (error) {
|
|
3934
4217
|
log('reconcileControlCohort: joined-strand sync failed (retrying next pass): %o', error);
|
|
3935
4218
|
}
|
|
3936
4219
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3937
|
-
return { dialed: [] };
|
|
4220
|
+
return { dialed: [], unconnected: false };
|
|
3938
4221
|
}
|
|
3939
4222
|
}
|
|
3940
4223
|
|
|
@@ -3949,11 +4232,11 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3949
4232
|
// because filling this table needs a connection and getting a connection
|
|
3950
4233
|
// needs this table. Fall back to the seed's bootstrap addresses; a solo
|
|
3951
4234
|
// cadre has none and this is a no-op. Must not throw or busy-loop.
|
|
3952
|
-
return
|
|
4235
|
+
return this.dialColdStartBootstrap();
|
|
3953
4236
|
}
|
|
3954
4237
|
// Re-guard after the await: a stop() may have raced the membership read.
|
|
3955
4238
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3956
|
-
return { dialed: [] };
|
|
4239
|
+
return { dialed: [], unconnected: false };
|
|
3957
4240
|
}
|
|
3958
4241
|
|
|
3959
4242
|
// 2. Classify backbone (always-on) members and select a bounded dial set.
|
|
@@ -3970,54 +4253,34 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3970
4253
|
// keeping what resolved for the dial loop below.
|
|
3971
4254
|
const resolved = await this.warmSiblingAddrBook(siblings);
|
|
3972
4255
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3973
|
-
return { dialed: [] };
|
|
4256
|
+
return { dialed: [], unconnected: false };
|
|
3974
4257
|
}
|
|
3975
4258
|
|
|
3976
4259
|
// 4. Skip already-connected peers (no re-dial / churn for live connections).
|
|
3977
4260
|
const connected = new Set(this.controlNode.getConnections().map((c) => c.remotePeer.toString()));
|
|
3978
4261
|
|
|
3979
4262
|
// 5. Dial each selected, not-yet-connected sibling, best-effort, from the
|
|
3980
|
-
// addresses step 3 already resolved for it.
|
|
4263
|
+
// addresses step 3 already resolved for it. A failed dial earns the pass a
|
|
4264
|
+
// redial wake (armReconcileWake); a sibling with nothing to dial does not.
|
|
3981
4265
|
const dialed: string[] = [];
|
|
4266
|
+
let unconnected = false;
|
|
3982
4267
|
for (const sibling of dials) {
|
|
3983
4268
|
if (!this._running || !this.controlNode) {
|
|
3984
|
-
return { dialed };
|
|
4269
|
+
return { dialed, unconnected: false };
|
|
3985
4270
|
}
|
|
3986
4271
|
if (connected.has(sibling.peerId)) {
|
|
3987
4272
|
continue;
|
|
3988
4273
|
}
|
|
3989
|
-
|
|
4274
|
+
const outcome = await this.dialControlSibling(sibling, resolved.get(sibling.peerId) ?? []);
|
|
4275
|
+
if (outcome === 'connected') {
|
|
3990
4276
|
dialed.push(sibling.peerId);
|
|
4277
|
+
} else if (outcome === 'failed') {
|
|
4278
|
+
unconnected = true;
|
|
3991
4279
|
}
|
|
3992
4280
|
}
|
|
3993
|
-
log('reconcileControlCohort: pass complete (siblings=%d, selected=%d, dialed=%d)',
|
|
3994
|
-
siblings.length, dials.length, dialed.length);
|
|
3995
|
-
return { dialed };
|
|
3996
|
-
}
|
|
3997
|
-
|
|
3998
|
-
/**
|
|
3999
|
-
* The owner-only half of the reconcile pass's ledger-marker step: file the singleton
|
|
4000
|
-
* `Revocation` marker ({@link SeedBootstrapService.openRevocationLedger}) unless this
|
|
4001
|
-
* process has already seen it filed. The connectivity gate is the caller's
|
|
4002
|
-
* ({@link runReconcileControlCohort}); why the marker exists is on
|
|
4003
|
-
* {@link ControlDatabase.openRevocationLedger}.
|
|
4004
|
-
*
|
|
4005
|
-
* Sets {@link revocationLedgerOpened} on `'opened'` or `'already-open'`. A node that
|
|
4006
|
-
* cannot sign as an owner does nothing: it picks the owner's marker up the first time it
|
|
4007
|
-
* reads the table, like any other replicated row. Best-effort like every step of the
|
|
4008
|
-
* pass — a failure is logged and leaves the flag clear, so the next connected pass retries.
|
|
4009
|
-
*/
|
|
4010
|
-
private async openRevocationLedgerIfDue(): Promise<void> {
|
|
4011
|
-
if (this.revocationLedgerOpened || !this.seedBootstrapService?.canAuthorize()) {
|
|
4012
|
-
return;
|
|
4013
|
-
}
|
|
4014
|
-
try {
|
|
4015
|
-
const outcome = await this.seedBootstrapService.openRevocationLedger();
|
|
4016
|
-
this.revocationLedgerOpened = true;
|
|
4017
|
-
log('reconcileControlCohort: revocation ledger marker %s', outcome);
|
|
4018
|
-
} catch (error) {
|
|
4019
|
-
log('reconcileControlCohort: filing the revocation ledger marker failed (retrying next pass): %o', error);
|
|
4020
|
-
}
|
|
4281
|
+
log('reconcileControlCohort: pass complete (siblings=%d, selected=%d, dialed=%d, unconnected=%s)',
|
|
4282
|
+
siblings.length, dials.length, dialed.length, unconnected);
|
|
4283
|
+
return { dialed, unconnected };
|
|
4021
4284
|
}
|
|
4022
4285
|
|
|
4023
4286
|
/**
|
|
@@ -4043,8 +4306,9 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4043
4306
|
* current ({@link refreshDialHint}).
|
|
4044
4307
|
*
|
|
4045
4308
|
* NOTE: this resolves EVERY sibling serially before the dial loop below runs,
|
|
4046
|
-
* so it costs one record query per sibling per reconcile pass (
|
|
4047
|
-
*
|
|
4309
|
+
* so it costs one record query per sibling per reconcile pass (served from the
|
|
4310
|
+
* membership rows view while it is current) and each one delays the pass's first
|
|
4311
|
+
* dial. A cadre is a handful of devices, so today
|
|
4048
4312
|
* that is a few extra local reads; if cadres ever grow large, batch the records
|
|
4049
4313
|
* into one query, merge only on change, or move the warm pass after the dials.
|
|
4050
4314
|
*/
|
|
@@ -4180,26 +4444,28 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4180
4444
|
}
|
|
4181
4445
|
|
|
4182
4446
|
/**
|
|
4183
|
-
* Dial one sibling from its already-resolved addresses, best-effort.
|
|
4184
|
-
* whether the dial resolved (false when no address resolves or the dial fails).
|
|
4447
|
+
* Dial one sibling from its already-resolved addresses, best-effort.
|
|
4185
4448
|
*
|
|
4186
4449
|
* A per-peer failure (NAT, offline, relay down, connection-gater denial, or
|
|
4187
4450
|
* the {@link controlDialBudget} expiring) is logged and swallowed so one
|
|
4188
4451
|
* unreachable sibling never aborts the pass — exactly like
|
|
4189
4452
|
* {@link SeedBootstrapService.applySeed}'s owner-dial loop. A failed dial is
|
|
4190
|
-
*
|
|
4191
|
-
*
|
|
4192
|
-
*
|
|
4193
|
-
|
|
4194
|
-
|
|
4453
|
+
* retried on the pass's redial ladder ({@link armReconcileWake}: 15 s, 30 s, 60 s,
|
|
4454
|
+
* then every backstop) and on any pass a change starts sooner. A sibling with no
|
|
4455
|
+
* dialable address, or whose every address relays through this node
|
|
4456
|
+
* ({@link SelfRelayOnlyError}), is not dialed at all and is `undialable`: it gets a
|
|
4457
|
+
* one-line log and no redial, since only a change — its record replicating in, or
|
|
4458
|
+
* the sibling connecting — can make it dialable, and each of those starts a pass.
|
|
4459
|
+
*/
|
|
4460
|
+
private async dialControlSibling(sibling: CohortPeerRow, resolved: Multiaddr[]): Promise<SiblingDialOutcome> {
|
|
4195
4461
|
const controlNode = this.controlNode;
|
|
4196
4462
|
if (!controlNode) {
|
|
4197
|
-
return
|
|
4463
|
+
return 'undialable';
|
|
4198
4464
|
}
|
|
4199
4465
|
const addrs = await this.resolveControlDialAddrs(sibling.peerId, resolved);
|
|
4200
4466
|
if (addrs.length === 0) {
|
|
4201
4467
|
log('reconcileControlCohort: no dialable control address for sibling %s; skipping', sibling.peerId);
|
|
4202
|
-
return
|
|
4468
|
+
return 'undialable';
|
|
4203
4469
|
}
|
|
4204
4470
|
try {
|
|
4205
4471
|
log('reconcileControlCohort: dialing sibling %s (%d addr(s))', sibling.peerId, addrs.length);
|
|
@@ -4210,14 +4476,14 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4210
4476
|
this.controlDialBudget(),
|
|
4211
4477
|
`reconcileControlCohort dial of sibling ${sibling.peerId}`
|
|
4212
4478
|
);
|
|
4213
|
-
return
|
|
4479
|
+
return 'connected';
|
|
4214
4480
|
} catch (error) {
|
|
4215
4481
|
if (error instanceof SelfRelayOnlyError) {
|
|
4216
4482
|
log('reconcileControlCohort: sibling %s is reachable only by relaying through this node; waiting for it to reconnect', sibling.peerId);
|
|
4217
|
-
|
|
4218
|
-
log('reconcileControlCohort: dial of sibling %s failed (continuing): %o', sibling.peerId, error);
|
|
4483
|
+
return 'undialable';
|
|
4219
4484
|
}
|
|
4220
|
-
|
|
4485
|
+
log('reconcileControlCohort: dial of sibling %s failed (continuing): %o', sibling.peerId, error);
|
|
4486
|
+
return 'failed';
|
|
4221
4487
|
}
|
|
4222
4488
|
}
|
|
4223
4489
|
|
|
@@ -4272,7 +4538,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4272
4538
|
* These addresses are never merged into the libp2p address book: they are
|
|
4273
4539
|
* unverified, and the address book takes only verified ones (see
|
|
4274
4540
|
* `mergePeerAddrs`). Layers that dial by bare peer id use the connection this
|
|
4275
|
-
* dial opens, and after it drops the
|
|
4541
|
+
* dial opens, and after it drops the reconcile pass the close starts dials again.
|
|
4276
4542
|
*
|
|
4277
4543
|
* NOTE: an entry is consulted here only for a current sibling, so a peer revoked
|
|
4278
4544
|
* by another owner is not dialed from it, even though the entry stays (only this
|
|
@@ -4327,8 +4593,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4327
4593
|
* Called for every seed this node accepts, on BOTH intake paths — the
|
|
4328
4594
|
* {@link applySeed} wrapper (which may run on a throwaway service, so the
|
|
4329
4595
|
* service itself is the wrong place to keep this) and the inbound
|
|
4330
|
-
* `/sereus/seed/1.0.0` handler via `onSeedApplied`.
|
|
4331
|
-
* skipped: there is nothing to dial.
|
|
4596
|
+
* `/sereus/seed/1.0.0` handler via `onSeedApplied`.
|
|
4332
4597
|
*
|
|
4333
4598
|
* `preferred` is the seed's own claim about its peers, exactly as
|
|
4334
4599
|
* `SeedBootstrapService.applySeed` uses it to choose its dial targets. That is
|
|
@@ -4336,14 +4601,15 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4336
4601
|
* signature-checked against a trust-anchored signer before this runs, and the
|
|
4337
4602
|
* flag only *selects a dial target*; a dial grants no authority.
|
|
4338
4603
|
*
|
|
4339
|
-
*
|
|
4604
|
+
* The peers retained are `seedDialTargets` (addressless peers and self dropped), the same
|
|
4605
|
+
* ones the service dials; they are returned for the `seed:applied` payload.
|
|
4340
4606
|
*/
|
|
4341
|
-
private recordSeedBootstrapPeers(seed: ControlNetworkSeed):
|
|
4342
|
-
|
|
4343
|
-
|
|
4344
|
-
|
|
4345
|
-
}
|
|
4607
|
+
private recordSeedBootstrapPeers(seed: ControlNetworkSeed): SeedDialTarget[] {
|
|
4608
|
+
const targets = seedDialTargets(seed, this.controlNode?.peerId.toString());
|
|
4609
|
+
for (const { peerId, multiaddrs } of targets) {
|
|
4610
|
+
this.retainDialTarget(peerId, multiaddrs, 'recordSeedBootstrapPeers');
|
|
4346
4611
|
}
|
|
4612
|
+
return targets;
|
|
4347
4613
|
}
|
|
4348
4614
|
|
|
4349
4615
|
/**
|
|
@@ -4358,9 +4624,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4358
4624
|
* failing a seed or an add that has already been accepted.
|
|
4359
4625
|
*
|
|
4360
4626
|
* Two peers are never retained. One with no address: there is nothing to dial.
|
|
4361
|
-
* And self: `
|
|
4362
|
-
*
|
|
4363
|
-
* could be handed this node's own id. Retaining self would make the cold-start
|
|
4627
|
+
* And self: `addDrone` could be handed this node's own id (seed intake arrives
|
|
4628
|
+
* already filtered by `seedDialTargets`). Retaining self would make the cold-start
|
|
4364
4629
|
* pass dial this node forever (the steady-state pass filters self out of its
|
|
4365
4630
|
* sibling list for the same reason).
|
|
4366
4631
|
*/
|
|
@@ -4395,33 +4660,35 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4395
4660
|
* help: it dials only siblings enumerated from that very table. Retrying here
|
|
4396
4661
|
* is the only way back in.
|
|
4397
4662
|
*
|
|
4398
|
-
* Unbounded
|
|
4399
|
-
*
|
|
4400
|
-
*
|
|
4401
|
-
* a
|
|
4402
|
-
*
|
|
4403
|
-
*
|
|
4404
|
-
*
|
|
4405
|
-
*
|
|
4406
|
-
*
|
|
4663
|
+
* Unbounded: the branch is already gated by "the control database has no
|
|
4664
|
+
* siblings", so it stops the moment the node is actually in the party, and a node
|
|
4665
|
+
* that is stranded MUST keep trying — a give-up rule would turn a transient outage
|
|
4666
|
+
* into a permanent one. A failed dial earns the pass a redial wake on the ladder
|
|
4667
|
+
* {@link armReconcileWake} runs (15 s, 30 s, 60 s, then every backstop, 2 minutes
|
|
4668
|
+
* by default), so the steady cost is one dial per bootstrap peer per backstop (a
|
|
4669
|
+
* seed prefers its minter and the party's always-on machines, a few at most), each
|
|
4670
|
+
* failing fast against an unreachable address.
|
|
4671
|
+
* NOTE: the ladder is per pass, not per peer; if seeds ever carry many preferred
|
|
4672
|
+
* peers, add per-peer backoff here.
|
|
4407
4673
|
*
|
|
4408
4674
|
* Best-effort per peer, exactly like {@link dialControlSibling}: one dead
|
|
4409
4675
|
* address never aborts the pass.
|
|
4410
4676
|
*/
|
|
4411
|
-
private async dialColdStartBootstrap(): Promise<
|
|
4677
|
+
private async dialColdStartBootstrap(): Promise<ControlCohortPassOutcome> {
|
|
4412
4678
|
const controlNode = this.controlNode;
|
|
4413
4679
|
// Snapshot up front: the store hands back a copy, so a seed applied mid-pass
|
|
4414
4680
|
// cannot mutate what we are iterating.
|
|
4415
4681
|
const targets = this.bootstrapPeerStore?.all();
|
|
4416
4682
|
if (!controlNode || !targets || targets.size === 0) {
|
|
4417
|
-
return [];
|
|
4683
|
+
return { dialed: [], unconnected: false };
|
|
4418
4684
|
}
|
|
4419
4685
|
// Same skip rule as step 3 of the steady-state pass: no churn on live links.
|
|
4420
4686
|
const connected = new Set(controlNode.getConnections().map((c) => c.remotePeer.toString()));
|
|
4421
4687
|
const dialed: string[] = [];
|
|
4688
|
+
let unconnected = false;
|
|
4422
4689
|
for (const [peerId, entry] of targets) {
|
|
4423
4690
|
if (!this._running || !this.controlNode) {
|
|
4424
|
-
return dialed;
|
|
4691
|
+
return { dialed, unconnected: false };
|
|
4425
4692
|
}
|
|
4426
4693
|
// NOTE: a connection this node still holds in `status: 'open'` after the remote
|
|
4427
4694
|
// has already aborted it counts as connected here, so the cold-start retry stays
|
|
@@ -4435,11 +4702,13 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4435
4702
|
}
|
|
4436
4703
|
if (await this.dialBootstrapPeer(peerId, entry.addrs)) {
|
|
4437
4704
|
dialed.push(peerId);
|
|
4705
|
+
} else {
|
|
4706
|
+
unconnected = true;
|
|
4438
4707
|
}
|
|
4439
4708
|
}
|
|
4440
|
-
log('reconcileControlCohort: cold-start pass complete (bootstrap=%d, connected=%d, dialed=%d)',
|
|
4441
|
-
targets.size, connected.size, dialed.length);
|
|
4442
|
-
return dialed;
|
|
4709
|
+
log('reconcileControlCohort: cold-start pass complete (bootstrap=%d, connected=%d, dialed=%d, unconnected=%s)',
|
|
4710
|
+
targets.size, connected.size, dialed.length, unconnected);
|
|
4711
|
+
return { dialed, unconnected };
|
|
4443
4712
|
}
|
|
4444
4713
|
|
|
4445
4714
|
/**
|
|
@@ -4458,32 +4727,36 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4458
4727
|
return this.normalizeDialAddrs(this.parseMultiaddrs(addrs), peerId);
|
|
4459
4728
|
}
|
|
4460
4729
|
|
|
4461
|
-
/**
|
|
4462
|
-
|
|
4730
|
+
/**
|
|
4731
|
+
* Dial one peer from its retained dial target's addresses, best-effort: a cold-start bootstrap
|
|
4732
|
+
* peer, or a just-claimed node ({@link redialClaimedNode}), which `context` names in the logs.
|
|
4733
|
+
* Returns whether it connected.
|
|
4734
|
+
*/
|
|
4735
|
+
private async dialBootstrapPeer(peerId: string, addrs: string[], context = 'reconcileControlCohort(cold-start)'): Promise<boolean> {
|
|
4463
4736
|
const controlNode = this.controlNode;
|
|
4464
4737
|
if (!controlNode) {
|
|
4465
4738
|
return false;
|
|
4466
4739
|
}
|
|
4467
4740
|
const parsed = this.bootstrapDialAddrs(peerId, addrs);
|
|
4468
4741
|
if (parsed.length === 0) {
|
|
4469
|
-
log('
|
|
4742
|
+
log('%s: no dialable address for bootstrap peer %s; skipping', context, peerId);
|
|
4470
4743
|
return false;
|
|
4471
4744
|
}
|
|
4472
4745
|
try {
|
|
4473
|
-
log('
|
|
4746
|
+
log('%s: dialing bootstrap peer %s (%d addr(s))', context, peerId, parsed.length);
|
|
4474
4747
|
await dialPeerAddrs(
|
|
4475
4748
|
controlNode,
|
|
4476
4749
|
peerId,
|
|
4477
4750
|
parsed,
|
|
4478
4751
|
this.controlDialBudget(),
|
|
4479
|
-
|
|
4752
|
+
`${context} dial of bootstrap peer ${peerId}`
|
|
4480
4753
|
);
|
|
4481
4754
|
return true;
|
|
4482
4755
|
} catch (error) {
|
|
4483
4756
|
if (error instanceof SelfRelayOnlyError) {
|
|
4484
|
-
log('
|
|
4757
|
+
log('%s: bootstrap peer %s is reachable only by relaying through this node; waiting for it to reconnect', context, peerId);
|
|
4485
4758
|
} else {
|
|
4486
|
-
log('
|
|
4759
|
+
log('%s: dial of bootstrap peer %s failed (continuing): %o', context, peerId, error);
|
|
4487
4760
|
}
|
|
4488
4761
|
return false;
|
|
4489
4762
|
}
|
|
@@ -4494,32 +4767,34 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4494
4767
|
//
|
|
4495
4768
|
// Optimystic's coordinator commits a control write WITHOUT broadcasting when the
|
|
4496
4769
|
// block's cluster has ≤1 member. An owner that authorizes/removes a peer or
|
|
4497
|
-
// (re)publishes its own record while no sibling is
|
|
4770
|
+
// (re)publishes its own record while no sibling is reachable therefore writes a
|
|
4498
4771
|
// row that exists ONLY in its local control DB — a sibling that connects later
|
|
4499
4772
|
// never observes it (pull-on-read routes to the block's cluster, which never
|
|
4500
4773
|
// learned of the write). reconcileControlCohort shrinks the alone window but
|
|
4501
|
-
// cannot close it (a write in the instant before any sibling
|
|
4502
|
-
// local-only). The remedy: detect "wrote while alone" (
|
|
4503
|
-
//
|
|
4504
|
-
//
|
|
4774
|
+
// cannot close it (a write in the instant before any sibling identifies is still
|
|
4775
|
+
// local-only). The remedy: detect "wrote while alone" (no identified sibling the
|
|
4776
|
+
// cohort would admit: committedAlone), queue the affected row, and RE-ISSUE the write
|
|
4777
|
+
// once one is reachable, as an idempotent monotonic update that now broadcasts.
|
|
4505
4778
|
// See docs/architecture.md (Control Network Convergence → "Writes made while alone").
|
|
4506
4779
|
// ============================================================================
|
|
4507
4780
|
|
|
4508
4781
|
/**
|
|
4509
|
-
* Whether a control write happening now would commit local-only
|
|
4510
|
-
*
|
|
4511
|
-
*
|
|
4512
|
-
*
|
|
4513
|
-
*
|
|
4514
|
-
*
|
|
4515
|
-
*
|
|
4516
|
-
* is acceptable as the agreed first cut.
|
|
4782
|
+
* Whether a control write happening now would commit local-only: no sibling is reachable
|
|
4783
|
+
* ({@link hasReachableControlSibling}, the public form, so the two cannot drift). A relay, a
|
|
4784
|
+
* bootstrap node or a peer that has not identified yet holds a connection without being in the
|
|
4785
|
+
* write's cohort, so a write made with only those connected is queued too. One made alone at
|
|
4786
|
+
* the revision the cohort reached meanwhile is a fork no later consult sees, because a consult
|
|
4787
|
+
* compares revision numbers only; the re-issue on the growth edge is what conflicts with the
|
|
4788
|
+
* cohort's copy and pulls it in.
|
|
4517
4789
|
*
|
|
4518
|
-
*
|
|
4519
|
-
* the
|
|
4790
|
+
* Still a proxy for the precise signal (the block's own cohort): a reachable sibling outside
|
|
4791
|
+
* this block's cohort leaves the write local-only, caught by the cohort's pull-on-read. On the
|
|
4792
|
+
* control network every block's cohort is the whole party, so that gap is a party larger than
|
|
4793
|
+
* `CONTROL_REPLICATION_BREADTH`. Re-issuing a replicated row is harmless (an idempotent
|
|
4794
|
+
* monotonic bump).
|
|
4520
4795
|
*/
|
|
4521
4796
|
private committedAlone(): boolean {
|
|
4522
|
-
return this.
|
|
4797
|
+
return !this.hasReachableControlSibling();
|
|
4523
4798
|
}
|
|
4524
4799
|
|
|
4525
4800
|
/**
|
|
@@ -4576,7 +4851,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4576
4851
|
/**
|
|
4577
4852
|
* Drain the write-while-alone re-replication queue: re-issue the writes that
|
|
4578
4853
|
* committed local-only now that the cohort can broadcast them. Public so the
|
|
4579
|
-
*
|
|
4854
|
+
* growth trigger ({@link updateControlSiblingReachability}) and tests can drive
|
|
4580
4855
|
* it; concurrent calls collapse into one in-flight drain
|
|
4581
4856
|
* ({@link drainControlReplicationInFlight}) so two growth signals never
|
|
4582
4857
|
* double-issue. Best-effort throughout — a per-row failure leaves that entry
|
|
@@ -4618,13 +4893,15 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4618
4893
|
// periodic-refresh property.
|
|
4619
4894
|
await this.drainPendingRevocations();
|
|
4620
4895
|
|
|
4621
|
-
// 2. Self rows.
|
|
4622
|
-
//
|
|
4623
|
-
//
|
|
4896
|
+
// 2. Self rows. A self write made while alone is always re-touched: the re-touch
|
|
4897
|
+
// is what carries it to the cohort. Otherwise the first growth re-touches a row
|
|
4898
|
+
// a prior process may have committed local-only, unless the row already holds a
|
|
4899
|
+
// current record this process signed (a redeemed invitation's admission).
|
|
4624
4900
|
// registerSelf is idempotent + single-flight, so this never races a
|
|
4625
4901
|
// duplicate publish with the heartbeat republish.
|
|
4626
4902
|
if (firstGrowth || this.pendingSelfPeerWrite) {
|
|
4627
|
-
|
|
4903
|
+
const publish = this.pendingSelfPeerWrite ? this.registerSelf() : this.publishSelfRecordUnlessCurrent();
|
|
4904
|
+
await publish.catch((error) => log('drain: registerSelf failed: %o', error));
|
|
4628
4905
|
this.pendingSelfPeerWrite = false;
|
|
4629
4906
|
}
|
|
4630
4907
|
if (firstGrowth || this.pendingSelfDeviceWrite) {
|
|
@@ -4782,8 +5059,11 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4782
5059
|
|
|
4783
5060
|
/**
|
|
4784
5061
|
* Body of {@link reconstructAuthoredMembership}'s sweep: re-issue each candidate
|
|
4785
|
-
* owner-authored row, returning how many were re-touched. A
|
|
4786
|
-
*
|
|
5062
|
+
* owner-authored row, returning how many were re-touched. A row the sweep already holds as
|
|
5063
|
+
* self-published is skipped without a call; the re-touch itself decides again over the row
|
|
5064
|
+
* it reads under the lock, so a row that vanished or was self-published since the sweep's
|
|
5065
|
+
* read is reported, not written. A per-row failure is logged and skipped; a teardown
|
|
5066
|
+
* mid-sweep stops it.
|
|
4787
5067
|
*/
|
|
4788
5068
|
private async reissueAuthoredMembershipRows(selfPeerId: string, members: CadrePeerRow[]): Promise<number> {
|
|
4789
5069
|
let touched = 0;
|
|
@@ -4791,17 +5071,15 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4791
5071
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
4792
5072
|
return touched;
|
|
4793
5073
|
}
|
|
4794
|
-
|
|
5074
|
+
// Self, queued for the drain (step 4 re-touches it once), or peer-owned (self-Sig
|
|
5075
|
+
// present) → not this sweep's to bump.
|
|
5076
|
+
if (member.peerId === selfPeerId || this.pendingPeerWrites.has(member.peerId) || member.sig) {
|
|
4795
5077
|
continue;
|
|
4796
5078
|
}
|
|
4797
5079
|
try {
|
|
4798
|
-
|
|
4799
|
-
|
|
4800
|
-
if (!record || record.sig) {
|
|
4801
|
-
continue;
|
|
5080
|
+
if (await this.reissuePeerAuthorize(member.peerId) === 'retouched') {
|
|
5081
|
+
touched++;
|
|
4802
5082
|
}
|
|
4803
|
-
await this.reissuePeerAuthorize(member.peerId, record.updatedAt);
|
|
4804
|
-
touched++;
|
|
4805
5083
|
} catch (error) {
|
|
4806
5084
|
log('drain: reconstruction re-touch of %s failed (continuing): %o', member.peerId, error);
|
|
4807
5085
|
}
|
|
@@ -4836,9 +5114,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4836
5114
|
}
|
|
4837
5115
|
|
|
4838
5116
|
/**
|
|
4839
|
-
* Body of {@link drainPendingPeerWrites}' loop. An entry is cleared
|
|
4840
|
-
*
|
|
4841
|
-
* the
|
|
5117
|
+
* Body of {@link drainPendingPeerWrites}' loop. An entry is cleared on every outcome the
|
|
5118
|
+
* re-touch reports — re-touched, or nothing left to re-issue because the row was removed
|
|
5119
|
+
* since or the peer self-published; the re-touch decides which over the row it reads under
|
|
5120
|
+
* the lock. A failure is logged and the entry stays queued for the next cohort growth.
|
|
4842
5121
|
*/
|
|
4843
5122
|
private async reissuePendingPeerWrites(): Promise<void> {
|
|
4844
5123
|
for (const peerId of [...this.pendingPeerWrites.keys()]) {
|
|
@@ -4846,13 +5125,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4846
5125
|
return;
|
|
4847
5126
|
}
|
|
4848
5127
|
try {
|
|
4849
|
-
const
|
|
4850
|
-
|
|
4851
|
-
// Removed since, or peer self-published — nothing for the owner to re-issue.
|
|
4852
|
-
this.pendingPeerWrites.delete(peerId);
|
|
4853
|
-
continue;
|
|
4854
|
-
}
|
|
4855
|
-
await this.reissuePeerAuthorize(peerId, record.updatedAt);
|
|
5128
|
+
const outcome = await this.reissuePeerAuthorize(peerId);
|
|
5129
|
+
log('drain: re-issue of %s (authorize): %s', peerId, outcome);
|
|
4856
5130
|
this.pendingPeerWrites.delete(peerId);
|
|
4857
5131
|
} catch (error) {
|
|
4858
5132
|
log('drain: re-issue of %s (authorize) failed; leaving queued for next growth: %o', peerId, error);
|
|
@@ -4865,60 +5139,202 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4865
5139
|
* `UpdatedAt` strictly above the stored value (and the wall clock) and re-sign via
|
|
4866
5140
|
* the owner branch of `CadrePeer.AuthorizedUpdate`. The row already exists
|
|
4867
5141
|
* locally (it committed there, just local-only), so this is an UPDATE, not the
|
|
4868
|
-
* original INSERT.
|
|
5142
|
+
* original INSERT. The bump, and whether the row is still the owner's to re-issue, are
|
|
5143
|
+
* decided over the row as read inside the write ({@link SeedBootstrapService.reauthorizePeer}'s
|
|
5144
|
+
* re-broadcast mode).
|
|
4869
5145
|
*/
|
|
4870
|
-
private async reissuePeerAuthorize(peerId: string
|
|
5146
|
+
private async reissuePeerAuthorize(peerId: string): Promise<CadrePeerRetouchOutcome> {
|
|
4871
5147
|
if (!this.seedBootstrapService) {
|
|
4872
|
-
|
|
5148
|
+
throw new Error('Seed bootstrap service not initialized. Call initializeSeedBootstrap() first.');
|
|
4873
5149
|
}
|
|
4874
|
-
|
|
4875
|
-
await this.seedBootstrapService.reauthorizePeer(peerId, updatedAt);
|
|
5150
|
+
return await this.seedBootstrapService.reauthorizePeer(peerId, 'rebroadcast');
|
|
4876
5151
|
}
|
|
4877
5152
|
|
|
4878
5153
|
/**
|
|
4879
|
-
*
|
|
4880
|
-
* write-while-alone re-replication queue (single-flight;
|
|
4881
|
-
* not on
|
|
4882
|
-
*
|
|
5154
|
+
* Re-evaluate {@link hasReachableControlSibling} and act on its edges: on false→true, drain
|
|
5155
|
+
* the write-while-alone re-replication queue (single-flight; best-effort, a drain failure is
|
|
5156
|
+
* logged, not thrown); on true→false, re-arm, so the next reachable sibling drains again.
|
|
5157
|
+
* Called when a peer identifies, when a connection closes, and after every membership refresh,
|
|
5158
|
+
* where a sibling that connected before this node authorized it (the production join order)
|
|
5159
|
+
* becomes reachable. A device counted in for its admission ({@link admittingDevices}) is not
|
|
5160
|
+
* a trigger: the admission's own writes are in flight, and its row landing refreshes
|
|
5161
|
+
* membership.
|
|
4883
5162
|
*/
|
|
4884
|
-
private
|
|
5163
|
+
private updateControlSiblingReachability(): void {
|
|
4885
5164
|
if (!this._running || !this.controlNode) {
|
|
4886
5165
|
return;
|
|
4887
5166
|
}
|
|
4888
|
-
const
|
|
4889
|
-
if (
|
|
5167
|
+
const reachable = this.hasReachableControlSibling();
|
|
5168
|
+
if (reachable === this.controlSiblingReachable) {
|
|
5169
|
+
return;
|
|
5170
|
+
}
|
|
5171
|
+
this.controlSiblingReachable = reachable;
|
|
5172
|
+
if (!reachable) {
|
|
4890
5173
|
return;
|
|
4891
5174
|
}
|
|
4892
|
-
|
|
4893
|
-
|
|
4894
|
-
|
|
4895
|
-
.catch((error) => log('Control re-replication drain (connection:open) failed: %o', error));
|
|
5175
|
+
log('Control cohort grew (a sibling is reachable); draining write-while-alone re-replication queue');
|
|
5176
|
+
void this.drainPendingControlReplication('sibling-reachable')
|
|
5177
|
+
.catch((error) => log('Control re-replication drain (sibling-reachable) failed: %o', error));
|
|
4896
5178
|
}
|
|
4897
5179
|
|
|
4898
|
-
/**
|
|
4899
|
-
|
|
4900
|
-
|
|
5180
|
+
/**
|
|
5181
|
+
* A connected peer identified as serving the control database: count it
|
|
5182
|
+
* ({@link identifiedControlPeers}), re-evaluate the growth edge, and run the sibling-open
|
|
5183
|
+
* reactions, once per connected period. libp2p re-emits `peer:identify` on every
|
|
5184
|
+
* identify-push from a connected peer, and those repeats start nothing.
|
|
5185
|
+
*/
|
|
5186
|
+
private onControlPeerIdentified(peerId: PeerId): void {
|
|
5187
|
+
const id = peerId.toString();
|
|
5188
|
+
// An identify that reports after its connection closed counts nobody.
|
|
5189
|
+
if (this.identifiedControlPeers.has(id) || !this.controlNode || this.controlNode.getConnections(peerId).length === 0) {
|
|
5190
|
+
return;
|
|
5191
|
+
}
|
|
5192
|
+
this.identifiedControlPeers.add(id);
|
|
5193
|
+
this.updateControlSiblingReachability();
|
|
5194
|
+
this.onControlSiblingConnectionChange(id, 'open');
|
|
5195
|
+
}
|
|
5196
|
+
|
|
5197
|
+
/**
|
|
5198
|
+
* A control connection closed: `remotePeer` stops counting once it has none left, and the
|
|
5199
|
+
* growth edge re-arms if no reachable sibling remains. The sibling-close reaction runs on every
|
|
5200
|
+
* close. libp2p's connection manager drops the closed connection from `getConnections` before
|
|
5201
|
+
* this listener runs: libp2p dispatches each event on its internal bus, where the connection
|
|
5202
|
+
* manager listens, before re-dispatching it to the node's own listeners.
|
|
5203
|
+
*/
|
|
5204
|
+
private onControlConnectionClosed(remotePeer: PeerId): void {
|
|
5205
|
+
const id = remotePeer.toString();
|
|
5206
|
+
if (!this.controlNode || this.controlNode.getConnections(remotePeer).length === 0) {
|
|
5207
|
+
this.identifiedControlPeers.delete(id);
|
|
5208
|
+
}
|
|
5209
|
+
this.updateControlSiblingReachability();
|
|
5210
|
+
this.onControlSiblingConnectionChange(id, 'close');
|
|
5211
|
+
}
|
|
5212
|
+
|
|
5213
|
+
/**
|
|
5214
|
+
* Count the peers that identified before {@link wireControlConnectionListeners} ran, whose
|
|
5215
|
+
* `peer:identify` has already fired, from their peer-store records. Updates the growth edge
|
|
5216
|
+
* but runs no sibling-open reaction, as a connection that predates the listeners never did. A
|
|
5217
|
+
* peer not yet in the peer store is still identifying, and its own event counts it.
|
|
5218
|
+
*/
|
|
5219
|
+
private async noteIdentifiedControlPeers(): Promise<void> {
|
|
5220
|
+
const node = this.controlNode;
|
|
5221
|
+
if (!node) {
|
|
4901
5222
|
return;
|
|
4902
5223
|
}
|
|
4903
|
-
|
|
4904
|
-
|
|
5224
|
+
const connected = new Map<string, PeerId>();
|
|
5225
|
+
for (const connection of node.getConnections()) {
|
|
5226
|
+
connected.set(connection.remotePeer.toString(), connection.remotePeer);
|
|
5227
|
+
}
|
|
5228
|
+
const partyId = this.config.controlNetwork.partyId;
|
|
5229
|
+
await Promise.all([...connected.values()].map(async (peerId) => {
|
|
5230
|
+
try {
|
|
5231
|
+
const { protocols } = await node.peerStore.get(peerId);
|
|
5232
|
+
// Still wired to this node: a stop while the read was out cleared the set.
|
|
5233
|
+
const wired = this.controlPeerIdentifyHandler !== null && this.controlNode === node;
|
|
5234
|
+
if (wired && servesControlDatabase(partyId, protocols) && node.getConnections(peerId).length > 0) {
|
|
5235
|
+
this.identifiedControlPeers.add(peerId.toString());
|
|
5236
|
+
}
|
|
5237
|
+
} catch (error) {
|
|
5238
|
+
if ((error as Error).name !== 'NotFoundError') {
|
|
5239
|
+
log('Peer store read for connected control peer %s failed — left to its identify: %o', peerId.toString(), error);
|
|
5240
|
+
}
|
|
5241
|
+
}
|
|
5242
|
+
}));
|
|
5243
|
+
this.updateControlSiblingReachability();
|
|
5244
|
+
}
|
|
5245
|
+
|
|
5246
|
+
/**
|
|
5247
|
+
* An AUTHORIZED sibling became reachable (its identify, once per connected period:
|
|
5248
|
+
* {@link onControlPeerIdentified}) or one of its connections closed: start a reconcile
|
|
5249
|
+
* pass. On open the pass asks the new sibling for strand addresses and re-arms the
|
|
5250
|
+
* delegate grants; on close it re-dials. First, on open, drop the membership rows
|
|
5251
|
+
* views: a machine that was offline while a sibling committed learns of the commit
|
|
5252
|
+
* only when it next reads the block from storage, and dropping the views makes the
|
|
5253
|
+
* pass's reads do that now rather than after the views' maximum age. Those reads can
|
|
5254
|
+
* fall inside the read-repair window, so the open also arms a second drop past it
|
|
5255
|
+
* ({@link armMembershipViewReload}). Any other
|
|
5256
|
+
* peer — a relay, a bootstrap node, another party's machine — starts nothing.
|
|
5257
|
+
*
|
|
5258
|
+
* On identify rather than `connection:open` because, until the sibling identifies,
|
|
5259
|
+
* Optimystic's cohort for every control block is this node alone: a pass started at the
|
|
5260
|
+
* open writes (the first self-record publish) and reads on a cohort of one.
|
|
5261
|
+
*/
|
|
5262
|
+
private onControlSiblingConnectionChange(remotePeer: string, change: 'open' | 'close'): void {
|
|
5263
|
+
if (!this.authorizedControlPeers.has(remotePeer)) {
|
|
5264
|
+
return;
|
|
4905
5265
|
}
|
|
5266
|
+
if (change === 'open') {
|
|
5267
|
+
// NOTE: a machine removed from the cadre that has not learned of it keeps redialing
|
|
5268
|
+
// its siblings, each of which admits it provisionally and hangs up at the deadline
|
|
5269
|
+
// (28.5 s at the default link), and every such open drops the views here and once
|
|
5270
|
+
// more past the read-repair window: two storage reloads, with their cohort consults
|
|
5271
|
+
// refused, per cycle until it learns. If a removed machine's churn ever shows up,
|
|
5272
|
+
// skip both drops for a peer whose previous connection the remote closed inside the
|
|
5273
|
+
// provisional deadline.
|
|
5274
|
+
this.controlDatabase?.invalidateMembershipViews();
|
|
5275
|
+
this.armMembershipViewReload();
|
|
5276
|
+
}
|
|
5277
|
+
// NOTE: a close starts a pass at once, and a sibling in its bring-up quiet period lets the
|
|
5278
|
+
// dial resolve and then drops the connection (`createMembershipConnectionGater`'s deny
|
|
5279
|
+
// timing), so a restarting sibling costs one dial-connect-close pass after another for its
|
|
5280
|
+
// whole bring-up (5-6 passes in ~200 ms against a claimed node restarting locally). If a
|
|
5281
|
+
// slow device's bring-up ever makes that churn show up, delay the pass a close starts when
|
|
5282
|
+
// the previous connection to that sibling lived only a few milliseconds.
|
|
5283
|
+
log('Control connection to sibling %s %s; reconcile pass triggered', remotePeer, change);
|
|
5284
|
+
this.controlReconcileRuns?.trigger(0);
|
|
5285
|
+
}
|
|
5286
|
+
|
|
5287
|
+
/**
|
|
5288
|
+
* Drop the membership views again {@link MEMBERSHIP_VIEW_RECONNECT_RELOAD_MS} after a
|
|
5289
|
+
* sibling connects, and start a pass whose reads reload them. This node's reads before
|
|
5290
|
+
* the connection stamped Optimystic's read-repair window on the control blocks, so the
|
|
5291
|
+
* reload the open itself starts is served from the local copy with no cohort consult,
|
|
5292
|
+
* and the view would keep rows that predate the sibling's commits until its maximum age.
|
|
5293
|
+
* The reload past the window starts the consults; one that brings a newer revision
|
|
5294
|
+
* raises the table change that drops the view and refreshes the gate. Re-armed by every
|
|
5295
|
+
* open, so it runs one window after the latest.
|
|
5296
|
+
*
|
|
5297
|
+
* NOTE: a consult Optimystic defers (16 already in flight) starts nothing and leaves the block
|
|
5298
|
+
* unstamped, so the view keeps its rows until the first read past its maximum age, which on an
|
|
5299
|
+
* idle node is the reconcile backstop. If `cluster-tx:read-repair-deferred` lines show at a
|
|
5300
|
+
* reconnect, re-arm this reload once more instead of waiting for the backstop.
|
|
5301
|
+
*/
|
|
5302
|
+
private armMembershipViewReload(): void {
|
|
5303
|
+
if (this.membershipViewReloadTimer) {
|
|
5304
|
+
clearTimeout(this.membershipViewReloadTimer);
|
|
5305
|
+
}
|
|
5306
|
+
this.membershipViewReloadTimer = setTimeout(() => {
|
|
5307
|
+
this.membershipViewReloadTimer = null;
|
|
5308
|
+
log('Read-repair window passed since a sibling connected; reloading the membership views');
|
|
5309
|
+
this.controlDatabase?.invalidateMembershipViews();
|
|
5310
|
+
this.controlReconcileRuns?.trigger(0);
|
|
5311
|
+
}, MEMBERSHIP_VIEW_RECONNECT_RELOAD_MS);
|
|
5312
|
+
// Node-only: don't keep the event loop alive solely for this timer.
|
|
5313
|
+
(this.membershipViewReloadTimer as { unref?: () => void } | null)?.unref?.();
|
|
4906
5314
|
}
|
|
4907
5315
|
|
|
4908
5316
|
/**
|
|
4909
|
-
* Wire the control-connection
|
|
4910
|
-
*
|
|
4911
|
-
*
|
|
4912
|
-
*
|
|
5317
|
+
* Wire the control-connection listeners, which keep {@link identifiedControlPeers}: the
|
|
5318
|
+
* growth edge that drives write-while-alone re-replication, and the sibling reconcile
|
|
5319
|
+
* trigger ({@link onControlSiblingConnectionChange}). Wired in {@link start} (not the
|
|
5320
|
+
* delayed refresh path) so no early identify is missed, and reads back the peers that
|
|
5321
|
+
* identified before it ran ({@link noteIdentifiedControlPeers}); torn down in
|
|
5322
|
+
* {@link stopRecordRefresh}. Idempotent.
|
|
4913
5323
|
*/
|
|
4914
5324
|
private wireControlConnectionListeners(): void {
|
|
4915
|
-
if (!this.controlNode || this.
|
|
5325
|
+
if (!this.controlNode || this.controlPeerIdentifyHandler) {
|
|
4916
5326
|
return;
|
|
4917
5327
|
}
|
|
4918
|
-
|
|
4919
|
-
this.
|
|
4920
|
-
|
|
5328
|
+
const partyId = this.config.controlNetwork.partyId;
|
|
5329
|
+
this.controlPeerIdentifyHandler = (event) => {
|
|
5330
|
+
if (servesControlDatabase(partyId, event.detail.protocols)) {
|
|
5331
|
+
this.onControlPeerIdentified(event.detail.peerId);
|
|
5332
|
+
}
|
|
5333
|
+
};
|
|
5334
|
+
this.controlConnectionCloseHandler = (event) => this.onControlConnectionClosed(event.detail.remotePeer);
|
|
5335
|
+
this.controlNode.addEventListener('peer:identify', this.controlPeerIdentifyHandler);
|
|
4921
5336
|
this.controlNode.addEventListener('connection:close', this.controlConnectionCloseHandler);
|
|
5337
|
+
void this.noteIdentifiedControlPeers();
|
|
4922
5338
|
}
|
|
4923
5339
|
|
|
4924
5340
|
// ============================================================================
|
|
@@ -5249,6 +5665,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
5249
5665
|
if (sAppId !== undefined) {
|
|
5250
5666
|
this.replicaSAppIds.set(strandId, sAppId);
|
|
5251
5667
|
log('Storage replica %s serves sApp %s', strandId, sAppId);
|
|
5668
|
+
this.strandWatcher?.sAppIdLearned(strandId);
|
|
5252
5669
|
}
|
|
5253
5670
|
} catch (error) {
|
|
5254
5671
|
log('Could not read the sApp id of storage replica %s (left unknown): %o', strandId, error);
|
|
@@ -5371,6 +5788,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
5371
5788
|
this.strandLaunchRefusals.clear();
|
|
5372
5789
|
this.discoveredStrands.clear();
|
|
5373
5790
|
this.replicaSAppIds.clear();
|
|
5791
|
+
this.formedPartyKeys.clear();
|
|
5374
5792
|
|
|
5375
5793
|
// Drop delegate-admission state: the grants are scoped to the session that
|
|
5376
5794
|
// recorded them, so a stop()/start() cycle on this object must not keep
|
|
@@ -5394,6 +5812,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
5394
5812
|
// Detach the membership hub first: nothing this teardown does should drive a
|
|
5395
5813
|
// gate refresh, and a notification arriving after this point is a no-op.
|
|
5396
5814
|
if (this.controlDatabase) {
|
|
5815
|
+
// The database's registry would dispose the gate watch too; disposing it here, before
|
|
5816
|
+
// the listener goes, keeps a closing commit from scheduling a gate refresh.
|
|
5817
|
+
this.membershipGateWatch?.dispose();
|
|
5818
|
+
this.membershipGateWatch = null;
|
|
5397
5819
|
this.controlDatabase.setMembershipChangeListener(null);
|
|
5398
5820
|
this.controlDatabase.setGuardedDeleteListener(null);
|
|
5399
5821
|
this.controlDatabase.setControlWriteAbandonedListener(null);
|
|
@@ -5753,8 +6175,9 @@ export class CadreNode implements SAppIdLookup {
|
|
|
5753
6175
|
* registered, deliberately: both an explicit retry and the {@link StrandWatcher}'s
|
|
5754
6176
|
* automatic relaunch need it. A failed launch here hands the strand back to that
|
|
5755
6177
|
* relaunch — `StrandWatcher.forgetStrand` drops the id from the watcher's
|
|
5756
|
-
* `knownStrands
|
|
5757
|
-
*
|
|
6178
|
+
* `knownStrands`, records the backoff and wakes the watcher when it expires, so the
|
|
6179
|
+
* strand is re-attempted on the same ladder a watcher-driven failure gets
|
|
6180
|
+
* (`min(strandWatchInterval, 5 s) * 2^(failures-1)`, capped at 5 minutes). Because the
|
|
5758
6181
|
* config stays registered, the retry takes `handleStrandAdded`'s auto-launch branch:
|
|
5759
6182
|
* what the app sees is `strand:error` per failed retry and `strand:started` when one
|
|
5760
6183
|
* succeeds, never a second `strand:discovered`. {@link detachStrand} (reached via
|
|
@@ -5797,6 +6220,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
5797
6220
|
// A claim also overrides any earlier `stopStrand` of this id: the watcher must track
|
|
5798
6221
|
// the strand again, or a party-wide removal would never stop it here.
|
|
5799
6222
|
this.strandWatcher?.unsuppressStrand(strandRow.Id);
|
|
6223
|
+
this.strandWatcher?.sAppIdLearned(strandRow.Id);
|
|
5800
6224
|
log('Registered sAppConfig for strand %s (sApp: %s, founder: %s)',
|
|
5801
6225
|
strandRow.Id, sAppConfig.id, founder ?? 'derived');
|
|
5802
6226
|
|
|
@@ -5822,7 +6246,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
5822
6246
|
// Hand the strand back to the watcher's retry ladder. Harmless on the founder path
|
|
5823
6247
|
// (`foundStrand` → `publishStrand` + this call), where the watcher may never have
|
|
5824
6248
|
// offered the strand: the `knownStrands` delete is a no-op and the recorded backoff
|
|
5825
|
-
// only delays the watcher's own first attempt by one
|
|
6249
|
+
// only delays the watcher's own first attempt by one ladder step, which is what
|
|
5826
6250
|
// should happen right after an attempt that just failed.
|
|
5827
6251
|
//
|
|
5828
6252
|
// NOTE: the retry relaunches from the CONTROL row (or, for a strand joined from
|
|
@@ -5906,7 +6330,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
5906
6330
|
* (the key exists nowhere else). No app-data wipe is needed for either.
|
|
5907
6331
|
*
|
|
5908
6332
|
* The insert is signed through this node's owner signer ({@link getOwnerSigner}), in one
|
|
5909
|
-
* approval with a closed strand's party
|
|
6333
|
+
* approval and one control transaction with a closed strand's party key
|
|
6334
|
+
* ({@link ControlDatabase.insertFoundingRows}). The signer's key is persisted on the row
|
|
5910
6335
|
* as `FounderOwnerKey` (the schema pins the column to the verified signer), and this node's
|
|
5911
6336
|
* peer id as `FounderPeerId` (bound into the signature), which is what later lets any
|
|
5912
6337
|
* launch derive "this machine is the founder" from the row alone.
|
|
@@ -5917,7 +6342,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
5917
6342
|
*
|
|
5918
6343
|
* A CLOSED strand's publish also seats this party's own membership identity key —
|
|
5919
6344
|
* the `StrandPartyKey` row {@link ensureStrandPartyKey} documents — which is what the
|
|
5920
|
-
* founder bootstrap derives `Member.Key`/`Manager.MemberKey` from
|
|
6345
|
+
* founder bootstrap derives `Member.Key`/`Manager.MemberKey` from; a key the party already
|
|
6346
|
+
* holds for the id is kept. The
|
|
5921
6347
|
* `memberPrivateKey` ARGUMENT stays the strand-wide read secret formation hands to
|
|
5922
6348
|
* joiners; it derives nobody's identity.
|
|
5923
6349
|
*
|
|
@@ -5937,23 +6363,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
5937
6363
|
*/
|
|
5938
6364
|
async publishStrand(strandId: string, type: 'o' | 'c' = 'o', memberPrivateKey?: string, sign?: OwnerSign): Promise<StrandRow> {
|
|
5939
6365
|
const signer = this.requireOwnerSigner(`publish strand ${strandId}`);
|
|
5940
|
-
|
|
5941
|
-
|
|
5942
|
-
const trimmed = requireNonBlank(strandId, 'strand id');
|
|
5943
|
-
// Refuse an unusable id HERE, not only at launch. `addStrand` already asserts, but it
|
|
5944
|
-
// runs after this row is written, and this row replicates: an id no node can turn into
|
|
5945
|
-
// a storage scope key would otherwise reach the whole party's control database and be
|
|
5946
|
-
// declined once per member. Asserted after the trim so the id checked is the id stored.
|
|
5947
|
-
assertStrandScopeKey(trimmed);
|
|
5948
|
-
// FounderPeerId records THIS machine as the founder, which is what later lets a relaunch
|
|
5949
|
-
// derive founder-ness; ControlDatabase.insertStrand writes the same id (its own node's).
|
|
5950
|
-
const desired: StrandRow = {
|
|
5951
|
-
Id: trimmed,
|
|
5952
|
-
Type: type,
|
|
5953
|
-
MemberPrivateKey: memberPrivateKey ?? null,
|
|
5954
|
-
FounderOwnerKey: signer.publicKey,
|
|
5955
|
-
FounderPeerId: this.controlNode!.peerId.toString(),
|
|
5956
|
-
};
|
|
6366
|
+
const desired = this.founderStrandRow(strandId, type, memberPrivateKey, signer);
|
|
6367
|
+
const trimmed = desired.Id;
|
|
5957
6368
|
|
|
5958
6369
|
// Read first rather than leaning on the insert's collision for BOTH readings. The
|
|
5959
6370
|
// ordinary resume (an app killed after its own publish committed) then needs no error
|
|
@@ -5974,53 +6385,158 @@ export class CadreNode implements SAppIdLookup {
|
|
|
5974
6385
|
log('publishStrand(%s): already published (type %s) with identical content — no-op', trimmed, type);
|
|
5975
6386
|
// A closed strand THIS machine published must hold a party key (its own membership
|
|
5976
6387
|
// identity — see ensureStrandPartyKey). Insert-if-absent, so the ordinary repeat
|
|
5977
|
-
// is a read-only no-op and a
|
|
5978
|
-
//
|
|
6388
|
+
// is a read-only no-op and a closed row stored without its key heals here. A row
|
|
6389
|
+
// another machine published is that machine's to mint for.
|
|
5979
6390
|
if (type === 'c' && this.isSelfFoundedRow(matched)) {
|
|
5980
6391
|
await this.ensureStrandPartyKey(trimmed, undefined, sign);
|
|
5981
6392
|
}
|
|
5982
6393
|
return matched;
|
|
5983
6394
|
}
|
|
5984
6395
|
|
|
5985
|
-
return await
|
|
5986
|
-
(approved) => this.insertStrandRow(desired, signer, approved));
|
|
6396
|
+
return (await this.writeFoundingRows(desired, undefined, signer, sign)).strandRow;
|
|
5987
6397
|
}
|
|
5988
6398
|
|
|
5989
6399
|
/**
|
|
5990
|
-
* The
|
|
5991
|
-
*
|
|
5992
|
-
*
|
|
6400
|
+
* The `Strand` row this machine publishes for `strandId`, as {@link publishStrand} and
|
|
6401
|
+
* {@link foundStrand} write it: the id trimmed and checked, this node's owner key and peer id
|
|
6402
|
+
* as its founder.
|
|
5993
6403
|
*/
|
|
5994
|
-
private
|
|
5995
|
-
|
|
6404
|
+
private founderStrandRow(strandId: string, type: 'o' | 'c', memberPrivateKey: string | undefined, signer: OwnerSigner): StrandRow {
|
|
6405
|
+
// Trim/reject here so the id that lands matches the one unpublishStrand looks up: it
|
|
6406
|
+
// trims too, and an untrimmed row would be unreachable by the same string.
|
|
6407
|
+
const trimmed = requireNonBlank(strandId, 'strand id');
|
|
6408
|
+
// Refuse an unusable id HERE, not only at launch. `addStrand` already asserts, but it
|
|
6409
|
+
// runs after this row is written, and this row replicates: an id no node can turn into
|
|
6410
|
+
// a storage scope key would otherwise reach the whole party's control database and be
|
|
6411
|
+
// declined once per member. Asserted after the trim so the id checked is the id stored.
|
|
6412
|
+
assertStrandScopeKey(trimmed);
|
|
6413
|
+
// FounderPeerId records THIS machine as the founder, which is what later lets a relaunch
|
|
6414
|
+
// derive founder-ness; ControlDatabase.insertFoundingRows writes the same id (its own node's).
|
|
6415
|
+
return {
|
|
6416
|
+
Id: trimmed,
|
|
6417
|
+
Type: type,
|
|
6418
|
+
MemberPrivateKey: memberPrivateKey ?? null,
|
|
6419
|
+
FounderOwnerKey: signer.publicKey,
|
|
6420
|
+
FounderPeerId: this.controlNode!.peerId.toString(),
|
|
6421
|
+
};
|
|
6422
|
+
}
|
|
6423
|
+
|
|
6424
|
+
/**
|
|
6425
|
+
* The write half of {@link publishStrand} and {@link foundStrand}: under one owner approval,
|
|
6426
|
+
* insert `desired`, a closed strand's party key and the requested invitation in one control
|
|
6427
|
+
* transaction ({@link ControlDatabase.insertFoundingRows}), adopting a matching row a
|
|
6428
|
+
* concurrent founder landed first. Reads nothing first: the caller has just read the
|
|
6429
|
+
* `Strand` row, and a stored key or row is resolved from the insert's own rejection.
|
|
6430
|
+
*/
|
|
6431
|
+
private async writeFoundingRows(
|
|
6432
|
+
desired: StrandRow,
|
|
6433
|
+
invite: FoundingInvite | undefined,
|
|
6434
|
+
signer: OwnerSigner,
|
|
6435
|
+
sign?: OwnerSign
|
|
6436
|
+
): Promise<FoundingWrite> {
|
|
6437
|
+
const action = invite
|
|
6438
|
+
? `Publish the strand ${desired.Id} to the cadre, with an invitation to the app ${invite.sAppId}`
|
|
6439
|
+
: `Publish the strand ${desired.Id} to the cadre`;
|
|
6440
|
+
return await withinOwnerApproval(signer, { partyId: this.partyId, action }, sign,
|
|
6441
|
+
(approved) => this.insertOrAdoptFounding(desired, invite, signer, approved));
|
|
6442
|
+
}
|
|
6443
|
+
|
|
6444
|
+
/** {@link writeFoundingRows} inside its approval. */
|
|
6445
|
+
private async insertOrAdoptFounding(
|
|
6446
|
+
desired: StrandRow,
|
|
6447
|
+
invite: FoundingInvite | undefined,
|
|
6448
|
+
signer: OwnerSigner,
|
|
6449
|
+
sign: OwnerSign
|
|
6450
|
+
): Promise<FoundingWrite> {
|
|
6451
|
+
let partyKey: string | undefined;
|
|
5996
6452
|
try {
|
|
5997
|
-
await this.
|
|
6453
|
+
partyKey = await this.seatFoundingRows(desired, invite, signer, sign);
|
|
5998
6454
|
} catch (error) {
|
|
5999
6455
|
// Only the uniqueness collision is a candidate for the idempotent reading; every
|
|
6000
6456
|
// other rejection (unauthorized signer, retired stamp) must keep surfacing.
|
|
6001
6457
|
if (!isStrandIdConflict(error, 'Strand')) {
|
|
6002
6458
|
throw error;
|
|
6003
6459
|
}
|
|
6004
|
-
const
|
|
6005
|
-
|
|
6006
|
-
|
|
6007
|
-
|
|
6008
|
-
|
|
6460
|
+
const strandRow = await this.adoptConcurrentFounding(desired, error);
|
|
6461
|
+
// The rejection rolled the invitation back with the row; a caller asking for one gets one.
|
|
6462
|
+
if (invite) {
|
|
6463
|
+
await this.publishFoundingInviteIfAbsent(strandRow.Id, invite, sign);
|
|
6464
|
+
}
|
|
6465
|
+
return { strandRow };
|
|
6466
|
+
}
|
|
6467
|
+
log('Published strand %s (type %s) to control DB under owner %s', desired.Id, desired.Type, signer.publicKey);
|
|
6468
|
+
if (invite) {
|
|
6469
|
+
this.registerPublishedInvitation(invite.token, invite.expiresAtMs);
|
|
6470
|
+
}
|
|
6471
|
+
return { strandRow: desired, partyKey };
|
|
6472
|
+
}
|
|
6473
|
+
|
|
6474
|
+
/**
|
|
6475
|
+
* Insert `desired`'s founding rows, minting a closed strand's party key — this party's own
|
|
6476
|
+
* membership identity, which the founder bootstrap derives `Member`/`Manager` from, distinct
|
|
6477
|
+
* from the row's shared `MemberPrivateKey` read secret. A key already stored for the id with
|
|
6478
|
+
* no strand row (one {@link ensureStrandPartyKey} seated, which it would adopt too) refuses
|
|
6479
|
+
* that transaction whole; the rows are then written again without a key.
|
|
6480
|
+
*
|
|
6481
|
+
* @returns The party key this seated; undefined when it seated none, so the launch reads the stored one.
|
|
6482
|
+
*/
|
|
6483
|
+
private async seatFoundingRows(
|
|
6484
|
+
desired: StrandRow,
|
|
6485
|
+
invite: FoundingInvite | undefined,
|
|
6486
|
+
signer: OwnerSigner,
|
|
6487
|
+
sign: OwnerSign
|
|
6488
|
+
): Promise<string | undefined> {
|
|
6489
|
+
const controlDatabase = this.controlDatabase!;
|
|
6490
|
+
const strand = { id: desired.Id, type: desired.Type, memberPrivateKey: desired.MemberPrivateKey ?? undefined };
|
|
6491
|
+
if (desired.Type !== 'c') {
|
|
6492
|
+
await controlDatabase.insertFoundingRows({ strand, formationInvite: invite }, signer.publicKey, sign);
|
|
6493
|
+
return undefined;
|
|
6494
|
+
}
|
|
6495
|
+
const partyKey = await generateStrandMemberKey();
|
|
6496
|
+
try {
|
|
6497
|
+
await controlDatabase.insertFoundingRows({ strand, partyKey, formationInvite: invite }, signer.publicKey, sign);
|
|
6498
|
+
return partyKey;
|
|
6499
|
+
} catch (error) {
|
|
6500
|
+
if (!isStrandIdConflict(error, 'StrandPartyKey')) {
|
|
6009
6501
|
throw error;
|
|
6010
6502
|
}
|
|
6011
|
-
log('publishStrand(%s):
|
|
6012
|
-
|
|
6503
|
+
log('publishStrand(%s): this party already holds a key for the id; publishing the row and adopting that key', desired.Id);
|
|
6504
|
+
await controlDatabase.insertFoundingRows({ strand, formationInvite: invite }, signer.publicKey, sign);
|
|
6505
|
+
return undefined;
|
|
6013
6506
|
}
|
|
6014
|
-
|
|
6015
|
-
|
|
6016
|
-
|
|
6017
|
-
|
|
6018
|
-
|
|
6019
|
-
|
|
6020
|
-
|
|
6021
|
-
|
|
6507
|
+
}
|
|
6508
|
+
|
|
6509
|
+
/**
|
|
6510
|
+
* Resolve a founding refused on `Strand.Id`: re-read the row that landed and adopt it when it
|
|
6511
|
+
* matches `desired` — a concurrent founder, such as two machines of one party on one id —
|
|
6512
|
+
* or throw naming the columns that differ.
|
|
6513
|
+
*/
|
|
6514
|
+
private async adoptConcurrentFounding(desired: StrandRow, error: unknown): Promise<StrandRow> {
|
|
6515
|
+
const landed = await this.controlDatabase!.queryStrand(desired.Id);
|
|
6516
|
+
if (!landed) {
|
|
6517
|
+
// Collided, yet no row is readable: not the race this branch handles (a row
|
|
6518
|
+
// deleted between the rejection and the re-read, or a stale local view). Surface
|
|
6519
|
+
// the original rather than guessing.
|
|
6520
|
+
throw error;
|
|
6022
6521
|
}
|
|
6023
|
-
|
|
6522
|
+
log('publishStrand(%s): lost a concurrent founding race; re-read the landed row', desired.Id);
|
|
6523
|
+
return requireMatchingStrandRow(landed, desired, 'landed concurrently from another founder');
|
|
6524
|
+
}
|
|
6525
|
+
|
|
6526
|
+
/**
|
|
6527
|
+
* Publish a founding's invitation on its own, bound to `strandId`, for a founding that adopted
|
|
6528
|
+
* a stored row rather than writing its own — unless the invitation is already stored, bound to
|
|
6529
|
+
* this strand, as a repeat of a founding whose transaction committed leaves it.
|
|
6530
|
+
*
|
|
6531
|
+
* @param sign - The open approval's `sign`; omitted, the publish asks for its own.
|
|
6532
|
+
*/
|
|
6533
|
+
private async publishFoundingInviteIfAbsent(strandId: string, invite: FoundingInvite, sign?: OwnerSign): Promise<void> {
|
|
6534
|
+
if ((await this.controlDatabase!.queryFormationInvite(invite.token))?.strandId === strandId) {
|
|
6535
|
+
log('foundStrand(%s): the invitation is already published', strandId);
|
|
6536
|
+
return;
|
|
6537
|
+
}
|
|
6538
|
+
const { token, sAppId, ...options } = invite;
|
|
6539
|
+
await this.publishFormationInvite(token, sAppId, { ...options, strandId }, sign);
|
|
6024
6540
|
}
|
|
6025
6541
|
|
|
6026
6542
|
/**
|
|
@@ -6028,6 +6544,11 @@ export class CadreNode implements SAppIdLookup {
|
|
|
6028
6544
|
* founder — in one resumable call. The single entry point for creating a strand; a
|
|
6029
6545
|
* caller joining one someone else founded uses {@link addStrand} alone.
|
|
6030
6546
|
*
|
|
6547
|
+
* The publish is one control transaction under one owner approval: the `Strand` row, a
|
|
6548
|
+
* closed strand's `StrandPartyKey` row, and the invitation in
|
|
6549
|
+
* {@link FoundStrandConfig.formationInvite} when one is given. The start then uses the party
|
|
6550
|
+
* key the publish seated rather than reading it back.
|
|
6551
|
+
*
|
|
6031
6552
|
* Safe to re-run from any point of interruption, which hand-rolling
|
|
6032
6553
|
* {@link publishStrand} + {@link addStrand} is not. Either half can already have
|
|
6033
6554
|
* happened:
|
|
@@ -6109,18 +6630,19 @@ export class CadreNode implements SAppIdLookup {
|
|
|
6109
6630
|
* derive whether this machine founds it. Every check that can refuse the founding is here,
|
|
6110
6631
|
* so both forms of `foundStrand` reject on it before returning.
|
|
6111
6632
|
*/
|
|
6112
|
-
private async publishFoundingRow(config: FoundStrandConfig): Promise<
|
|
6113
|
-
const { strandId, type = 'o', memberPrivateKey } = config;
|
|
6633
|
+
private async publishFoundingRow(config: FoundStrandConfig): Promise<FoundingPublish> {
|
|
6634
|
+
const { strandId, type = 'o', memberPrivateKey, formationInvite, sAppConfig } = config;
|
|
6114
6635
|
if (!this._running || !this.controlDatabase) {
|
|
6115
6636
|
throw new Error(`CadreNode must be started before attempting to found strand ${strandId}`);
|
|
6116
6637
|
}
|
|
6117
6638
|
const controlDatabase = this.controlDatabase;
|
|
6118
6639
|
const trimmed = requireNonBlank(strandId, 'strand id');
|
|
6640
|
+
const invite: FoundingInvite | undefined = formationInvite && { ...formationInvite, sAppId: sAppConfig.id };
|
|
6119
6641
|
const timed = <T>(step: string, op: () => Promise<T>) => timedStep('foundStrand', trimmed, step, op);
|
|
6120
6642
|
const published = await timed('queryStrand', () => controlDatabase.queryStrand(trimmed));
|
|
6121
|
-
const strandRow = published
|
|
6122
|
-
? this.
|
|
6123
|
-
: await timed('publishStrand', () => this.
|
|
6643
|
+
const { strandRow, partyKey } = published
|
|
6644
|
+
? await this.resumeFounding(published, type, memberPrivateKey, invite)
|
|
6645
|
+
: await timed('publishStrand', () => this.publishFoundingStrand(trimmed, type, memberPrivateKey, invite));
|
|
6124
6646
|
// Derived, not hardcoded `true`: adopting a row another machine published means that
|
|
6125
6647
|
// machine runs the bootstrap — this one must attach or it would write a second Header.
|
|
6126
6648
|
const founded = this.isSelfFoundedRow(strandRow);
|
|
@@ -6129,20 +6651,52 @@ export class CadreNode implements SAppIdLookup {
|
|
|
6129
6651
|
'(FounderPeerId is not this node\'s peer id) — attaching as a joiner; ' +
|
|
6130
6652
|
'the founder bootstrap runs on that machine', trimmed);
|
|
6131
6653
|
}
|
|
6132
|
-
return { strandRow, founded };
|
|
6654
|
+
return { strandRow, founded, partyKey };
|
|
6655
|
+
}
|
|
6656
|
+
|
|
6657
|
+
/** {@link publishFoundingRow} when no row is stored: write the founding rows under this node's owner. */
|
|
6658
|
+
private publishFoundingStrand(
|
|
6659
|
+
strandId: string,
|
|
6660
|
+
type: 'o' | 'c',
|
|
6661
|
+
memberPrivateKey: string | undefined,
|
|
6662
|
+
invite: FoundingInvite | undefined
|
|
6663
|
+
): Promise<FoundingWrite> {
|
|
6664
|
+
const signer = this.requireOwnerSigner(`publish strand ${strandId}`);
|
|
6665
|
+
return this.writeFoundingRows(this.founderStrandRow(strandId, type, memberPrivateKey, signer), invite, signer);
|
|
6666
|
+
}
|
|
6667
|
+
|
|
6668
|
+
/**
|
|
6669
|
+
* {@link publishFoundingRow} over a stored row: adopt it ({@link adoptPublishedStrand}), then
|
|
6670
|
+
* publish the requested invitation on its own, under its own approval.
|
|
6671
|
+
*/
|
|
6672
|
+
private async resumeFounding(
|
|
6673
|
+
published: StrandRow,
|
|
6674
|
+
type: 'o' | 'c',
|
|
6675
|
+
memberPrivateKey: string | undefined,
|
|
6676
|
+
invite: FoundingInvite | undefined
|
|
6677
|
+
): Promise<FoundingWrite> {
|
|
6678
|
+
const strandRow = this.adoptPublishedStrand(published, type, memberPrivateKey);
|
|
6679
|
+
if (invite) {
|
|
6680
|
+
await timedStep('foundStrand', strandRow.Id, 'publishFormationInvite',
|
|
6681
|
+
() => this.publishFoundingInviteIfAbsent(strandRow.Id, invite));
|
|
6682
|
+
}
|
|
6683
|
+
return { strandRow };
|
|
6133
6684
|
}
|
|
6134
6685
|
|
|
6135
6686
|
/**
|
|
6136
6687
|
* The start half of {@link foundStrand}. Its timing lines (`addStrand: start`, then the
|
|
6137
6688
|
* duration or `failed after`) are logged when the start settles, also in the background
|
|
6138
6689
|
* form, so a start that hangs still shows as a start line with no end line.
|
|
6690
|
+
*
|
|
6691
|
+
* A party key the publish half just seated is handed to the launch, which would otherwise
|
|
6692
|
+
* read back the row the same founding wrote.
|
|
6139
6693
|
*/
|
|
6140
6694
|
private attachFoundingRow(
|
|
6141
|
-
{ strandRow, founded }:
|
|
6695
|
+
{ strandRow, founded, partyKey }: FoundingPublish,
|
|
6142
6696
|
{ sAppConfig, awaitFirstSync }: FoundStrandConfig
|
|
6143
6697
|
): Promise<StrandInstance> {
|
|
6144
6698
|
return timedStep('foundStrand', strandRow.Id, 'addStrand',
|
|
6145
|
-
() => this.addStrand({ strandRow, sAppConfig, founder: founded, awaitFirstSync }));
|
|
6699
|
+
() => this.addStrand({ strandRow, sAppConfig, founder: founded, awaitFirstSync, partyMemberPrivateKey: partyKey }));
|
|
6146
6700
|
}
|
|
6147
6701
|
|
|
6148
6702
|
/**
|
|
@@ -6178,9 +6732,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
6178
6732
|
* inverse of {@link publishStrand}.
|
|
6179
6733
|
*
|
|
6180
6734
|
* Party-wide, unlike {@link stopStrand}, which only stops the strand on THIS node: every
|
|
6181
|
-
* cadre node watching the table
|
|
6182
|
-
*
|
|
6183
|
-
*
|
|
6735
|
+
* cadre node watching the table hears the row's removal and stops its own instance (a
|
|
6736
|
+
* machine outside the `Strand` block's cohort, on a cadre larger than 16, only at its
|
|
6737
|
+
* watcher's backstop read, default 5 minutes). This node converges immediately — the
|
|
6738
|
+
* method forces a watcher run and stops any still-running local instance before resolving. Removing the row
|
|
6184
6739
|
* removes OUR party's participation only: other parties in the strand keep their own
|
|
6185
6740
|
* rows and the strand network carries on, so no cross-party sign-off is involved — this
|
|
6186
6741
|
* is an owner-signed control-plane write like every other, not "destroy the network".
|
|
@@ -6267,16 +6822,16 @@ export class CadreNode implements SAppIdLookup {
|
|
|
6267
6822
|
// deadline rather than dropping it (a binding cleared late is still worth more than
|
|
6268
6823
|
// one never cleared).
|
|
6269
6824
|
await this.strandManager.clearOwnMemberPeerBinding(trimmed);
|
|
6270
|
-
// Converge locally
|
|
6271
|
-
// onStrandRemoved for a strand it tracked; the explicit teardown below
|
|
6272
|
-
// whose strandFilter never admitted this strand (the watcher never knew
|
|
6273
|
-
// never fire). StrandInstanceManager.stopStrand no-ops on an unknown id, so the two
|
|
6825
|
+
// Converge locally before resolving rather than when the write wakes the watcher. The
|
|
6826
|
+
// watcher fires onStrandRemoved for a strand it tracked; the explicit teardown below
|
|
6827
|
+
// covers a node whose strandFilter never admitted this strand (the watcher never knew
|
|
6828
|
+
// it, so it will never fire). StrandInstanceManager.stopStrand no-ops on an unknown id, so the two
|
|
6274
6829
|
// paths cannot double-stop into an error.
|
|
6275
6830
|
//
|
|
6276
6831
|
// detachStrand, NOT stopStrand: this is a party-wide removal, not a local abandonment
|
|
6277
6832
|
// of a strand that still exists, so it must not SUPPRESS the id in the watcher. The
|
|
6278
6833
|
// row is already gone — there is nothing to suppress — and an id re-published before
|
|
6279
|
-
// the next
|
|
6834
|
+
// the next run's suppression cleanup would then never be offered again this session.
|
|
6280
6835
|
// (Same reasoning as handleStrandRemoved, which also avoids stopStrand's `_running`
|
|
6281
6836
|
// guard; here the guard is already satisfied by requireOwnerSigner above.)
|
|
6282
6837
|
await this.strandWatcher?.forcePoll();
|
|
@@ -6300,9 +6855,11 @@ export class CadreNode implements SAppIdLookup {
|
|
|
6300
6855
|
* as `MemberPrivateKey`; docs/strands.md → "Closed-Strand Member Key Handling") and is
|
|
6301
6856
|
* never put on the formation wire.
|
|
6302
6857
|
*
|
|
6303
|
-
*
|
|
6304
|
-
*
|
|
6305
|
-
*
|
|
6858
|
+
* A founding seats the key in the same transaction as the strand row, not through here.
|
|
6859
|
+
* Called internally by {@link publishStrand}'s repeat of a published closed strand and by a
|
|
6860
|
+
* founder launch that finds no row, which heal a closed strand row stored without its key
|
|
6861
|
+
* (one seated by `ControlDatabase.insertStrand` directly, or whose key was removed); public
|
|
6862
|
+
* so a test harness — or a joiner flow that persists a formation-issued identity — can seat
|
|
6306
6863
|
* a specific key deliberately.
|
|
6307
6864
|
*
|
|
6308
6865
|
* @param strandId - The strand the key is this party's identity for.
|
|
@@ -6353,8 +6910,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
6353
6910
|
* Resolve the party membership key a closed strand's launch threads into the founder
|
|
6354
6911
|
* bootstrap: the explicit attach-time key when given, else the party's persisted
|
|
6355
6912
|
* `StrandPartyKey` row, else — on the one machine the row names as founder (`FounderPeerId`)
|
|
6356
|
-
* — a freshly minted-and-persisted key, which seats the identity for a
|
|
6357
|
-
*
|
|
6913
|
+
* — a freshly minted-and-persisted key, which seats the identity for a closed row stored
|
|
6914
|
+
* without one (see {@link ensureStrandPartyKey}). Everyone else resolves undefined: non-founding
|
|
6358
6915
|
* machines never mint (no mint race between a party's machines), and only a founder
|
|
6359
6916
|
* bootstrap needs the key at all.
|
|
6360
6917
|
*
|
|
@@ -6374,6 +6931,11 @@ export class CadreNode implements SAppIdLookup {
|
|
|
6374
6931
|
if (explicitKey !== undefined) {
|
|
6375
6932
|
return explicitKey;
|
|
6376
6933
|
}
|
|
6934
|
+
const formed = this.formedPartyKeys.get(strand.Id);
|
|
6935
|
+
if (formed !== undefined) {
|
|
6936
|
+
this.formedPartyKeys.delete(strand.Id);
|
|
6937
|
+
return formed;
|
|
6938
|
+
}
|
|
6377
6939
|
if (!this.controlDatabase) {
|
|
6378
6940
|
return undefined;
|
|
6379
6941
|
}
|
|
@@ -6398,8 +6960,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
6398
6960
|
* Counterpart to {@link createOpenInvitation}, which only mints the
|
|
6399
6961
|
* out-of-band {@link OpenInvitation} envelope: persisting the matching
|
|
6400
6962
|
* `FormationInvite` row is what makes the token *redeemable* — the host's
|
|
6401
|
-
* {@link ControlFormationUsageRecorder}
|
|
6402
|
-
*
|
|
6963
|
+
* {@link ControlFormationUsageRecorder} reads it (`readToken`) to check a
|
|
6964
|
+
* redemption. A host minting a closed-strand invite does both (mint +
|
|
6403
6965
|
* publish), exactly as the integration harness's `createInvitation` does.
|
|
6404
6966
|
*
|
|
6405
6967
|
* Signs through the same owner signer as {@link publishStrand}, whose key must be an
|
|
@@ -6422,16 +6984,21 @@ export class CadreNode implements SAppIdLookup {
|
|
|
6422
6984
|
const signer = this.requireOwnerSigner(`publish formation invite ${token}`);
|
|
6423
6985
|
await withinOwnerApproval(signer, { partyId: this.partyId, action: `Publish an invitation to the app ${sAppId}` }, sign,
|
|
6424
6986
|
(approved) => this.controlDatabase!.insertFormationInvite(token, sAppId, signer.publicKey, approved, options));
|
|
6425
|
-
|
|
6426
|
-
// it locally opens the connection gate's formation exemption immediately
|
|
6427
|
-
// rather than waiting for the durable row to become readable.
|
|
6428
|
-
this.strandSolicitationService?.registerMintedInvitation(
|
|
6429
|
-
token,
|
|
6430
|
-
options.expiresAtMs ?? Number.POSITIVE_INFINITY
|
|
6431
|
-
);
|
|
6987
|
+
this.registerPublishedInvitation(token, options.expiresAtMs);
|
|
6432
6988
|
log('Published formation invite %s (sApp %s) under owner %s', token, sAppId, signer.publicKey);
|
|
6433
6989
|
}
|
|
6434
6990
|
|
|
6991
|
+
/**
|
|
6992
|
+
* Register an invitation this node just published with the strand solicitation service. A
|
|
6993
|
+
* host may publish one whose token was minted elsewhere; registering it locally opens the
|
|
6994
|
+
* connection gate's formation exemption immediately rather than waiting for the durable row
|
|
6995
|
+
* to become readable. A token {@link createOpenInvitation} minted here is already registered,
|
|
6996
|
+
* and this records the same expiry again.
|
|
6997
|
+
*/
|
|
6998
|
+
private registerPublishedInvitation(token: string, expiresAtMs: number | undefined): void {
|
|
6999
|
+
this.strandSolicitationService?.registerMintedInvitation(token, expiresAtMs ?? Number.POSITIVE_INFINITY);
|
|
7000
|
+
}
|
|
7001
|
+
|
|
6435
7002
|
/**
|
|
6436
7003
|
* Enroll an approver public key allowed to sign off on `ValidationUrl` redemptions.
|
|
6437
7004
|
*
|
|
@@ -6547,7 +7114,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
6547
7114
|
* fan-out, so a rediscovery costs nothing beyond the map lookup plus the (pure)
|
|
6548
7115
|
* founder derivation. NOT a silent no-op any more when the launch resolves as a
|
|
6549
7116
|
* FOUNDER: the tracked instance may have been launched first as a joiner (an app
|
|
6550
|
-
* attach, or this node's own watcher
|
|
7117
|
+
* attach, or this node's own watcher run winning the `resolveCohortSeed` window
|
|
6551
7118
|
* below), which used to drop the founder request and leave the strand headerless —
|
|
6552
7119
|
* now the tracked instance is founded in place
|
|
6553
7120
|
* ({@link StrandInstanceManager.foundExistingStrand}), waking it first if quiesced
|
|
@@ -6559,9 +7126,9 @@ export class CadreNode implements SAppIdLookup {
|
|
|
6559
7126
|
*
|
|
6560
7127
|
* An absent `sAppConfig` launches a storage replica ({@link CadreNodeConfig.hostUnclaimedStrands}),
|
|
6561
7128
|
* which is ALWAYS a joiner whatever the row says: the founder bootstrap writes the sApp
|
|
6562
|
-
* into `Strand.Header`, and a replica has none. So a self-founded row whose watcher
|
|
7129
|
+
* into `Strand.Header`, and a replica has none. So a self-founded row whose watcher run
|
|
6563
7130
|
* wins the race against its app's `addStrand` after a restart comes up as a replica — and
|
|
6564
|
-
* so does a `foundStrand` whose publish a watcher
|
|
7131
|
+
* so does a `foundStrand` whose publish a watcher run saw before its attach. A claim
|
|
6565
7132
|
* that finds a tracked replica upgrades it in place, over the same node and store
|
|
6566
7133
|
* ({@link StrandInstanceManager.attachSApp}), BEFORE any founding: the founder bootstrap
|
|
6567
7134
|
* needs the sApp the attach supplies.
|
|
@@ -6604,8 +7171,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
6604
7171
|
|
|
6605
7172
|
// A closed strand's launch carries the party's OWN membership identity key: the
|
|
6606
7173
|
// explicit attach-time key, else the control-layer StrandPartyKey row — minted here
|
|
6607
|
-
// when this machine is the row-derived founder and no row exists yet (a
|
|
6608
|
-
//
|
|
7174
|
+
// when this machine is the row-derived founder and no row exists yet (a closed row
|
|
7175
|
+
// stored without its key). A joiner with no persisted key threads undefined;
|
|
6609
7176
|
// only a FOUNDER bootstrap needs the key, and that path throws loudly without one
|
|
6610
7177
|
// (StrandDatabase).
|
|
6611
7178
|
const partyMemberPrivateKey = strand.Type === 'c'
|
|
@@ -6948,7 +7515,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
6948
7515
|
*
|
|
6949
7516
|
* Timestamps are recorded OPTIMISTICALLY, whatever `collectStrandAddrs` reports
|
|
6950
7517
|
* per peer: recording only on success would re-announce to an unreachable relay
|
|
6951
|
-
* on every reconcile
|
|
7518
|
+
* on every reconcile pass, one dial timeout each. A failed INITIAL announce costs
|
|
6952
7519
|
* the strand supervisor its first attempt only (the relay denies the
|
|
6953
7520
|
* reservation; every re-drive re-announces first through
|
|
6954
7521
|
* {@link announceDelegateToRelay}); a failed REFRESH retries within
|
|
@@ -7092,9 +7659,11 @@ export class CadreNode implements SAppIdLookup {
|
|
|
7092
7659
|
*
|
|
7093
7660
|
* Due times are per (sibling, strand) and set from each sibling's own outcome, so
|
|
7094
7661
|
* a sibling that connects late — a phone joining after the party's always-on
|
|
7095
|
-
* machines — is asked on the
|
|
7096
|
-
* (its view of the membership may not include us yet) is retried
|
|
7097
|
-
* {@link STRAND_PEER_ADDR_RETRY_MS} rather than
|
|
7662
|
+
* machines — is asked on the pass its connection starts, and one that could not
|
|
7663
|
+
* answer or refused (its view of the membership may not include us yet) is retried
|
|
7664
|
+
* within {@link STRAND_PEER_ADDR_RETRY_MS} rather than
|
|
7665
|
+
* {@link STRAND_PEER_ADDR_REFRESH_MS}; the pass that stamped the retry arms a wake
|
|
7666
|
+
* for it ({@link armReconcileWake}).
|
|
7098
7667
|
*
|
|
7099
7668
|
* Distinct from {@link refreshDelegateGrants}, deliberately: that pass covers
|
|
7100
7669
|
* RELAYS on a `DELEGATE_GRANT_TTL_MS / 2` throttle to keep circuit-relay
|
|
@@ -7144,15 +7713,17 @@ export class CadreNode implements SAppIdLookup {
|
|
|
7144
7713
|
*
|
|
7145
7714
|
* `connectedSiblingTargets`' membership read is unbounded, and with zero
|
|
7146
7715
|
* connections its answer is empty whatever the table holds, so decide it from the
|
|
7147
|
-
* connection list instead of paying for the read once a
|
|
7716
|
+
* connection list instead of paying for the read once a pass.
|
|
7148
7717
|
*
|
|
7149
|
-
* NOTE: otherwise the read runs on EVERY
|
|
7718
|
+
* NOTE: otherwise the read runs on EVERY pass with a running strand, not only when
|
|
7150
7719
|
* some (sibling, strand) is due: pruning a departed sibling needs the current
|
|
7151
7720
|
* target set, and whether anyone is due cannot be told without it (a connected
|
|
7152
|
-
* non-member never gets a due time to compare). One more `CadrePeer` read per
|
|
7153
|
-
*
|
|
7154
|
-
*
|
|
7155
|
-
*
|
|
7721
|
+
* non-member never gets a due time to compare). One more `CadrePeer` read per
|
|
7722
|
+
* pass, on top of the two `runReconcileControlCohort` already makes, all served
|
|
7723
|
+
* from the membership rows view: an in-memory scan while the view is current,
|
|
7724
|
+
* storage only after the table changed or the view's maximum age passed. Sharing
|
|
7725
|
+
* one row set across the pass would save no storage read and was declined (see
|
|
7726
|
+
* the NOTE there).
|
|
7156
7727
|
*
|
|
7157
7728
|
* NOTE: one strand-addr RPC per (running strand × due sibling) — each a tiny
|
|
7158
7729
|
* request/response on an already-open control connection. If a node ever runs
|
|
@@ -7243,8 +7814,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
7243
7814
|
*
|
|
7244
7815
|
* NOTE: every pass serialises the whole FRET table (`exportTable()`), verifies one
|
|
7245
7816
|
* signature per record and makes one `peerStore.merge` per peer, per running strand
|
|
7246
|
-
* per
|
|
7247
|
-
* nothing in libp2p's persistent peer store. If many strands or large tables make it
|
|
7817
|
+
* per pass — every change-driven pass and the 2-minute backstop. Fine at strand table
|
|
7818
|
+
* sizes, and an unchanged address set writes nothing in libp2p's persistent peer store. If many strands or large tables make it
|
|
7248
7819
|
* show up, read only the entries whose address stamp is near the one-hour mark.
|
|
7249
7820
|
*
|
|
7250
7821
|
* @returns the ids of the peers the table holds a usable record for.
|
|
@@ -7550,8 +8121,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
7550
8121
|
* The stop + emit are skipped when the strand manager holds no instance for `strandId` —
|
|
7551
8122
|
* e.g. a party owner that published a strand's row but never ran it locally, or an
|
|
7552
8123
|
* explicit {@link stopStrand} for an id this node never started. `hibernationManager`
|
|
7553
|
-
* untrack and the `sAppConfigs` / {@link strandLaunchRefusals} / {@link discoveredStrands}
|
|
7554
|
-
* deletes stay unconditional (all no-ops when there is nothing to remove), so a launch
|
|
8124
|
+
* untrack and the `sAppConfigs` / {@link strandLaunchRefusals} / {@link discoveredStrands} /
|
|
8125
|
+
* {@link formedPartyKeys} deletes stay unconditional (all no-ops when there is nothing to remove), so a launch
|
|
7555
8126
|
* that failed before an instance was ever tracked still gets its stray entries cleared.
|
|
7556
8127
|
*
|
|
7557
8128
|
* Dropping the {@link discoveredStrands} entry here covers both callers, and both
|
|
@@ -7568,6 +8139,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
7568
8139
|
this.sAppConfigs.delete(strandId);
|
|
7569
8140
|
this.strandLaunchRefusals.delete(strandId);
|
|
7570
8141
|
this.discoveredStrands.delete(strandId);
|
|
8142
|
+
this.formedPartyKeys.delete(strandId);
|
|
7571
8143
|
if (!this.strandManager.hasStrand(strandId)) {
|
|
7572
8144
|
log('detachStrand: strand %s not tracked locally — no-op (no stop, no strand:stopped)', strandId);
|
|
7573
8145
|
return;
|
|
@@ -7956,21 +8528,37 @@ export class CadreNode implements SAppIdLookup {
|
|
|
7956
8528
|
}
|
|
7957
8529
|
|
|
7958
8530
|
/**
|
|
7959
|
-
* Open control-network connections right now
|
|
7960
|
-
*
|
|
7961
|
-
*
|
|
7962
|
-
*
|
|
7963
|
-
* It approximates the precise signal (the block's cluster size), and a caller that
|
|
7964
|
-
* samples it AFTER a write samples a slightly wider window than `committedAlone`
|
|
7965
|
-
* does inside the write itself. It exists so an embedder can warn an owner before a
|
|
7966
|
-
* control write that this machine currently sees none of its siblings, and report
|
|
7967
|
-
* after one that the write may not have travelled — a warning shown unconditionally
|
|
7968
|
-
* is a warning people learn to ignore.
|
|
8531
|
+
* Open control-network connections right now, of every kind: siblings, relays, bootstrap
|
|
8532
|
+
* nodes and peers still identifying. A report of connectivity, not of replication reach: a
|
|
8533
|
+
* control write can commit local-only with connections open, and whether one would is
|
|
8534
|
+
* {@link hasReachableControlSibling}.
|
|
7969
8535
|
*/
|
|
7970
8536
|
getControlConnectionCount(): number {
|
|
7971
8537
|
return this.controlNode?.getConnections().length ?? 0;
|
|
7972
8538
|
}
|
|
7973
8539
|
|
|
8540
|
+
/**
|
|
8541
|
+
* Whether a control write made now reaches another machine: a connected peer has identified
|
|
8542
|
+
* as serving the control database and is one this node sends control writes to (an
|
|
8543
|
+
* authorized member, or a device whose admission it is writing). Optimystic counts a peer
|
|
8544
|
+
* into a control block's cohort on those same terms; with none, a write commits on this node
|
|
8545
|
+
* alone and is queued for re-issue once one is reachable (docs/architecture.md → "Writes made
|
|
8546
|
+
* while alone").
|
|
8547
|
+
*
|
|
8548
|
+
* It exists so an embedder can warn an owner before a control write that this machine
|
|
8549
|
+
* currently reaches none of its siblings, and report after one that the write may not have
|
|
8550
|
+
* travelled; a warning shown unconditionally is a warning people learn to ignore. A caller
|
|
8551
|
+
* that samples it AFTER a write samples a slightly wider window than the write did.
|
|
8552
|
+
*/
|
|
8553
|
+
hasReachableControlSibling(): boolean {
|
|
8554
|
+
for (const peerId of this.identifiedControlPeers) {
|
|
8555
|
+
if (this.isControlCohortMember(peerId)) {
|
|
8556
|
+
return true;
|
|
8557
|
+
}
|
|
8558
|
+
}
|
|
8559
|
+
return false;
|
|
8560
|
+
}
|
|
8561
|
+
|
|
7974
8562
|
/**
|
|
7975
8563
|
* Force a poll of the strand watcher (for testing)
|
|
7976
8564
|
*/
|
|
@@ -8052,7 +8640,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
8052
8640
|
}), this.controlNode, this.controlDatabase);
|
|
8053
8641
|
log('Owner signer wired (owner key %s)', signer.publicKey);
|
|
8054
8642
|
// Embedders wire the owner key after start(), so the runner's first pass usually found
|
|
8055
|
-
// this machine not yet an owner;
|
|
8643
|
+
// this machine not yet an owner; wiring a signer writes no table it watches, so without
|
|
8644
|
+
// this it would wait for its backstop.
|
|
8056
8645
|
this.pendingJoinRunner?.kick();
|
|
8057
8646
|
}
|
|
8058
8647
|
|
|
@@ -8290,6 +8879,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
8290
8879
|
if (usage === undefined || invite === undefined) {
|
|
8291
8880
|
return false;
|
|
8292
8881
|
}
|
|
8882
|
+
// NOTE: accepted tradeoff — three ed25519 verifies per invitation-admitted row on every
|
|
8883
|
+
// refresh, CPU only since the rows come from the membership views; a memo of verdicts would
|
|
8884
|
+
// also have to follow the trusted-owner anchor, which syncOwnerAnchor rewrites. Revisit if
|
|
8885
|
+
// the gate's admission deadline ever shows it.
|
|
8293
8886
|
return verifyInvitationAdmission(this.partyId, row, usage, invite, key => this.trustedOwnerStore?.has(key) ?? false);
|
|
8294
8887
|
}
|
|
8295
8888
|
|
|
@@ -8313,7 +8906,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
8313
8906
|
* builds its snapshot against that anchor, which is also what its previous snapshot was
|
|
8314
8907
|
* judged against. Caught here rather than failing the refresh, because the membership read
|
|
8315
8908
|
* tolerates an unreachable `Revocation` block on an isolated node
|
|
8316
|
-
* (`ControlDatabase.
|
|
8909
|
+
* (`ControlDatabase.readRevocationRowsOrNone`) and `queryRevocations` does not: failing the
|
|
8317
8910
|
* whole refresh on that would freeze such a node's gate snapshot.
|
|
8318
8911
|
* NOTE: on such a node the sync is skipped at every refresh until the block is held; if an
|
|
8319
8912
|
* isolated node ever has to derive owner additions while cut off, give `queryRevocations`
|
|
@@ -8322,9 +8915,12 @@ export class CadreNode implements SAppIdLookup {
|
|
|
8322
8915
|
* NOTE: cost per membership refresh is one `OwnerKey` read and one `Revocation` read (the
|
|
8323
8916
|
* tombstone list, whose `OwnerKey` stamps are handed to `queryOwnerKeyRows` so it does not
|
|
8324
8917
|
* read the retired set again, and which the hang-up of removed peers shares through
|
|
8325
|
-
* `loadTombstones`), plus the owner-grant chain when some owner row is
|
|
8326
|
-
*
|
|
8327
|
-
*
|
|
8918
|
+
* `loadTombstones`), plus the owner-grant chain's two reads when some owner row is
|
|
8919
|
+
* grant-seated. All are served from the control database's membership views, so they reach
|
|
8920
|
+
* storage only after their table changed or the view's maximum age passed
|
|
8921
|
+
* (`control-founding-consult-budget.spec.ts` pins the counts). On a party founded before
|
|
8922
|
+
* genesis created every control collection, whose `Revocation` table has never been
|
|
8923
|
+
* written, each `Revocation` load is a cohort consult until the first tombstone.
|
|
8328
8924
|
*
|
|
8329
8925
|
* @param retry - forwarded to every read; `false` on the refresh path, for the reason on
|
|
8330
8926
|
* {@link refreshAuthorizedControlPeers}.
|
|
@@ -8345,9 +8941,6 @@ export class CadreNode implements SAppIdLookup {
|
|
|
8345
8941
|
const tombstones = await loadTombstones();
|
|
8346
8942
|
const retired = new Set(tombstones.filter(tombstone => tombstone.tableName === 'OwnerKey').map(tombstone => tombstone.stampId));
|
|
8347
8943
|
const rows = await controlDatabase.queryOwnerKeyRows(retry, retired);
|
|
8348
|
-
// NOTE: the grant chain is two whole-table reads per membership refresh while any owner
|
|
8349
|
-
// row is grant-seated, over tables that are append-only; the memo suggested on
|
|
8350
|
-
// loadInvitationChain would cover it too.
|
|
8351
8944
|
const grantChain = rows.some(isConsentSeated) ? await this.loadOwnerGrantChain(controlDatabase, retry) : null;
|
|
8352
8945
|
const derivation = deriveOwnerAnchor({ partyId: this.partyId, anchor: before, rows, tombstones, grantChain });
|
|
8353
8946
|
if (derivation.refusedRemovals.length > 0) {
|
|
@@ -8413,15 +9006,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
8413
9006
|
/**
|
|
8414
9007
|
* The usage and invitation rows {@link hasAnchoredProof} resolves an invitation-admitted
|
|
8415
9008
|
* `CadrePeer` row through, keyed for lookup. Both reads honour `retry` for
|
|
8416
|
-
* {@link listAuthorizedMembers}' reason.
|
|
8417
|
-
*
|
|
8418
|
-
* NOTE: once invitations are the only admission route (ticket
|
|
8419
|
-
* `cadre-invitations-redeemable-by-any-member`), every cadre with a member not yet
|
|
8420
|
-
* re-vouched pays these two whole-table reads and three ed25519 verifies per row on every
|
|
8421
|
-
* membership refresh. Both tables are append-only and a verified chain never changes (expiry,
|
|
8422
|
-
* use count and withdrawal are not re-checked), so if the gate's admission deadline ever
|
|
8423
|
-
* shows this, memoise verified (row stamp → usage stamp) pairs per node and skip the reads
|
|
8424
|
-
* when every admitted row is already memoised.
|
|
9009
|
+
* {@link listAuthorizedMembers}' reason. The rows come from the membership views, so they
|
|
9010
|
+
* reach storage only after the table changed or the view's maximum age passed.
|
|
8425
9011
|
*/
|
|
8426
9012
|
private async loadInvitationChain(controlDatabase: ControlDatabase, retry: boolean): Promise<InvitationChain> {
|
|
8427
9013
|
const [usages, invites] = await Promise.all([
|
|
@@ -8587,16 +9173,16 @@ export class CadreNode implements SAppIdLookup {
|
|
|
8587
9173
|
* write-driven refresh never fires for it — yet applying it can ANCHOR a new
|
|
8588
9174
|
* owner key, which flips rows already present from unauthorized to authorized.
|
|
8589
9175
|
* Hence the explicit refresh here; without it a peer the freshly anchored owner
|
|
8590
|
-
* vouched for stays denied until the next
|
|
9176
|
+
* vouched for stays denied until the next membership change or the backstop pass.
|
|
8591
9177
|
*/
|
|
8592
9178
|
private seedEventCallbacks(): SeedEventCallbacks {
|
|
8593
9179
|
return {
|
|
8594
9180
|
onSeedReceived: (partyId, peerId) => this.emit('seed:received', { partyId, peerId }),
|
|
8595
9181
|
onSeedApplied: (partyId, peersAdded, seed) => {
|
|
8596
|
-
this.emit('seed:applied', { partyId, peersAdded });
|
|
8597
9182
|
// A wire-delivered seed never passes through the `applySeed` wrapper, so
|
|
8598
9183
|
// this is where its preferred peers become cold-start bootstrap targets.
|
|
8599
|
-
this.recordSeedBootstrapPeers(seed);
|
|
9184
|
+
const dialTargets = this.recordSeedBootstrapPeers(seed);
|
|
9185
|
+
this.emit('seed:applied', { partyId, peersAdded, dialTargets });
|
|
8600
9186
|
void this.refreshMembershipGate('seed-applied');
|
|
8601
9187
|
},
|
|
8602
9188
|
onSeedError: (partyId, error) => this.emit('seed:error', { partyId, error }),
|
|
@@ -8622,13 +9208,14 @@ export class CadreNode implements SAppIdLookup {
|
|
|
8622
9208
|
* NO `CadrePeer` writer needs to call this: the control database notifies it
|
|
8623
9209
|
* after every committed member-row write (`ControlDatabase.mutateCadrePeer`,
|
|
8624
9210
|
* wired in {@link start}), which is exactly what makes the refresh automatic
|
|
8625
|
-
* rather than a caller obligation.
|
|
8626
|
-
*
|
|
8627
|
-
*
|
|
8628
|
-
*
|
|
8629
|
-
*
|
|
8630
|
-
*
|
|
8631
|
-
*
|
|
9211
|
+
* rather than a caller obligation. A membership row that arrived by REPLICATION
|
|
9212
|
+
* refreshes it too, through the gate watch {@link start} takes on the membership
|
|
9213
|
+
* tables (within its 50 ms window; a change this node's storage never hears of
|
|
9214
|
+
* waits for the reconcile pass's backstop). It stays public for the one change
|
|
9215
|
+
* that writes no row and raises no table event: a newly anchored trusted owner
|
|
9216
|
+
* key ({@link applySeed}), which flips rows ALREADY present from unauthorized to
|
|
9217
|
+
* authorized without touching them. A caller may also await it to observe a
|
|
9218
|
+
* refresh that began after its own change.
|
|
8632
9219
|
*
|
|
8633
9220
|
* Coalescing: marks the snapshot stale and resolves once a refresh that began
|
|
8634
9221
|
* after this call has completed, so a caller that awaits it always observes
|
|
@@ -8654,13 +9241,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
8654
9241
|
* connected peer it newly covers — the strand-side counterpart to
|
|
8655
9242
|
* {@link refreshMembershipGate}.
|
|
8656
9243
|
*
|
|
8657
|
-
*
|
|
8658
|
-
*
|
|
8659
|
-
*
|
|
8660
|
-
*
|
|
8661
|
-
* The gate therefore polls (default 30 s). An app that has just called
|
|
8662
|
-
* `revokeMember` or `leaveStrand` should follow the write with this call so
|
|
8663
|
-
* the cut is immediate rather than up to one poll interval late.
|
|
9244
|
+
* The gate already refreshes on its own when the strand's `Member` or
|
|
9245
|
+
* `MemberPeer` table changes: at once for a `revokeMember` or `leaveStrand`
|
|
9246
|
+
* written on this machine, and when a replicated removal reaches this node's
|
|
9247
|
+
* storage. Call this to await the cut — it resolves after the sweep.
|
|
8664
9248
|
*
|
|
8665
9249
|
* Quiet no-op for a strand that is not running, is quiesced, is open, or has
|
|
8666
9250
|
* the gate disabled. Never rejects; resolves once the sweep has finished.
|
|
@@ -8711,8 +9295,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
8711
9295
|
* CONCURRENT external write's refresh for the life of the scope: that writer's
|
|
8712
9296
|
* promise resolves before its peer is admitted, and admission lands at scope
|
|
8713
9297
|
* exit instead. Acceptable — scope exit is milliseconds away (bounded by the
|
|
8714
|
-
* drain), versus the
|
|
8715
|
-
*
|
|
9298
|
+
* drain), versus the reconcile pass that was the alternative before any of this
|
|
9299
|
+
* existed — but it is the reason these scopes stay short and rare.
|
|
8716
9300
|
*/
|
|
8717
9301
|
private async deferMembershipGateRefresh<T>(reason: string, body: () => Promise<T>): Promise<T> {
|
|
8718
9302
|
this.membershipGateDeferDepth++;
|
|
@@ -8854,7 +9438,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
8854
9438
|
throw new Error('CadreNode must be started before planning an owner key removal');
|
|
8855
9439
|
}
|
|
8856
9440
|
const key = requireEd25519PublicKeyB64(ownerPublicKeyB64, 'owner key');
|
|
8857
|
-
return
|
|
9441
|
+
return await this.planOwnerKeyRemovalWith(this.controlDatabase, key);
|
|
8858
9442
|
}
|
|
8859
9443
|
|
|
8860
9444
|
/**
|
|
@@ -8888,7 +9472,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
8888
9472
|
throw new Error('Seed bootstrap service not initialized. Call initializeSeedBootstrap() first.');
|
|
8889
9473
|
}
|
|
8890
9474
|
const key = requireEd25519PublicKeyB64(ownerPublicKeyB64, 'owner key');
|
|
8891
|
-
const
|
|
9475
|
+
const plan = await this.planOwnerKeyRemovalWith(this.controlDatabase, key);
|
|
8892
9476
|
if (plan.refusal !== null) {
|
|
8893
9477
|
throw new OwnerKeyRemovalRefusedError(key, plan.refusal);
|
|
8894
9478
|
}
|
|
@@ -8903,19 +9487,19 @@ export class CadreNode implements SAppIdLookup {
|
|
|
8903
9487
|
}
|
|
8904
9488
|
const action = `Remove the owner key ${namedOwnerKey(key, plan.ownerKey?.label ?? null)} from the cadre`;
|
|
8905
9489
|
const removed = await signer.approve({ partyId: this.partyId, action }, async (sign) => {
|
|
8906
|
-
await this.revouchDependents(service, revouch,
|
|
9490
|
+
await this.revouchDependents(service, revouch, sign);
|
|
8907
9491
|
return await service.removeOwner(key, sign);
|
|
8908
9492
|
});
|
|
8909
9493
|
log('removeOwner(%s): removed=%s revouched owners=%d peers=%d', key, removed, revouch.owners.length, revouch.peers.length);
|
|
8910
9494
|
return { removed, revouched: revouch };
|
|
8911
9495
|
}
|
|
8912
9496
|
|
|
8913
|
-
/** {@link planOwnerKeyRemoval}
|
|
8914
|
-
private async planOwnerKeyRemovalWith(controlDatabase: ControlDatabase, key: string): Promise<
|
|
9497
|
+
/** {@link planOwnerKeyRemoval} over `controlDatabase`, for a caller that already holds it. */
|
|
9498
|
+
private async planOwnerKeyRemovalWith(controlDatabase: ControlDatabase, key: string): Promise<OwnerKeyRemovalPlan> {
|
|
8915
9499
|
const [rows, owners, grants] = await Promise.all([
|
|
8916
9500
|
controlDatabase.queryCadrePeers(), controlDatabase.queryOwnerKeyRows(), controlDatabase.listOwnerGrantStatuses()
|
|
8917
9501
|
]);
|
|
8918
|
-
|
|
9502
|
+
return planOwnerKeyRemoval({
|
|
8919
9503
|
key,
|
|
8920
9504
|
rows,
|
|
8921
9505
|
owners,
|
|
@@ -8924,7 +9508,6 @@ export class CadreNode implements SAppIdLookup {
|
|
|
8924
9508
|
selfOwnerKey: this.ownerSignerEnrolledIn(new Set(owners.map((owner) => owner.key)))?.publicKey ?? null,
|
|
8925
9509
|
anchor: this.trustedOwnerStore?.all() ?? new Set(),
|
|
8926
9510
|
});
|
|
8927
|
-
return { plan, rows };
|
|
8928
9511
|
}
|
|
8929
9512
|
|
|
8930
9513
|
/**
|
|
@@ -8934,12 +9517,14 @@ export class CadreNode implements SAppIdLookup {
|
|
|
8934
9517
|
* the only rebinding `OwnerKey.NoUpdate` allows); peers through the owner re-touch
|
|
8935
9518
|
* (`reauthorizePeer`), which rewrites `VouchOwner`/`VouchSig` and clears `VouchUsage`.
|
|
8936
9519
|
*
|
|
8937
|
-
* The peer re-touch
|
|
8938
|
-
* `
|
|
8939
|
-
*
|
|
8940
|
-
*
|
|
8941
|
-
*
|
|
8942
|
-
*
|
|
9520
|
+
* The peer re-touch is the re-vouch mode of `reauthorizePeer`, which leaves the row's
|
|
9521
|
+
* `UpdatedAt` untouched and writes whether or not the machine has self-published. The owner
|
|
9522
|
+
* branch of `CadrePeer.AuthorizedUpdate` does not require a bump, and bumping a self-published
|
|
9523
|
+
* row would leave its `Sig` unverifiable, so unresolvable and unlisted as self-published, until
|
|
9524
|
+
* the machine's next heartbeat (the NOTE on that constraint); the write-while-alone drain's
|
|
9525
|
+
* re-broadcast mode bumps because it re-issues an otherwise unchanged row, whereas here the
|
|
9526
|
+
* voucher itself changes. A dependent whose row vanished since the plan is reported absent and
|
|
9527
|
+
* skipped.
|
|
8943
9528
|
*
|
|
8944
9529
|
* A failure rethrows and stops the call before the removal; re-vouches already made stay
|
|
8945
9530
|
* (the row is only re-signed by a current owner) and the caller retries. An owner re-vouch
|
|
@@ -8953,7 +9538,6 @@ export class CadreNode implements SAppIdLookup {
|
|
|
8953
9538
|
private async revouchDependents(
|
|
8954
9539
|
service: SeedBootstrapService,
|
|
8955
9540
|
revouch: OwnerKeyDependents,
|
|
8956
|
-
rows: readonly CadrePeerRow[],
|
|
8957
9541
|
sign: OwnerSign,
|
|
8958
9542
|
): Promise<void> {
|
|
8959
9543
|
for (const key of revouch.owners) {
|
|
@@ -8964,9 +9548,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
8964
9548
|
}
|
|
8965
9549
|
}
|
|
8966
9550
|
for (const dependent of revouch.peers) {
|
|
8967
|
-
|
|
8968
|
-
const updatedAt = row !== undefined && row.updatedAt > 0 ? row.updatedAt : Date.now();
|
|
8969
|
-
await service.reauthorizePeer(dependent, updatedAt, sign);
|
|
9551
|
+
await service.reauthorizePeer(dependent, 'revouch', sign);
|
|
8970
9552
|
}
|
|
8971
9553
|
}
|
|
8972
9554
|
|
|
@@ -8993,6 +9575,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
8993
9575
|
throw new Error('CadreNode must be started before planning a machine removal');
|
|
8994
9576
|
}
|
|
8995
9577
|
const machines = await this.readMachines(this.controlDatabase);
|
|
9578
|
+
// NOTE: the invitation and usage lists come from the membership views, so on a node that
|
|
9579
|
+
// heard of a row with no storage event (outside the block's cohort) an invitation issued
|
|
9580
|
+
// elsewhere in the last MEMBERSHIP_VIEW_MAX_AGE_MS can be missing and left unwithdrawn. If
|
|
9581
|
+
// removal must close that window, read both tables fresh here.
|
|
8996
9582
|
const [invites, usages, owners] = await Promise.all([
|
|
8997
9583
|
this.controlDatabase.listCadreInviteStatuses(),
|
|
8998
9584
|
this.controlDatabase.queryCadreInviteUsages(),
|
|
@@ -9224,12 +9810,13 @@ export class CadreNode implements SAppIdLookup {
|
|
|
9224
9810
|
* that connection from the addresses it was handed, on this launch or a later
|
|
9225
9811
|
* one.
|
|
9226
9812
|
*
|
|
9227
|
-
* Nothing is dialed here: the drone has not received the seed yet.
|
|
9228
|
-
*
|
|
9229
|
-
*
|
|
9230
|
-
*
|
|
9231
|
-
*
|
|
9232
|
-
*
|
|
9813
|
+
* Nothing is dialed here: the drone has not received the seed yet. Its `CadrePeer`
|
|
9814
|
+
* insert is a membership change, so a reconcile pass dials the retained addresses
|
|
9815
|
+
* within about 50 ms of this call returning, whether or not the caller runs one, and
|
|
9816
|
+
* a failed dial is retried on the redial ladder (15 s, 30 s, 60 s, then every
|
|
9817
|
+
* backstop). A caller that wants to await the connection calls
|
|
9818
|
+
* {@link reconcileControlCohort} itself, twice in a row: a call made while a pass is
|
|
9819
|
+
* in flight joins it, and that pass may have listed siblings before this add.
|
|
9233
9820
|
*/
|
|
9234
9821
|
async addDrone(options: AddDroneOptions): Promise<DroneInitResult> {
|
|
9235
9822
|
if (!this.seedBootstrapService) {
|
|
@@ -9275,12 +9862,19 @@ export class CadreNode implements SAppIdLookup {
|
|
|
9275
9862
|
* by the node (its anchored branch), the row insert is a no-op on an existing row
|
|
9276
9863
|
* (`ControlDatabase.insertCadrePeer`), and the dial target is replaced, not duplicated.
|
|
9277
9864
|
*
|
|
9278
|
-
*
|
|
9279
|
-
*
|
|
9280
|
-
*
|
|
9281
|
-
*
|
|
9282
|
-
*
|
|
9283
|
-
*
|
|
9865
|
+
* Once it has acknowledged, the node restarts into this party (cadre-cli's
|
|
9866
|
+
* `restartIntoClaimedParty`), closing the connection the claim used. The dial target puts
|
|
9867
|
+
* the address that connection was dialed on first, so every later dial tries the address
|
|
9868
|
+
* that just worked before the others (a QR code lists public addresses before LAN ones, and
|
|
9869
|
+
* a home router often drops a connection to its own public address from inside). A
|
|
9870
|
+
* reconcile pass is kicked (not awaited) now, and the vouch's `CadrePeer` insert starts
|
|
9871
|
+
* another about 50 ms later; for the first {@link CONTROL_COHORT_REDIAL_MS} after the
|
|
9872
|
+
* acknowledgement this node also retries the node every second while it restarts
|
|
9873
|
+
* ({@link redialClaimedNode}). A caller that wants to wait for the connection awaits
|
|
9874
|
+
* {@link reconcileControlCohort} itself, with `addDrone`'s caveat that a pass already in
|
|
9875
|
+
* flight is joined rather than restarted. The claimed node dials this one back too, as the
|
|
9876
|
+
* seed's minter, when this node has a listen or relay address; a phone with neither is
|
|
9877
|
+
* reached by its own dials alone.
|
|
9284
9878
|
*
|
|
9285
9879
|
* @throws {ClaimRefusedError} when the node was reached and refused the claim.
|
|
9286
9880
|
* @throws {PeerUnreachableError} when the node was never reached.
|
|
@@ -9295,6 +9889,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
9295
9889
|
if (!signer) {
|
|
9296
9890
|
throw new Error('Owner key required to create seeds');
|
|
9297
9891
|
}
|
|
9892
|
+
let dialAddrs = target.multiaddrs;
|
|
9298
9893
|
await signer.approve({ partyId: this.partyId, action: `Claim the node ${target.peerId} for the cadre` }, async (sign) => {
|
|
9299
9894
|
const seed = await service.createSeed(sign);
|
|
9300
9895
|
const proof = claimProof(secret, target.peerId, seed.signerKey, seedDigest(seed));
|
|
@@ -9303,14 +9898,91 @@ export class CadreNode implements SAppIdLookup {
|
|
|
9303
9898
|
if (!ack.accepted) {
|
|
9304
9899
|
throw new ClaimRefusedError(target.peerId, ack.reason ?? 'no reason given', ack.code);
|
|
9305
9900
|
}
|
|
9901
|
+
// Read before anything else is awaited: the node's restart closes the connection.
|
|
9902
|
+
dialAddrs = this.connectedAddrFirst(target.peerId, target.multiaddrs);
|
|
9306
9903
|
log('claimNode: %s accepted the claim; vouching it into the cadre', target.peerId);
|
|
9307
9904
|
// The wrapper queues the write for re-replication: a phone claiming its first
|
|
9308
9905
|
// always-on node has no control connection yet, so the insert commits local-only.
|
|
9906
|
+
// NOTE: accepted tradeoff — the node's own signed record is not carried in the
|
|
9907
|
+
// acknowledgement and written with this vouch: it would be signed before the node
|
|
9908
|
+
// restarts, so its addresses may not be the restarted node's (a new relay reservation),
|
|
9909
|
+
// and the write-while-alone drain re-sends only rows whose `Sig` is null
|
|
9910
|
+
// (drainPendingPeerWrites, reconstructAuthoredMembership), so a vouch carrying it would
|
|
9911
|
+
// never reach this cadre's other members. The node publishes its record once it has
|
|
9912
|
+
// restarted. Revisit if the drain ever re-sends signed rows.
|
|
9309
9913
|
await this.vouchPeer(target.peerId, target.multiaddrs, sign);
|
|
9310
9914
|
});
|
|
9311
|
-
this.retainDialTarget(target.peerId,
|
|
9915
|
+
this.retainDialTarget(target.peerId, dialAddrs, 'claimNode');
|
|
9312
9916
|
void this.reconcileControlCohort().catch((error) =>
|
|
9313
9917
|
log('claimNode: reconcile kick after claiming %s failed: %o', target.peerId, error));
|
|
9918
|
+
void this.redialClaimedNode(target.peerId).catch((error) =>
|
|
9919
|
+
log('claimNode: redialing %s while it restarts failed: %o', target.peerId, error));
|
|
9920
|
+
}
|
|
9921
|
+
|
|
9922
|
+
/**
|
|
9923
|
+
* `addrs` with the address this node's open connection to `peerId` was dialed on moved to
|
|
9924
|
+
* the front, when it is one of them; `addrs` unchanged otherwise. Compared without a
|
|
9925
|
+
* trailing `/p2p/<peerId>`, which a QR code's address may or may not carry ({@link addrKey}).
|
|
9926
|
+
*/
|
|
9927
|
+
private connectedAddrFirst(peerId: string, addrs: string[]): string[] {
|
|
9928
|
+
const controlNode = this.controlNode;
|
|
9929
|
+
if (!controlNode) {
|
|
9930
|
+
return addrs;
|
|
9931
|
+
}
|
|
9932
|
+
const remote = peerIdFromString(peerId);
|
|
9933
|
+
const connected = new Set(controlNode.getConnections(remote).map((c) => addrKey(c.remoteAddr.toString(), remote)));
|
|
9934
|
+
const index = addrs.findIndex((addr) => connected.has(addrKey(addr, remote)));
|
|
9935
|
+
return index <= 0 ? addrs : [addrs[index], ...addrs.slice(0, index), ...addrs.slice(index + 1)];
|
|
9936
|
+
}
|
|
9937
|
+
|
|
9938
|
+
/**
|
|
9939
|
+
* Retry a node this node has just claimed ({@link claimNode}) once every
|
|
9940
|
+
* {@link CLAIM_RESTART_REDIAL_STEP_MS} for {@link CONTROL_COHORT_REDIAL_MS} after it accepted.
|
|
9941
|
+
* The node restarts into this party right after it acknowledges, and while it does, a
|
|
9942
|
+
* reconcile pass's dial either fails or opens a connection the restarting node closes at
|
|
9943
|
+
* once. The first failure puts the pass's next redial 15 s out ({@link armReconcileWake}),
|
|
9944
|
+
* and an owner that cannot be dialed has no other way back to the node.
|
|
9945
|
+
*
|
|
9946
|
+
* Each step dials once from the retained dial target, unless a connection to the node is
|
|
9947
|
+
* open: the one the claim used until the restart closes it, or one to the restarted node
|
|
9948
|
+
* that has not identified yet. A dial that connects does not end the loop, because the
|
|
9949
|
+
* restarting node may still close it. The loop ends once the node has identified as serving
|
|
9950
|
+
* this party's control database ({@link identifiedControlPeers}, which the placeholder node
|
|
9951
|
+
* it ran before the claim never joins), at the window's end, when this node stops, or when
|
|
9952
|
+
* the target is forgotten ({@link removePeer}). Steps run one after another, so a dial
|
|
9953
|
+
* finishes on its own budget first. The redial streak is left alone: the ladder carries on
|
|
9954
|
+
* after the window. One loop per peer: a repeated claim while one runs starts none.
|
|
9955
|
+
*/
|
|
9956
|
+
private async redialClaimedNode(peerId: string): Promise<void> {
|
|
9957
|
+
if (this.claimRedialPeers.has(peerId)) {
|
|
9958
|
+
return;
|
|
9959
|
+
}
|
|
9960
|
+
this.claimRedialPeers.add(peerId);
|
|
9961
|
+
try {
|
|
9962
|
+
const endsAt = Date.now() + CONTROL_COHORT_REDIAL_MS;
|
|
9963
|
+
while (Date.now() < endsAt && await this.redialClaimedNodeStep(peerId)) {
|
|
9964
|
+
await new Promise<void>((resolve) => {
|
|
9965
|
+
const timer = setTimeout(resolve, CLAIM_RESTART_REDIAL_STEP_MS);
|
|
9966
|
+
// Node-only: a stop mid-window must not keep the event loop alive for the step.
|
|
9967
|
+
(timer as { unref?: () => void }).unref?.();
|
|
9968
|
+
});
|
|
9969
|
+
}
|
|
9970
|
+
} finally {
|
|
9971
|
+
this.claimRedialPeers.delete(peerId);
|
|
9972
|
+
}
|
|
9973
|
+
}
|
|
9974
|
+
|
|
9975
|
+
/** One step of {@link redialClaimedNode}; false ends the loop. */
|
|
9976
|
+
private async redialClaimedNodeStep(peerId: string): Promise<boolean> {
|
|
9977
|
+
const controlNode = this.controlNode;
|
|
9978
|
+
const target = this.bootstrapPeerStore?.all().get(peerId);
|
|
9979
|
+
if (!this._running || !controlNode || !target || this.identifiedControlPeers.has(peerId)) {
|
|
9980
|
+
return false;
|
|
9981
|
+
}
|
|
9982
|
+
if (controlNode.getConnections(peerIdFromString(peerId)).length === 0) {
|
|
9983
|
+
await this.dialBootstrapPeer(peerId, target.addrs, 'claimNode(redial)');
|
|
9984
|
+
}
|
|
9985
|
+
return true;
|
|
9314
9986
|
}
|
|
9315
9987
|
|
|
9316
9988
|
/**
|
|
@@ -9481,11 +10153,13 @@ export class CadreNode implements SAppIdLookup {
|
|
|
9481
10153
|
* Steps: refuse an invitation for another party; pin its `ownerKeys` into the node-local
|
|
9482
10154
|
* anchor (`'invite'` provenance — the pin sticks even if the redemption then fails, as it
|
|
9483
10155
|
* does for a seed invite, so a later attempt or seed from the same owner is anchored);
|
|
9484
|
-
* sign the request
|
|
9485
|
-
* acceptance verify the reply against the pinned
|
|
9486
|
-
* store, retain the answering member as a
|
|
9487
|
-
*
|
|
9488
|
-
* the connection it already holds, the same unified behaviour as after a seed.
|
|
10156
|
+
* sign the request, which carries this node's own signed address record; dial the listed
|
|
10157
|
+
* members in order (`redeemAtMembers`); on acceptance verify the reply against the pinned
|
|
10158
|
+
* keys, merge its dial hints into the peer store, retain the answering member as a
|
|
10159
|
+
* cold-start dial target and start a cohort reconcile pass. This node then syncs the control
|
|
10160
|
+
* database over the connection it already holds, the same unified behaviour as after a seed.
|
|
10161
|
+
* The member writes the record on this node's `CadrePeer` row when its signature verifies, so
|
|
10162
|
+
* the row resolves to this node's addresses from its admission.
|
|
9489
10163
|
*
|
|
9490
10164
|
* A device that is already a member (a restart with the invitation still configured) is
|
|
9491
10165
|
* answered as accepted by the member's idempotent redemption, with no second usage row.
|
|
@@ -9529,12 +10203,20 @@ export class CadreNode implements SAppIdLookup {
|
|
|
9529
10203
|
}
|
|
9530
10204
|
await this.trustOwnerKeys(invitation.ownerKeys, 'invite');
|
|
9531
10205
|
const pinned = new Set(invitation.ownerKeys);
|
|
10206
|
+
const selfPeerId = controlNode.peerId.toString();
|
|
10207
|
+
// The addresses a publish would carry, sanitized before signing: the member stores the
|
|
10208
|
+
// signature only over the exact list it stores.
|
|
10209
|
+
const record = this.signSelfRecord(selfPeerId, signer, sanitizeAddrs(await this.collectSelfAddrs()), null);
|
|
9532
10210
|
const request = signRedeemRequest(
|
|
9533
10211
|
invitation,
|
|
9534
10212
|
{ peerKey: signer.publicKeyB64, peerPrivateKey: signer.privateKeyB64 },
|
|
9535
|
-
generateStampId(
|
|
9536
|
-
|
|
10213
|
+
generateStampId(selfPeerId),
|
|
10214
|
+
record
|
|
9537
10215
|
);
|
|
10216
|
+
// Before dialing, not on acceptance: the member's commit can reach this node before its
|
|
10217
|
+
// reply does, and a reconcile pass that reads the row first must find it current. A record
|
|
10218
|
+
// no member wrote matches no row, so a failed redemption leaves nothing to undo.
|
|
10219
|
+
this.lastSignedSelfRecord = record;
|
|
9538
10220
|
const { reply, memberPeerId, memberAddr } = await redeemInPasses(controlNode, {
|
|
9539
10221
|
invitation,
|
|
9540
10222
|
request,
|
|
@@ -9546,7 +10228,6 @@ export class CadreNode implements SAppIdLookup {
|
|
|
9546
10228
|
});
|
|
9547
10229
|
log('Cadre invitation %s redeemed at %s', invitation.invite.key, memberAddr);
|
|
9548
10230
|
|
|
9549
|
-
const selfPeerId = controlNode.peerId.toString();
|
|
9550
10231
|
await mergeSeedPeers(controlNode, reply.peers.filter((peer) => peer.peerId !== selfPeerId));
|
|
9551
10232
|
if (memberPeerId !== null) {
|
|
9552
10233
|
// Every address the bundle gave the answering member, so a restart has the same choices.
|
|
@@ -9555,9 +10236,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
9555
10236
|
.map((addr) => addr.toString());
|
|
9556
10237
|
this.retainDialTarget(memberPeerId, memberAddrs, 'redeemCadreInvitation');
|
|
9557
10238
|
}
|
|
9558
|
-
//
|
|
9559
|
-
//
|
|
9560
|
-
await this.refreshMembershipGate('invitation-redeemed');
|
|
10239
|
+
// No gate refresh here: the rows about to replicate in are judged against the pins above
|
|
10240
|
+
// when they land, by the gate watch's refresh and the head refresh of this pass.
|
|
9561
10241
|
void this.reconcileControlCohort().catch((error) =>
|
|
9562
10242
|
log('redeemCadreInvitation: the post-redemption reconcile pass failed: %o', error));
|
|
9563
10243
|
return { peerId: memberPeerId, redeemedAt: new Date().toISOString() };
|
|
@@ -9737,25 +10417,31 @@ export class CadreNode implements SAppIdLookup {
|
|
|
9737
10417
|
}
|
|
9738
10418
|
|
|
9739
10419
|
/**
|
|
9740
|
-
* {@link redeemOwnerGrant}'s local path: the alone refusal, the seat
|
|
9741
|
-
* then one approval around the consent signature and the
|
|
9742
|
-
* the approval, so a prompting signer never asks for a
|
|
10420
|
+
* {@link redeemOwnerGrant}'s local path: the alone refusal, the seat (only when this node holds
|
|
10421
|
+
* no grant row), the liveness refusal, then one approval around the consent signature and the
|
|
10422
|
+
* write. Every refusal comes before the approval, so a prompting signer never asks for a
|
|
10423
|
+
* redemption this node would refuse. The held row is read once and carried into the liveness
|
|
10424
|
+
* check and the write, as the member handler carries it (`MemberRedemptionHandler`).
|
|
9743
10425
|
*/
|
|
9744
10426
|
private async redeemOwnerGrantLocally(controlDatabase: ControlDatabase, bundle: OwnerGrantBundle, signer: OwnerSigner): Promise<RedeemOwnerGrantResult> {
|
|
9745
10427
|
if (await this.ownerKeyWriteAlone()) {
|
|
9746
10428
|
throw new OwnerKeyAddRefusedError(signer.publicKey, 'alone');
|
|
9747
10429
|
}
|
|
9748
|
-
|
|
10430
|
+
const held = await controlDatabase.queryOwnerGrant(bundle.grant.key);
|
|
10431
|
+
if (held === null) {
|
|
10432
|
+
await refusingUnsyncedGrant(() => controlDatabase.seatOwnerGrant(bundle.grant));
|
|
10433
|
+
}
|
|
9749
10434
|
// The write answers a key that is already an owner without reading the grant, so only a
|
|
9750
10435
|
// key that is not one needs the grant live (the same permissive pre-check the handler's
|
|
9751
10436
|
// admission uses; the write enforces every condition).
|
|
9752
|
-
if (!(await controlDatabase.getOwnerKeys()).has(signer.publicKey) && !await controlDatabase.isOwnerGrantLive(bundle.grant)) {
|
|
10437
|
+
if (!(await controlDatabase.getOwnerKeys()).has(signer.publicKey) && !await controlDatabase.isOwnerGrantLive(bundle.grant, held)) {
|
|
9753
10438
|
throw new OwnerGrantRejectedError('grant-spent', 'The grant is expired, withdrawn, exhausted, or its issuer is no longer an owner');
|
|
9754
10439
|
}
|
|
9755
10440
|
const { alreadyOwner } = await signer.approve(ownerGrantApproval(bundle), async (sign) => {
|
|
9756
10441
|
const request = await this.signOwnerGrantRequest(bundle, signer, sign);
|
|
9757
10442
|
return await refusingUnsyncedGrant(() => controlDatabase.redeemOwnerGrant({
|
|
9758
10443
|
grantKey: bundle.grant.key,
|
|
10444
|
+
...(held ? { held } : {}),
|
|
9759
10445
|
seatedKey: signer.publicKey,
|
|
9760
10446
|
usageStampId: request.usageStampId,
|
|
9761
10447
|
grantSig: request.grantSig,
|
|
@@ -9950,8 +10636,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
9950
10636
|
// Overridden on the same grounds: only this node holds the running strand
|
|
9951
10637
|
// instance's database and this party's `StrandPartyKey` identity, and a wrong
|
|
9952
10638
|
// issuer here would admit joiners under someone else's authority.
|
|
9953
|
-
issueMembershipInvite: (
|
|
9954
|
-
this.issueStrandMembershipInvite(strandId, signal)
|
|
10639
|
+
issueMembershipInvite: (host, signal) => this.issueStrandMembershipInvite(host, signal)
|
|
9955
10640
|
});
|
|
9956
10641
|
|
|
9957
10642
|
const previous = this.strandSolicitationService;
|
|
@@ -10259,10 +10944,17 @@ export class CadreNode implements SAppIdLookup {
|
|
|
10259
10944
|
selfId: this.controlNode!.peerId.toString(),
|
|
10260
10945
|
linkRoundTripMs: this.config.network?.linkRoundTripMs,
|
|
10261
10946
|
isOwner: () => this.isOwner(),
|
|
10262
|
-
// NOTE: one owner-key read and one pending-join read (three tables) per pass (30 s); for a
|
|
10263
|
-
// party that never asked for a join the second reads never-written blocks, which consults
|
|
10264
|
-
// the cohort. If it shows up in a device profile, poll more slowly while the tables read empty.
|
|
10265
10947
|
readRows: () => database().queryPendingJoins(),
|
|
10948
|
+
// What `isOwner` and `queryPendingJoins` read: the owner keys, the joins and their
|
|
10949
|
+
// outcomes, and the tombstones both filter by. A dismissal writes its tombstone with the
|
|
10950
|
+
// request's delete, but the two can replicate separately.
|
|
10951
|
+
watchChanges: (onChange, options) => database().watchTables([
|
|
10952
|
+
'CadreControl.JoinRequest',
|
|
10953
|
+
'CadreControl.JoinSuccess',
|
|
10954
|
+
'CadreControl.JoinFailure',
|
|
10955
|
+
'CadreControl.OwnerKey',
|
|
10956
|
+
'CadreControl.Revocation',
|
|
10957
|
+
], onChange, options),
|
|
10266
10958
|
readRow: (id) => database().queryPendingJoin(id),
|
|
10267
10959
|
attempt: (join) => this.formStrand(this.pendingJoinInvitation(join), parseStoredDisclosure(join)),
|
|
10268
10960
|
record: (expected, outcome) => ownerWrite(`Record the outcome of the pending join ${expected.Id}`,
|
|
@@ -10328,6 +11020,13 @@ export class CadreNode implements SAppIdLookup {
|
|
|
10328
11020
|
* token is spent, and a join no store names would vanish at the next restart.
|
|
10329
11021
|
*/
|
|
10330
11022
|
private async rememberFormedStrand(result: FormStrandResult): Promise<void> {
|
|
11023
|
+
// NOTE: accepted tradeoff — the party-wide `JoinedStrand` row is not written here, in
|
|
11024
|
+
// formStrand's own approval, but later by an owner machine's connected reconcile pass
|
|
11025
|
+
// (`JoinedStrandSession.syncWithParty`). That publish is gated on a control connection so a
|
|
11026
|
+
// machine cut off from its own party never commits it alone, and during formStrand the
|
|
11027
|
+
// count always includes the connection to the other party's responder, so the gate would
|
|
11028
|
+
// always pass here. Non-owner and signer-less machines need this local path regardless.
|
|
11029
|
+
// Revisit if the gate can tell a connection to this party's own machines from any other.
|
|
10331
11030
|
try {
|
|
10332
11031
|
await this.joinedStrands!.remember({
|
|
10333
11032
|
Id: result.strandId,
|
|
@@ -10371,7 +11070,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
10371
11070
|
*/
|
|
10372
11071
|
private async adoptFormationMembershipInvite(strandId: string, invite: StrandMembershipInvite): Promise<void> {
|
|
10373
11072
|
try {
|
|
10374
|
-
await this.ensureStrandPartyKey(strandId);
|
|
11073
|
+
this.formedPartyKeys.set(strandId, await this.ensureStrandPartyKey(strandId));
|
|
10375
11074
|
} catch (error) {
|
|
10376
11075
|
throw new FormationPostApprovalError(
|
|
10377
11076
|
strandId,
|
|
@@ -10422,6 +11121,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
10422
11121
|
* implements): mint a single-use `Strand.Invite` against the LIVE host strand so a
|
|
10423
11122
|
* validated joiner can seat its own `Strand.Member` row.
|
|
10424
11123
|
*
|
|
11124
|
+
* `host` is the strand as the formation session resolved it from its `Strand` row, read
|
|
11125
|
+
* once there; this method reads no `Strand` row of its own. A row removed in between is
|
|
11126
|
+
* refused when the consent row is written (the `StrandExists` CHECK).
|
|
11127
|
+
*
|
|
10425
11128
|
* - Open host strand → `null` (no members, nothing to invite into).
|
|
10426
11129
|
* - Closed host strand whose founder launch was refused as pre-split
|
|
10427
11130
|
* ({@link strandLaunchRefusals}) → rethrow the recorded `PreSplitStrandIdentityError`;
|
|
@@ -10470,19 +11173,14 @@ export class CadreNode implements SAppIdLookup {
|
|
|
10470
11173
|
* not a retry.
|
|
10471
11174
|
*/
|
|
10472
11175
|
private async issueStrandMembershipInvite(
|
|
10473
|
-
|
|
11176
|
+
host: BoundHostStrand,
|
|
10474
11177
|
signal?: AbortSignal
|
|
10475
11178
|
): Promise<StrandMembershipInvite | null> {
|
|
11179
|
+
const { strandId } = host;
|
|
10476
11180
|
if (!this.controlDatabase) {
|
|
10477
11181
|
throw new Error(`Cannot issue a membership invitation for strand ${strandId}: control database unavailable`);
|
|
10478
11182
|
}
|
|
10479
|
-
|
|
10480
|
-
if (!row) {
|
|
10481
|
-
// The bound resolution saw this row moments ago; a vanished row is a concurrent
|
|
10482
|
-
// unpublish — reject rather than invite into a strand this party just removed.
|
|
10483
|
-
throw new Error(`Cannot issue a membership invitation for strand ${strandId}: its Strand row is gone`);
|
|
10484
|
-
}
|
|
10485
|
-
if (row.Type !== 'c') {
|
|
11183
|
+
if (host.strandType !== 'c') {
|
|
10486
11184
|
return null;
|
|
10487
11185
|
}
|
|
10488
11186
|
const refusal = this.strandLaunchRefusals.get(strandId);
|
|
@@ -10504,8 +11202,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
10504
11202
|
'not live on this responder (not launched, or still starting)'
|
|
10505
11203
|
);
|
|
10506
11204
|
}
|
|
10507
|
-
if (
|
|
10508
|
-
await assertNotPreSplitStrand(db, strandId, strandMemberKeyPair(
|
|
11205
|
+
if (host.memberPrivateKey) {
|
|
11206
|
+
await assertNotPreSplitStrand(db, strandId, strandMemberKeyPair(host.memberPrivateKey).publicKeyB64);
|
|
10509
11207
|
}
|
|
10510
11208
|
if (signal?.aborted) {
|
|
10511
11209
|
throw new Error(
|