@serfab/cadre-core 1.17.0 → 1.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/dist/cadre-invite-protocol.d.ts +35 -13
- package/dist/cadre-invite-protocol.js +90 -25
- package/dist/cadre-invite-protocol.js.map +1 -1
- package/dist/cadre-node.d.ts +586 -255
- package/dist/cadre-node.js +1182 -566
- package/dist/cadre-node.js.map +1 -1
- package/dist/control-cohort.d.ts +66 -9
- package/dist/control-cohort.js +67 -9
- package/dist/control-cohort.js.map +1 -1
- package/dist/control-database.d.ts +451 -193
- package/dist/control-database.js +701 -359
- package/dist/control-database.js.map +1 -1
- package/dist/control-formation-recorder.d.ts +46 -43
- package/dist/control-formation-recorder.js +77 -78
- package/dist/control-formation-recorder.js.map +1 -1
- package/dist/control-protocol-guard.d.ts +6 -0
- package/dist/control-protocol-guard.js +14 -2
- package/dist/control-protocol-guard.js.map +1 -1
- package/dist/control-schema-version.d.ts +48 -31
- package/dist/control-schema-version.js +67 -45
- package/dist/control-schema-version.js.map +1 -1
- package/dist/control-schema.d.ts +8 -8
- package/dist/control-schema.js +16 -16
- package/dist/control-schema.js.map +1 -1
- package/dist/control-write-retry.d.ts +42 -7
- package/dist/control-write-retry.js +87 -19
- package/dist/control-write-retry.js.map +1 -1
- package/dist/enrolled-machine-store.d.ts +3 -3
- package/dist/enrolled-machine-store.js +3 -3
- package/dist/formation-token.d.ts +48 -0
- package/dist/formation-token.js +26 -0
- package/dist/formation-token.js.map +1 -0
- package/dist/index.d.ts +15 -14
- package/dist/index.js +17 -14
- package/dist/index.js.map +1 -1
- package/dist/joined-strand-store.d.ts +26 -9
- package/dist/joined-strand-store.js +63 -24
- package/dist/joined-strand-store.js.map +1 -1
- package/dist/link-budget.d.ts +9 -7
- package/dist/link-budget.js +9 -7
- package/dist/link-budget.js.map +1 -1
- package/dist/member-redemption.d.ts +15 -9
- package/dist/member-redemption.js +13 -8
- package/dist/member-redemption.js.map +1 -1
- package/dist/membership-view.d.ts +91 -0
- package/dist/membership-view.js +110 -0
- package/dist/membership-view.js.map +1 -0
- package/dist/owner-grant-protocol.d.ts +11 -7
- package/dist/owner-grant-protocol.js +31 -17
- package/dist/owner-grant-protocol.js.map +1 -1
- package/dist/peer-addr-book.d.ts +10 -1
- package/dist/peer-addr-book.js +4 -3
- package/dist/peer-addr-book.js.map +1 -1
- package/dist/peer-join-backfill.d.ts +96 -30
- package/dist/peer-join-backfill.js +272 -128
- package/dist/peer-join-backfill.js.map +1 -1
- package/dist/pending-join-runner.d.ts +48 -18
- package/dist/pending-join-runner.js +74 -56
- package/dist/pending-join-runner.js.map +1 -1
- package/dist/seed-bootstrap.d.ts +47 -40
- package/dist/seed-bootstrap.js +66 -61
- package/dist/seed-bootstrap.js.map +1 -1
- package/dist/self-address-update.d.ts +13 -0
- package/dist/self-address-update.js +19 -0
- package/dist/self-address-update.js.map +1 -0
- package/dist/strand-database.d.ts +23 -2
- package/dist/strand-database.js +26 -6
- package/dist/strand-database.js.map +1 -1
- package/dist/strand-first-sync-gate.d.ts +176 -89
- package/dist/strand-first-sync-gate.js +288 -122
- package/dist/strand-first-sync-gate.js.map +1 -1
- package/dist/strand-formation-deadlines.d.ts +9 -9
- package/dist/strand-formation-deadlines.js +9 -9
- package/dist/strand-formation-manager.d.ts +23 -17
- package/dist/strand-formation-manager.js +46 -44
- package/dist/strand-formation-manager.js.map +1 -1
- package/dist/strand-formation-protocol.d.ts +5 -8
- package/dist/strand-formation-protocol.js +101 -16
- package/dist/strand-formation-protocol.js.map +1 -1
- package/dist/strand-instance-manager.d.ts +26 -17
- package/dist/strand-instance-manager.js +119 -50
- package/dist/strand-instance-manager.js.map +1 -1
- package/dist/strand-membership-reconciler.d.ts +16 -13
- package/dist/strand-membership-reconciler.js +28 -22
- package/dist/strand-membership-reconciler.js.map +1 -1
- package/dist/strand-membership-writer.d.ts +40 -28
- package/dist/strand-membership-writer.js +212 -153
- package/dist/strand-membership-writer.js.map +1 -1
- package/dist/strand-move.js +5 -2
- package/dist/strand-move.js.map +1 -1
- package/dist/strand-revocation-enforcer.d.ts +134 -60
- package/dist/strand-revocation-enforcer.js +144 -71
- package/dist/strand-revocation-enforcer.js.map +1 -1
- package/dist/strand-solicitation.d.ts +50 -39
- package/dist/strand-solicitation.js +10 -3
- package/dist/strand-solicitation.js.map +1 -1
- package/dist/strand-watcher.d.ts +80 -13
- package/dist/strand-watcher.js +152 -48
- package/dist/strand-watcher.js.map +1 -1
- package/dist/table-change-watch.d.ts +63 -0
- package/dist/table-change-watch.js +209 -0
- package/dist/table-change-watch.js.map +1 -0
- package/dist/timeout-scheduler.d.ts +4 -3
- package/dist/timeout-scheduler.js +4 -3
- package/dist/timeout-scheduler.js.map +1 -1
- package/dist/types.d.ts +90 -49
- package/dist/types.js +4 -4
- package/dist/types.js.map +1 -1
- package/package.json +8 -8
- package/src/cadre-invite-protocol.ts +119 -31
- package/src/cadre-node.ts +1289 -588
- package/src/control-cohort.ts +86 -9
- package/src/control-database.ts +848 -403
- package/src/control-formation-recorder.ts +359 -353
- package/src/control-protocol-guard.ts +16 -2
- package/src/control-schema-version.ts +101 -51
- package/src/control-schema.ts +16 -16
- package/src/control-write-retry.ts +91 -19
- package/src/delegate-admission.ts +351 -351
- package/src/enrolled-machine-store.ts +3 -3
- package/src/formation-token.ts +63 -0
- package/src/index.ts +30 -18
- package/src/joined-strand-store.ts +69 -24
- package/src/link-budget.ts +9 -7
- package/src/member-redemption.ts +20 -13
- package/src/membership-view.ts +144 -0
- package/src/owner-grant-protocol.ts +33 -18
- package/src/peer-addr-book.ts +4 -3
- package/src/peer-join-backfill.ts +961 -761
- package/src/pending-join-runner.ts +76 -53
- package/src/seed-bootstrap.ts +72 -67
- package/src/self-address-update.ts +19 -0
- package/src/strand-database.ts +352 -322
- package/src/strand-first-sync-gate.ts +334 -134
- package/src/strand-formation-deadlines.ts +227 -227
- package/src/strand-formation-manager.ts +731 -725
- package/src/strand-formation-protocol.ts +122 -24
- package/src/strand-instance-manager.ts +2092 -2026
- package/src/strand-membership-reconciler.ts +31 -25
- package/src/strand-membership-writer.ts +2093 -2019
- package/src/strand-move.ts +5 -2
- package/src/strand-revocation-enforcer.ts +740 -654
- package/src/strand-solicitation.ts +61 -34
- package/src/strand-watcher.ts +175 -56
- package/src/table-change-watch.ts +285 -0
- package/src/timeout-scheduler.ts +29 -28
- package/src/types.ts +89 -51
package/src/cadre-node.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import debug from 'debug';
|
|
2
2
|
import { toString as uint8ArrayToString, fromString as uint8ArrayFromString } from 'uint8arrays';
|
|
3
|
-
import type { Libp2p, PeerId, PrivateKey } from '@libp2p/interface';
|
|
3
|
+
import type { Connection, IdentifyResult, Libp2p, PeerId, PeerUpdate, PrivateKey } from '@libp2p/interface';
|
|
4
4
|
import { peerIdFromString, peerIdFromPrivateKey } from '@libp2p/peer-id';
|
|
5
5
|
import { generateKeyPair } from '@libp2p/crypto/keys';
|
|
6
6
|
import { createLibp2pNode, type IRawStorage } from '@optimystic/db-p2p';
|
|
@@ -19,6 +19,7 @@ import type {
|
|
|
19
19
|
SAppConfig,
|
|
20
20
|
CadreNodeEvents,
|
|
21
21
|
ControlNetworkSeed,
|
|
22
|
+
SeedDialTarget,
|
|
22
23
|
ApplySeedResult,
|
|
23
24
|
SeedAckMessage,
|
|
24
25
|
SeedDeliveryTarget,
|
|
@@ -94,7 +95,7 @@ import {
|
|
|
94
95
|
type JoinedStrandStore,
|
|
95
96
|
type PartyJoinedStrandLedger
|
|
96
97
|
} from './joined-strand-store.js';
|
|
97
|
-
import { mergePeerAddrs, groupAddrsByPeerId, type MergeAddrsResult } from './peer-addr-book.js';
|
|
98
|
+
import { addrKey, mergePeerAddrs, groupAddrsByPeerId, type MergeAddrsResult } from './peer-addr-book.js';
|
|
98
99
|
import { strandFretPeerAddrs } from './strand-fret-addrs.js';
|
|
99
100
|
import { verifyCadrePeerVoucher, verifyInvitationAdmission, verifyRevocationSigner } from './peer-authorization.js';
|
|
100
101
|
import { deriveOwnerAnchor } from './owner-anchor-sync.js';
|
|
@@ -116,17 +117,22 @@ import {
|
|
|
116
117
|
verifyDeviceTokenSignature,
|
|
117
118
|
isPushPlatform
|
|
118
119
|
} from './device-token.js';
|
|
119
|
-
import { StrandWatcher, type StrandQueryable, type SAppIdLookup } from './strand-watcher.js';
|
|
120
|
+
import { StrandWatcher, DEFAULT_STRAND_WATCH_BACKSTOP_MS, type StrandQueryable, type SAppIdLookup } from './strand-watcher.js';
|
|
120
121
|
import { StrandInstanceManager, liveStrandStatus } from './strand-instance-manager.js';
|
|
121
122
|
import { PeerJoinBackfill } from './peer-join-backfill.js';
|
|
122
123
|
import { deriveCohortMembers } from './strand-cohort.js';
|
|
123
124
|
import type { CohortPeerRow } from './strand-cohort.js';
|
|
124
125
|
import {
|
|
125
126
|
selectControlCohortDials,
|
|
127
|
+
nextReconcileWakeMs,
|
|
128
|
+
CLAIM_RESTART_REDIAL_STEP_MS,
|
|
129
|
+
CONTROL_COHORT_REDIAL_MS,
|
|
126
130
|
DEFAULT_CONTROL_COHORT_RECONCILE_MS,
|
|
127
131
|
DEFAULT_CONTROL_COHORT_TARGET_DEGREE,
|
|
128
132
|
type ControlCohortReconcileResult
|
|
129
133
|
} from './control-cohort.js';
|
|
134
|
+
import { scheduleRuns, type TableChangeWatch } from './table-change-watch.js';
|
|
135
|
+
import { MEMBERSHIP_VIEW_RECONNECT_RELOAD_MS } from './membership-view.js';
|
|
130
136
|
import {
|
|
131
137
|
dialPeerAddrs,
|
|
132
138
|
SelfRelayOnlyError,
|
|
@@ -138,11 +144,12 @@ import {
|
|
|
138
144
|
import { ADMISSION_DECISION_TIMEOUT_MS, declaredCohortReadDeadlineMs, optimysticDialLimits, peerJoinPushBudget, relayReservationBudgetMs, relayedDialBudgetMs, relayedRequestBudgetMs, resolveLinkRoundTripMs } from './link-budget.js';
|
|
139
145
|
import { EnrollmentService } from './enrollment.js';
|
|
140
146
|
import { HibernationManager, type HibernationCallbacks } from './hibernation-manager.js';
|
|
141
|
-
import { ControlDatabase, generateStampId, isPendingJoinConflict, isStrandIdConflict, IssuedRowNotHereYetError, pendingJoinId, type JoinRequestFields, type RevokedRowRef } from './control-database.js';
|
|
147
|
+
import { ControlDatabase, generateStampId, isPendingJoinConflict, isStrandIdConflict, IssuedRowNotHereYetError, pendingJoinId, type CadrePeerRetouchOutcome, type FoundingInvite, type JoinRequestFields, type RevokedRowRef } from './control-database.js';
|
|
142
148
|
import {
|
|
143
149
|
CadreInviteHandler,
|
|
144
150
|
encodeCadreInvitation,
|
|
145
151
|
redeemInPasses,
|
|
152
|
+
sanitizeAddrs,
|
|
146
153
|
signRedeemRequest,
|
|
147
154
|
type CadreInvitation
|
|
148
155
|
} from './cadre-invite-protocol.js';
|
|
@@ -164,11 +171,12 @@ import {
|
|
|
164
171
|
requestedPendingJoin
|
|
165
172
|
} from './pending-join-runner.js';
|
|
166
173
|
import type { ControlRetryAbandonment } from './control-retry.js';
|
|
167
|
-
import { SeedBootstrapService, mergeSeedPeers, seedDigest, type SeedEventCallbacks, type SeedBootstrapConfig } from './seed-bootstrap.js';
|
|
174
|
+
import { SeedBootstrapService, mergeSeedPeers, seedDialTargets, seedDigest, type SeedEventCallbacks, type SeedBootstrapConfig } from './seed-bootstrap.js';
|
|
168
175
|
import { claimSecretTrustPolicy, type SeedTrustPolicy } from './seed-trust-policy.js';
|
|
169
176
|
import { ClaimRefusedError, claimProof, parseClaimSecret } from './claim-proof.js';
|
|
170
177
|
import {
|
|
171
178
|
StrandSolicitationService,
|
|
179
|
+
type BoundHostStrand,
|
|
172
180
|
type StrandSolicitationServiceOptions
|
|
173
181
|
} from './strand-solicitation.js';
|
|
174
182
|
import { ControlFormationUsageRecorder } from './control-formation-recorder.js';
|
|
@@ -183,7 +191,7 @@ import {
|
|
|
183
191
|
UnauthorizedReservationBudget,
|
|
184
192
|
type InboundConnectionVerdict
|
|
185
193
|
} from './membership-connection-gater.js';
|
|
186
|
-
import { controlNetworkName, installControlProtocolGuard } from './control-protocol-guard.js';
|
|
194
|
+
import { controlNetworkName, installControlProtocolGuard, servesControlDatabase } from './control-protocol-guard.js';
|
|
187
195
|
import { StrandWakeService, dialWake } from './strand-wake-protocol.js';
|
|
188
196
|
import { StrandAddrService, collectStrandAddrs, type StrandAddrPeer, type StrandAddrOutcome, type StrandAddrCollection } from './strand-addr-protocol.js';
|
|
189
197
|
import {
|
|
@@ -197,6 +205,7 @@ import {
|
|
|
197
205
|
type CircuitRelayTarget
|
|
198
206
|
} from './delegate-admission.js';
|
|
199
207
|
import { relayCircuitAddrs, resolveListenAddrs, resolveTransportOptions, RelayReservationFailedError } from './relay-addrs.js';
|
|
208
|
+
import { selfAddressesChanged } from './self-address-update.js';
|
|
200
209
|
import { nodeTransportsOption } from './node-transports.js';
|
|
201
210
|
import { resolveRelayServer, type ResolvedRelayServer } from './relay-server.js';
|
|
202
211
|
import { replacesAdvertisedAddrs, resolveAnnounceAddrs } from './announce-addrs.js';
|
|
@@ -226,8 +235,8 @@ const timing = debug('sereus:cadre:timing');
|
|
|
226
235
|
*
|
|
227
236
|
* Ten minutes sits comfortably inside the peerStore's one-hour address expiry
|
|
228
237
|
* (`MAX_ADDRESS_AGE`, see `peer-addr-book.ts`) with headroom for several missed
|
|
229
|
-
* passes, and far above the
|
|
230
|
-
* strand-addr RPC fan-out stays cheap.
|
|
238
|
+
* passes, and far above the reconcile pass's 2-minute backstop the refresh rides
|
|
239
|
+
* on, so the strand-addr RPC fan-out stays cheap.
|
|
231
240
|
*/
|
|
232
241
|
export const STRAND_PEER_ADDR_REFRESH_MS = 10 * 60 * 1000;
|
|
233
242
|
|
|
@@ -237,11 +246,29 @@ export const STRAND_PEER_ADDR_REFRESH_MS = 10 * 60 * 1000;
|
|
|
237
246
|
* phone asking before its `CadrePeer` row has replicated to the sibling is refused
|
|
238
247
|
* and its delegate grant goes unrecorded, and waiting the full
|
|
239
248
|
* {@link STRAND_PEER_ADDR_REFRESH_MS} for the next try is the delay this bounds.
|
|
240
|
-
*
|
|
241
|
-
*
|
|
249
|
+
* A pass arms its own wake for the earliest due time it leaves behind
|
|
250
|
+
* (`nextReconcileWakeMs`), so a sibling that keeps failing costs one timed-out
|
|
251
|
+
* RPC a minute, with no pass in between.
|
|
242
252
|
*/
|
|
243
253
|
export const STRAND_PEER_ADDR_RETRY_MS = 60 * 1000;
|
|
244
254
|
|
|
255
|
+
/**
|
|
256
|
+
* What one reconcile pass did, for the wrapper that arms the next wake; callers get only
|
|
257
|
+
* its `dialed` ({@link ControlCohortReconcileResult}).
|
|
258
|
+
*/
|
|
259
|
+
interface ControlCohortPassOutcome extends ControlCohortReconcileResult {
|
|
260
|
+
/**
|
|
261
|
+
* The pass dialed a selected sibling, or a cold-start bootstrap peer, and the dial failed.
|
|
262
|
+
* A sibling the pass could not dial at all — no dialable address, or every address relays
|
|
263
|
+
* through this node — does not count: a retry would dial nothing, and every way it gains
|
|
264
|
+
* an address (its record replicating in, a connection it opens) starts a pass of its own.
|
|
265
|
+
*/
|
|
266
|
+
unconnected: boolean;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/** How {@link CadreNode.dialControlSibling} ended: connected, a failed dial, or nothing to dial. */
|
|
270
|
+
type SiblingDialOutcome = 'connected' | 'failed' | 'undialable';
|
|
271
|
+
|
|
245
272
|
/**
|
|
246
273
|
* Default lifetime of an owner grant ({@link CadreNode.createOwnerGrant}): whoever holds the
|
|
247
274
|
* bundle can become an owner, so it is a bearer credential for admin rights and lives only
|
|
@@ -441,6 +468,19 @@ function requireMatchingStrandRow(live: StrandRow, desired: StrandRow, situation
|
|
|
441
468
|
return live;
|
|
442
469
|
}
|
|
443
470
|
|
|
471
|
+
/**
|
|
472
|
+
* What a founding's control write resolved: the live `Strand` row, and the party key the write
|
|
473
|
+
* seated for a closed strand. No key when it adopted a stored row or a stored key; the launch
|
|
474
|
+
* then reads the stored one.
|
|
475
|
+
*/
|
|
476
|
+
interface FoundingWrite {
|
|
477
|
+
strandRow: StrandRow;
|
|
478
|
+
partyKey?: string;
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
/** The publish half of `foundStrand`, handed to its start half. */
|
|
482
|
+
type FoundingPublish = FoundingWrite & Pick<FoundStrandResult, 'founded'>;
|
|
483
|
+
|
|
444
484
|
/**
|
|
445
485
|
* Await `work` unless `signal` aborts first: `true` once `work` resolves, `false` when the
|
|
446
486
|
* signal aborted before it did (already at the call, or during the wait). A rejection of
|
|
@@ -663,8 +703,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
663
703
|
*
|
|
664
704
|
* Cleared by {@link cleanup} only, deliberately NOT by {@link detachStrand}: the watcher's
|
|
665
705
|
* filter rejection stops a replica through that very method, and forgetting the id there
|
|
666
|
-
* would have the next
|
|
667
|
-
* relaunch the replica —
|
|
706
|
+
* would have the next watcher run find it unknown again, re-admit the strand provisionally
|
|
707
|
+
* and relaunch the replica — on every other run, forever. A strand id's sApp never changes,
|
|
668
708
|
* and the map is bounded by the strands this party has published, like
|
|
669
709
|
* {@link discoveredStrands}.
|
|
670
710
|
*/
|
|
@@ -695,19 +735,24 @@ export class CadreNode implements SAppIdLookup {
|
|
|
695
735
|
* wait that ends in mutual denial (the upstream gate is fail-closed on
|
|
696
736
|
* timeout, unlike the fail-open connection gater). Optimystic's predicate is
|
|
697
737
|
* synchronous by contract. So the set is refreshed OUT OF BAND instead — after
|
|
698
|
-
* start,
|
|
699
|
-
*
|
|
700
|
-
*
|
|
701
|
-
*
|
|
738
|
+
* start, at the head of every {@link reconcileControlCohort} pass, whenever a
|
|
739
|
+
* membership table changes (the gate watch {@link start} takes through
|
|
740
|
+
* `ControlDatabase.watchMembershipTables`, which hears a local commit and a change
|
|
741
|
+
* replicated into this node's storage alike, within a 50 ms window), and immediately
|
|
742
|
+
* after every LOCAL membership mutation so a just-vouched peer is admitted
|
|
743
|
+
* before the writer's call returns. That last refresh is AUTOMATIC: the
|
|
744
|
+
* control database notifies {@link refreshMembershipGate} after every
|
|
702
745
|
* committed `CadrePeer` write (see `ControlDatabase.mutateCadrePeer`), so no
|
|
703
|
-
* writer has to remember. A change
|
|
704
|
-
*
|
|
705
|
-
*
|
|
706
|
-
*
|
|
707
|
-
*
|
|
708
|
-
*
|
|
709
|
-
*
|
|
710
|
-
*
|
|
746
|
+
* writer has to remember. A change this node's storage never hears of — a
|
|
747
|
+
* machine outside the block's cohort on a party larger than the replication
|
|
748
|
+
* breadth — is picked up at the pass's 2-minute backstop: bounded staleness,
|
|
749
|
+
* acceptable because this gate is defense in depth: rows an unadmitted peer
|
|
750
|
+
* manages to write are still disbelieved at read time by the voucher-anchored
|
|
751
|
+
* predicate. For cohort choice the same window means a member whose row
|
|
752
|
+
* arrived by replication is left out of this node's control-block cohorts
|
|
753
|
+
* until that refresh. Optimystic reads the predicate live on every lookup, so
|
|
754
|
+
* replacing this set is the whole re-evaluation; there is no per-peer
|
|
755
|
+
* reclassify call to make.
|
|
711
756
|
*/
|
|
712
757
|
private authorizedControlPeers: Set<string> = new Set();
|
|
713
758
|
|
|
@@ -771,9 +816,9 @@ export class CadreNode implements SAppIdLookup {
|
|
|
771
816
|
* When this node last ANNOUNCED a delegate, keyed `<targetPeerId>\n<strandId>`
|
|
772
817
|
* (the peer announced TO, not the delegate). Throttles
|
|
773
818
|
* {@link refreshDelegateGrants} to once per `DELEGATE_GRANT_TTL_MS / 2` per
|
|
774
|
-
* (relay, strand) so
|
|
775
|
-
* chatter; the launch/resume announce passes record here too.
|
|
776
|
-
* strand is no longer running are pruned on each reconcile pass.
|
|
819
|
+
* (relay, strand) so a run of change-driven reconcile passes never becomes
|
|
820
|
+
* per-pass RPC chatter; the launch/resume announce passes record here too.
|
|
821
|
+
* Keys whose strand is no longer running are pruned on each reconcile pass.
|
|
777
822
|
*/
|
|
778
823
|
private readonly delegateAnnounceAt = new Map<string, number>();
|
|
779
824
|
|
|
@@ -802,12 +847,13 @@ export class CadreNode implements SAppIdLookup {
|
|
|
802
847
|
* strand, keyed `peerStrandKey(siblingControlPeerId, strandId)` → epoch ms; a
|
|
803
848
|
* missing key is due. Bounds {@link refreshStrandPeerAddrs}'s fan-out per
|
|
804
849
|
* (sibling, strand) rather than per strand, so a sibling that connects after a
|
|
805
|
-
* pass is asked on the
|
|
806
|
-
* sibling's answer set. An answer (even an empty one) makes the sibling
|
|
807
|
-
* in {@link STRAND_PEER_ADDR_REFRESH_MS}; no answer, in
|
|
808
|
-
* {@link STRAND_PEER_ADDR_RETRY_MS}
|
|
809
|
-
*
|
|
810
|
-
*
|
|
850
|
+
* pass is asked on the pass its connection starts instead of waiting out a stamp
|
|
851
|
+
* another sibling's answer set. An answer (even an empty one) makes the sibling
|
|
852
|
+
* due again in {@link STRAND_PEER_ADDR_REFRESH_MS}; no answer, in
|
|
853
|
+
* {@link STRAND_PEER_ADDR_RETRY_MS}, and each pass arms its own wake for the
|
|
854
|
+
* earliest due time it leaves behind (`nextReconcileWakeMs`). Keys whose strand
|
|
855
|
+
* stopped running or whose sibling is no longer connected are pruned every pass,
|
|
856
|
+
* so a resumed strand and a reconnected sibling are both asked at once.
|
|
811
857
|
*/
|
|
812
858
|
private readonly strandAddrAskDueAt = new Map<string, number>();
|
|
813
859
|
|
|
@@ -893,6 +939,19 @@ export class CadreNode implements SAppIdLookup {
|
|
|
893
939
|
*/
|
|
894
940
|
private readonly pendingMembershipInvites = new Map<string, StrandMembershipInvite>();
|
|
895
941
|
|
|
942
|
+
/**
|
|
943
|
+
* The party key {@link adoptFormationMembershipInvite} just seated or read, by strand id,
|
|
944
|
+
* so the launch that follows a join does not read the `StrandPartyKey` row again. Taken
|
|
945
|
+
* (deleted) by the {@link resolveStrandPartyKey} that uses it, so any later launch reads
|
|
946
|
+
* the row; dropped by {@link detachStrand}. The joiner's counterpart of the founder's
|
|
947
|
+
* `StrandConfig.partyMemberPrivateKey`, without the app carrying the secret.
|
|
948
|
+
*
|
|
949
|
+
* NOTE: an entry is not dropped when the row is removed, because no `CadreNode` path removes
|
|
950
|
+
* it; an embedder calling `getControlDatabase().deleteStrandPartyKey` between a join and its
|
|
951
|
+
* launch would get the deleted key. If a node-level rotation is added, drop the entry there.
|
|
952
|
+
*/
|
|
953
|
+
private readonly formedPartyKeys = new Map<string, string>();
|
|
954
|
+
|
|
896
955
|
/**
|
|
897
956
|
* Invite keys of the membership invitations this process has staged, by
|
|
898
957
|
* {@link adoptFormationMembershipInvite} or from a `JoinSuccess` row
|
|
@@ -992,7 +1051,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
992
1051
|
|
|
993
1052
|
/**
|
|
994
1053
|
* This session's joined strands: {@link joinedStrandStore} and the party-wide
|
|
995
|
-
* `JoinedStrand` table, unioned with the control rows each watcher
|
|
1054
|
+
* `JoinedStrand` table, unioned with the control rows each watcher run, published and
|
|
996
1055
|
* removed by the reconcile pass. Per SESSION, unlike the store — rebuilt at every
|
|
997
1056
|
* {@link start}, which is what ends a revoked join's "keep offering until this session
|
|
998
1057
|
* ends" (see `JoinedStrandSession.forgetAfterThisSession`).
|
|
@@ -1023,19 +1082,43 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1023
1082
|
/** TTL heartbeat that re-publishes the self record before it goes stale. */
|
|
1024
1083
|
private recordRefreshTimer: ReturnType<typeof setInterval> | null = null;
|
|
1025
1084
|
/** Listener that re-publishes the self record when reachable addresses change. */
|
|
1026
|
-
private selfPeerUpdateHandler: (() => void) | null = null;
|
|
1027
|
-
/**
|
|
1028
|
-
*
|
|
1029
|
-
*
|
|
1030
|
-
*
|
|
1031
|
-
* {@link
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
*
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1085
|
+
private selfPeerUpdateHandler: ((event: CustomEvent<PeerUpdate>) => void) | null = null;
|
|
1086
|
+
/**
|
|
1087
|
+
* The reconcile pass's schedule (see {@link reconcileControlCohort}): single-flight
|
|
1088
|
+
* runs on `table-change-watch.ts`'s rule, with no table subscription of its own — the
|
|
1089
|
+
* gate watch ({@link membershipGateWatch}), the control connection listeners,
|
|
1090
|
+
* `self:peer:update`, and the pass's own wake ({@link armReconcileWake}) `trigger` it,
|
|
1091
|
+
* and its backstop (`network.controlCohort.reconcileMs`, default
|
|
1092
|
+
* `DEFAULT_CONTROL_COHORT_RECONCILE_MS`) runs it when nothing has. Built alongside
|
|
1093
|
+
* {@link recordRefreshTimer} in {@link startRecordRefresh}, about 1 s after start, and
|
|
1094
|
+
* disposed in {@link stopRecordRefresh}; its timers never keep the loop alive.
|
|
1095
|
+
*/
|
|
1096
|
+
private controlReconcileRuns: TableChangeWatch | null = null;
|
|
1097
|
+
/**
|
|
1098
|
+
* The subscription on the membership tables (`ControlDatabase.watchMembershipTables`) that
|
|
1099
|
+
* re-materializes the membership gate and triggers {@link controlReconcileRuns} on every
|
|
1100
|
+
* change ({@link onMembershipGateTablesChanged}).
|
|
1101
|
+
* Taken in {@link start} right after the membership listener is wired, disposed in
|
|
1102
|
+
* {@link cleanup} before the control database closes.
|
|
1103
|
+
*/
|
|
1104
|
+
private membershipGateWatch: TableChangeWatch | null = null;
|
|
1105
|
+
/** The second membership-view drop after a sibling connects ({@link armMembershipViewReload}); cleared in {@link stopRecordRefresh}. */
|
|
1106
|
+
private membershipViewReloadTimer: ReturnType<typeof setTimeout> | null = null;
|
|
1107
|
+
/**
|
|
1108
|
+
* Consecutive passes whose dial of a selected sibling or a cold-start bootstrap peer
|
|
1109
|
+
* failed (`ControlCohortPassOutcome.unconnected`), for the redial ladder
|
|
1110
|
+
* ({@link armReconcileWake}); 0 after a pass none of whose dials did. Per session:
|
|
1111
|
+
* reset in {@link stopRecordRefresh}.
|
|
1112
|
+
*/
|
|
1113
|
+
private reconcileRedialStreak = 0;
|
|
1114
|
+
/** Peers a {@link redialClaimedNode} loop is running for, so a repeated claim starts no second one. */
|
|
1115
|
+
private readonly claimRedialPeers = new Set<string>();
|
|
1116
|
+
/**
|
|
1117
|
+
* Single-flight guard for {@link reconcileControlCohort}. An explicit call, the
|
|
1118
|
+
* `self:peer:update` trigger and a scheduled run can fire close together; collapsing
|
|
1119
|
+
* concurrent passes into one in-flight run prevents two passes from double-dialing the
|
|
1120
|
+
* same siblings (mirrors {@link registerSelfInFlight}). A scheduled run waits for this
|
|
1121
|
+
* pass and then starts its own rather than joining it.
|
|
1039
1122
|
*/
|
|
1040
1123
|
private reconcileControlCohortInFlight: Promise<ControlCohortReconcileResult> | null = null;
|
|
1041
1124
|
/**
|
|
@@ -1058,20 +1141,28 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1058
1141
|
* machine in the party is already discarding this node's address as stale.
|
|
1059
1142
|
*/
|
|
1060
1143
|
private lastSelfRecordPublishAt: number | null = null;
|
|
1144
|
+
/**
|
|
1145
|
+
* The newest record this process signed for its own `CadrePeer` row: written by either path
|
|
1146
|
+
* of {@link publishSelfRecord}, or handed to a member to write by {@link redeemCadreInvitation}.
|
|
1147
|
+
* What {@link publishSelfRecordUnlessCurrent} compares the row against. Per session, cleared
|
|
1148
|
+
* with {@link lastSelfRecordPublishAt}: a row carried over from an earlier process is never
|
|
1149
|
+
* taken as current.
|
|
1150
|
+
*/
|
|
1151
|
+
private lastSignedSelfRecord: PeerAddressRecord | null = null;
|
|
1061
1152
|
/** Say-once latch for {@link escalateIfSelfRecordStale}; re-armed by the next publish. */
|
|
1062
1153
|
private selfRecordStaleWarned = false;
|
|
1063
1154
|
|
|
1064
1155
|
// ── Write-while-alone re-replication (control-write-ensure-replicated) ──────
|
|
1065
1156
|
/**
|
|
1066
1157
|
* Re-replication queue for owner `authorizePeer` writes that committed while
|
|
1067
|
-
* this node was alone
|
|
1068
|
-
*
|
|
1069
|
-
*
|
|
1158
|
+
* this node was alone ({@link committedAlone}) — the Optimystic commit was
|
|
1159
|
+
* local-only (the block's cohort was this node) and never broadcast. Maps the
|
|
1160
|
+
* affected subject peerId → 'authorize', to
|
|
1070
1161
|
* re-issue once the cohort grows. Removals are NOT tracked here — a delete
|
|
1071
1162
|
* leaves a `Revocation` tombstone, tracked by {@link pendingRevocations}; a
|
|
1072
1163
|
* removePeer also clears any queued authorize for the same subject (the row is
|
|
1073
|
-
* gone, so re-issuing the insert would be wrong). Drained
|
|
1074
|
-
*
|
|
1164
|
+
* gone, so re-issuing the insert would be wrong). Drained when a sibling becomes
|
|
1165
|
+
* reachable ({@link updateControlSiblingReachability}) by {@link drainPendingControlReplication}.
|
|
1075
1166
|
*/
|
|
1076
1167
|
private pendingPeerWrites: Map<string, 'authorize'> = new Map();
|
|
1077
1168
|
/**
|
|
@@ -1095,14 +1186,6 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1095
1186
|
* exactly a "removals from before this lifetime" case).
|
|
1096
1187
|
*/
|
|
1097
1188
|
private reissuedHeldRevocations = false;
|
|
1098
|
-
/**
|
|
1099
|
-
* Whether this process has seen the singleton `Revocation` ledger marker filed — its own
|
|
1100
|
-
* `'opened'` or an `'already-open'` answer, in {@link openRevocationLedgerIfDue}. A
|
|
1101
|
-
* record of work done, not a cached authorization answer: the marker can never be
|
|
1102
|
-
* deleted (`NoDelete`), so this can only go stale if the database itself is replaced,
|
|
1103
|
-
* which is why stop() clears it with the other per-process replication state.
|
|
1104
|
-
*/
|
|
1105
|
-
private revocationLedgerOpened = false;
|
|
1106
1189
|
/** This node's own `CadrePeer` self-write committed local-only (re-touched on growth). */
|
|
1107
1190
|
private pendingSelfPeerWrite = false;
|
|
1108
1191
|
/** This node's own `DeviceToken` self-write committed local-only (re-touched on growth). */
|
|
@@ -1117,15 +1200,25 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1117
1200
|
/** Single-flight guard for the re-replication drain (mirrors {@link registerSelfInFlight}). */
|
|
1118
1201
|
private drainControlReplicationInFlight: Promise<void> | null = null;
|
|
1119
1202
|
/**
|
|
1120
|
-
*
|
|
1121
|
-
*
|
|
1122
|
-
*
|
|
1203
|
+
* Connected control peers whose identify says they serve the control database
|
|
1204
|
+
* ({@link onControlPeerIdentified}): the peers Optimystic can count into a control block's
|
|
1205
|
+
* cohort, before its membership check. A peer joins on its first `peer:identify` of a
|
|
1206
|
+
* connected period, not on `connection:open`: until it identifies, every control block's
|
|
1207
|
+
* cohort is this node alone. It leaves when its last connection closes. Read through
|
|
1208
|
+
* {@link hasReachableControlSibling}, which applies the membership check.
|
|
1123
1209
|
*/
|
|
1124
|
-
private
|
|
1125
|
-
/**
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1210
|
+
private readonly identifiedControlPeers = new Set<string>();
|
|
1211
|
+
/**
|
|
1212
|
+
* {@link hasReachableControlSibling} at its last evaluation, so the drain fires on the
|
|
1213
|
+
* false→true edge only (the earliest point a re-issue reaches the cohort), not on every later
|
|
1214
|
+
* identify. Re-armed to false when no reachable sibling remains
|
|
1215
|
+
* ({@link updateControlSiblingReachability}).
|
|
1216
|
+
*/
|
|
1217
|
+
private controlSiblingReachable = false;
|
|
1218
|
+
/** `peer:identify` listener: {@link onControlPeerIdentified} (teardown in {@link stopRecordRefresh}). */
|
|
1219
|
+
private controlPeerIdentifyHandler: ((event: CustomEvent<IdentifyResult>) => void) | null = null;
|
|
1220
|
+
/** `connection:close` listener: {@link onControlConnectionClosed} (teardown in {@link stopRecordRefresh}). */
|
|
1221
|
+
private controlConnectionCloseHandler: ((event: CustomEvent<Connection>) => void) | null = null;
|
|
1129
1222
|
|
|
1130
1223
|
constructor(config: CadreNodeConfig) {
|
|
1131
1224
|
this.config = config;
|
|
@@ -1318,7 +1411,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1318
1411
|
// loaded before the first reconcile pass could consult them.
|
|
1319
1412
|
this.initializeBootstrapPeerStore();
|
|
1320
1413
|
|
|
1321
|
-
// The remembered cross-party joins, which the strand watcher built below
|
|
1414
|
+
// The remembered cross-party joins, which the strand watcher built below reads
|
|
1322
1415
|
// beside the control rows. Before network bring-up, so a mis-scoped injected store
|
|
1323
1416
|
// fails closed like the two above.
|
|
1324
1417
|
this.initializeJoinedStrandStore();
|
|
@@ -1404,6 +1497,21 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1404
1497
|
// rest of start() and the refresh early-returns while it is — a write in that
|
|
1405
1498
|
// window is covered by start's own pass below.
|
|
1406
1499
|
this.controlDatabase.setMembershipChangeListener((reason) => this.refreshMembershipGate(reason));
|
|
1500
|
+
// The gate watch: a change to any membership table, replicated in or committed here,
|
|
1501
|
+
// re-materializes the gate and starts a reconcile pass. Separate from the pass so a
|
|
1502
|
+
// replicated revocation does not wait behind a pass spending its dial budget on an
|
|
1503
|
+
// unreachable sibling. The watch handler only arms a timer, so nothing runs under the
|
|
1504
|
+
// write lock the membership listener is called with; `onChange` runs from that timer.
|
|
1505
|
+
//
|
|
1506
|
+
// NOTE: a table this `Database` has not yet touched hears local commits only
|
|
1507
|
+
// (`table-change-watch.ts`). The start refresh reads `CadrePeer`, `OwnerKey` and
|
|
1508
|
+
// `Revocation`; the four invitation and grant tables are first read when some row is
|
|
1509
|
+
// admitted through one, and a load the committed-read path served is not kept and
|
|
1510
|
+
// arms no bridge, so a chain row that replicates in before then waits for the next
|
|
1511
|
+
// change on a touched table or the backstop. Nothing depends on a chain row until a
|
|
1512
|
+
// `CadrePeer` or `OwnerKey` row refers to it, and that row is itself such a change.
|
|
1513
|
+
this.membershipGateWatch = this.controlDatabase.watchMembershipTables(
|
|
1514
|
+
() => this.onMembershipGateTablesChanged(), { label: 'membership-gate' });
|
|
1407
1515
|
// Committed-delete seam → write-while-alone revocation queue. Same wiring
|
|
1408
1516
|
// window argument as the membership listener above.
|
|
1409
1517
|
this.controlDatabase.setGuardedDeleteListener((revocation) => this.noteGuardedDelete(revocation));
|
|
@@ -1423,7 +1531,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1423
1531
|
onStrandRemoved: async (strandId) => this.handleStrandRemoved(strandId)
|
|
1424
1532
|
},
|
|
1425
1533
|
this.config.strandFilter ?? { mode: 'all' },
|
|
1426
|
-
this.config.strandWatchInterval ??
|
|
1534
|
+
this.config.strandWatchInterval ?? DEFAULT_STRAND_WATCH_BACKSTOP_MS,
|
|
1427
1535
|
this // CadreNode implements SAppIdLookup
|
|
1428
1536
|
);
|
|
1429
1537
|
|
|
@@ -1506,8 +1614,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1506
1614
|
log('CadreNode started successfully');
|
|
1507
1615
|
|
|
1508
1616
|
// Wire the control-connection growth listeners immediately (not via the
|
|
1509
|
-
// delayed refresh path) so no early `
|
|
1510
|
-
// write-while-alone re-replication drain must fire
|
|
1617
|
+
// delayed refresh path) so no early `peer:identify` is missed — the
|
|
1618
|
+
// write-while-alone re-replication drain must fire when the first sibling is reachable.
|
|
1511
1619
|
this.wireControlConnectionListeners();
|
|
1512
1620
|
|
|
1513
1621
|
// Arm the control network's peer-join block catch-up now that the control
|
|
@@ -1898,7 +2006,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1898
2006
|
);
|
|
1899
2007
|
}
|
|
1900
2008
|
this.joinedStrandStore = store;
|
|
1901
|
-
this.joinedStrands = new JoinedStrandSession(store, this.createPartyJoinedStrandLedger()
|
|
2009
|
+
this.joinedStrands = new JoinedStrandSession(store, this.createPartyJoinedStrandLedger(),
|
|
2010
|
+
() => this.strandWatcher?.refresh());
|
|
1902
2011
|
}
|
|
1903
2012
|
|
|
1904
2013
|
/**
|
|
@@ -2054,8 +2163,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2054
2163
|
|
|
2055
2164
|
/**
|
|
2056
2165
|
* Arm the CONTROL network's peer-join block catch-up: push every block in the
|
|
2057
|
-
* control database's own raw store to each
|
|
2058
|
-
*
|
|
2166
|
+
* control database's own raw store to each AUTHORIZED member on each
|
|
2167
|
+
* (re)connection, skipping what it is known to hold. This is what physically replicates
|
|
2059
2168
|
* control blocks committed while the writer was alone — the named
|
|
2060
2169
|
* collection-header blocks written once at genesis above all, whose revision
|
|
2061
2170
|
* never moves again, so no later commit ever carries them to a member that
|
|
@@ -2064,7 +2173,9 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2064
2173
|
* for peers it knew about before the restart).
|
|
2065
2174
|
*
|
|
2066
2175
|
* Unlike the per-strand instances (see `strand-instance-manager.ts`), pushes
|
|
2067
|
-
* are gated on {@link
|
|
2176
|
+
* are gated on the authorized-member snapshot ({@link authorizedControlPeers},
|
|
2177
|
+
* which {@link refreshAuthorizedControlPeers} materializes from the same rows and
|
|
2178
|
+
* rule {@link isAuthorizedMember} queries), judged at push time: the control
|
|
2068
2179
|
* network's inbound connection gate deliberately admits non-members in
|
|
2069
2180
|
* several states (seed delivery to an un-enrolled node, an open enrollment
|
|
2070
2181
|
* window, an outstanding invitation, configured bootstrap/relay peers), and
|
|
@@ -2080,23 +2191,19 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2080
2191
|
* The one non-member the gate admits is a device whose invitation
|
|
2081
2192
|
* redemption this member is admitting ({@link admitRedeemingDevice}).
|
|
2082
2193
|
*
|
|
2083
|
-
* No-ops (logged) when
|
|
2084
|
-
*
|
|
2085
|
-
* no way to dial.
|
|
2194
|
+
* No-ops (logged) when {@link controlBackfillStorage} answers nothing (disabled,
|
|
2195
|
+
* or no control storage configured) or the node exposes no key network — the
|
|
2196
|
+
* backfill would have nothing to read or no way to dial.
|
|
2086
2197
|
*/
|
|
2087
2198
|
private startControlBackfill(): void {
|
|
2088
|
-
if (
|
|
2199
|
+
if (!this.controlNode) {
|
|
2089
2200
|
return;
|
|
2090
2201
|
}
|
|
2091
|
-
|
|
2092
|
-
// idempotent (it returns the same wrapped store `buildControlNodeOptions` already
|
|
2093
|
-
// resolved), so the catch-up cannot be silently disarmed by a future reordering that
|
|
2094
|
-
// moves this call ahead of the one that populated the field.
|
|
2095
|
-
const storage = this.resolveControlStorage();
|
|
2202
|
+
const storage = this.controlBackfillStorage();
|
|
2096
2203
|
const keyNetwork = (this.controlNode as Libp2pNodeWithRepo).keyNetwork;
|
|
2097
2204
|
if (!storage || !keyNetwork) {
|
|
2098
2205
|
log('Control peer-join block catch-up is inert: %s',
|
|
2099
|
-
!storage ? 'no control storage configured' : 'control node exposes no keyNetwork');
|
|
2206
|
+
!storage ? 'disabled, or no control storage configured' : 'control node exposes no keyNetwork');
|
|
2100
2207
|
return;
|
|
2101
2208
|
}
|
|
2102
2209
|
const networkName = this.controlNetworkName();
|
|
@@ -2109,7 +2216,11 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2109
2216
|
// under — derived from the same networkName binding the node options
|
|
2110
2217
|
// used, never re-spelled here.
|
|
2111
2218
|
protocolPrefix: `/optimystic/${networkName}`,
|
|
2112
|
-
|
|
2219
|
+
reportBlockHeld: (this.controlNode as Libp2pNodeWithRepo).reportBlockHeld,
|
|
2220
|
+
// The membership snapshot, not {@link isControlNetworkMember}: that predicate admits
|
|
2221
|
+
// everyone while the snapshot is empty and admits bootstrap/relay peers outright, and
|
|
2222
|
+
// neither may receive the control store.
|
|
2223
|
+
authorizePeer: async (peerId) => this.admittingDevices.has(peerId) || this.authorizedControlPeers.has(peerId)
|
|
2113
2224
|
}, {
|
|
2114
2225
|
// A shorter settle window than the strand default (1000 ms): the control
|
|
2115
2226
|
// store is small (a party's membership — dozens of blocks), a re-push is
|
|
@@ -2165,8 +2276,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2165
2276
|
const release = this.countAdmittingDevice(peerId);
|
|
2166
2277
|
try {
|
|
2167
2278
|
const result = await backfill.forceCatchUpPeer(peerIdFromString(peerId));
|
|
2168
|
-
log('Device %s caught up before its admission: offered=%d accepted=%d rejected=%d',
|
|
2169
|
-
peerId, result.offered, result.accepted, result.rejected.length);
|
|
2279
|
+
log('Device %s caught up before its admission: offered=%d alreadyHeld=%d accepted=%d rejected=%d',
|
|
2280
|
+
peerId, result.offered, result.alreadyHeld, result.accepted, result.rejected.length);
|
|
2170
2281
|
return release;
|
|
2171
2282
|
} catch (error) {
|
|
2172
2283
|
release();
|
|
@@ -2230,6 +2341,20 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2230
2341
|
return this.controlStorage;
|
|
2231
2342
|
}
|
|
2232
2343
|
|
|
2344
|
+
/**
|
|
2345
|
+
* The store the control network's peer-join block catch-up reads, or `undefined` when this
|
|
2346
|
+
* runtime does not arm it: `controlBackfill.enabled: false`, or no control storage. Both of
|
|
2347
|
+
* its readers ask here so they cannot disagree: {@link startControlBackfill} arms on it, and
|
|
2348
|
+
* {@link buildControlNodeOptions} turns Optimystic's rebalance reaction off on it.
|
|
2349
|
+
*
|
|
2350
|
+
* Through the resolver, not the {@link controlStorage} memo field: the resolver is
|
|
2351
|
+
* idempotent, so the catch-up cannot be silently disarmed by a future reordering that moves
|
|
2352
|
+
* a read ahead of the one that populated the field.
|
|
2353
|
+
*/
|
|
2354
|
+
private controlBackfillStorage(): IRawStorage | undefined {
|
|
2355
|
+
return this.config.controlBackfill?.enabled === false ? undefined : this.resolveControlStorage();
|
|
2356
|
+
}
|
|
2357
|
+
|
|
2233
2358
|
/**
|
|
2234
2359
|
* Warn the operator when `network.announceAddrs` will silently cost this node the
|
|
2235
2360
|
* relay reachability it also configured.
|
|
@@ -2337,6 +2462,19 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2337
2462
|
cohortQueryTimeoutMs: declaredCohortReadDeadlineMs(network)
|
|
2338
2463
|
}),
|
|
2339
2464
|
arachnode: { enableRingZulu: profile === 'storage' },
|
|
2465
|
+
// One join catch-up per node, for the strand node's reason (`strand-instance-manager.ts`,
|
|
2466
|
+
// at its `rebalance` option): Optimystic's rebalance growth arm would push a joining
|
|
2467
|
+
// member each control block again, one request per block. Every peer the growth arm
|
|
2468
|
+
// pushes to, the control catch-up pushes to as well, except the ones it deliberately
|
|
2469
|
+
// withholds the control store from: the cohort predicate (`isNetworkMember` below) admits
|
|
2470
|
+
// everyone while the membership snapshot is empty, and bootstrap/relay peers outright.
|
|
2471
|
+
// With the catch-up disabled the reaction stays on.
|
|
2472
|
+
//
|
|
2473
|
+
// NOTE: no ring shifts and no reclaimable shed blocks on this node either, unused today.
|
|
2474
|
+
// If storage nodes ever need them, turn the reaction back on: the catch-up already reports
|
|
2475
|
+
// its accepted pushes to the rebalance monitor (`reportBlockHeld`); measure whether the
|
|
2476
|
+
// growth arm still re-sends.
|
|
2477
|
+
...(this.controlBackfillStorage() && { rebalance: { enabled: false } }),
|
|
2340
2478
|
...(identityKey && { privateKey: identityKey }),
|
|
2341
2479
|
// With `network.relayAddrs` set, built with libp2p's relay discovery off (given a
|
|
2342
2480
|
// transport list to wrap — `node-transports.ts`), so the supervisor
|
|
@@ -2405,10 +2543,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2405
2543
|
// Who this node SENDS control writes to (cohort and coordinator choice), where
|
|
2406
2544
|
// `authorizeInboundStream` is who may send to it. Without it Optimystic picks cohort
|
|
2407
2545
|
// members from the protocols a peer's peer-store record lists, which any connected
|
|
2408
|
-
// peer can fill.
|
|
2409
|
-
|
|
2410
|
-
// whose verified redemption is being written, which must hold the rows that admit it.
|
|
2411
|
-
isNetworkMember: (peerId) => this.isControlNetworkMember(peerId) || this.admittingDevices.has(peerId)
|
|
2546
|
+
// peer can fill.
|
|
2547
|
+
isNetworkMember: (peerId) => this.isControlCohortMember(peerId)
|
|
2412
2548
|
};
|
|
2413
2549
|
|
|
2414
2550
|
return nodeOptions;
|
|
@@ -2492,8 +2628,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2492
2628
|
* (it redeems here by seating the row from its bundle, after which check 7
|
|
2493
2629
|
* admits) or from a sibling whose membership row is still replicating (check 5
|
|
2494
2630
|
* admits once it lands; if it has not landed by the deadline, the connection
|
|
2495
|
-
* is closed and
|
|
2496
|
-
* dials are never gated). The protocol guard refuses all of them every
|
|
2631
|
+
* is closed and the reconcile pass the close starts on either side re-forms it,
|
|
2632
|
+
* since outbound dials are never gated). The protocol guard refuses all of them every
|
|
2497
2633
|
* members-only protocol meanwhile. A circuit-relay reservation admitted by
|
|
2498
2634
|
* {@link admitControlRelayReservation} disarms the deadline (see
|
|
2499
2635
|
* `membership-connection-gater.ts` → "The relay-reservation seam").
|
|
@@ -2724,6 +2860,18 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2724
2860
|
return this.authorizedControlPeers.size === 0 || this.authorizedControlPeers.has(remotePeerId);
|
|
2725
2861
|
}
|
|
2726
2862
|
|
|
2863
|
+
/**
|
|
2864
|
+
* Whether this node sends control writes to `remotePeerId`: Optimystic's `isNetworkMember`
|
|
2865
|
+
* for the control node (cohort and coordinator choice), and the membership half of
|
|
2866
|
+
* {@link hasReachableControlSibling}. The stream gate's rule ({@link isControlNetworkMember}),
|
|
2867
|
+
* unlogged, because Optimystic asks several times per peer per cohort assembly and counts
|
|
2868
|
+
* refusals on its own debug line; plus a device whose verified redemption is being written
|
|
2869
|
+
* ({@link admittingDevices}), which must hold the rows that admit it.
|
|
2870
|
+
*/
|
|
2871
|
+
private isControlCohortMember(remotePeerId: string): boolean {
|
|
2872
|
+
return this.isControlNetworkMember(remotePeerId) || this.admittingDevices.has(remotePeerId);
|
|
2873
|
+
}
|
|
2874
|
+
|
|
2727
2875
|
/**
|
|
2728
2876
|
* Per-stream authorization for the protocol guard's `members-snapshot` class
|
|
2729
2877
|
* (`control-protocol-guard.ts`): the four Optimystic control-DB protocols
|
|
@@ -2765,7 +2913,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2765
2913
|
* this listener with its write lock HELD (`notifyMembershipChanged`), so a retrying
|
|
2766
2914
|
* read here would sleep its backoff holding the lock and stall every other local
|
|
2767
2915
|
* writer. Nothing is lost — this refresh already keeps the previous snapshot on
|
|
2768
|
-
* failure and is re-driven by the next membership
|
|
2916
|
+
* failure and is re-driven by the next membership change (local or replicated, through
|
|
2917
|
+
* the gate watch) and by the reconcile pass's backstop.
|
|
2769
2918
|
*
|
|
2770
2919
|
* The trusted-owner anchor sync ({@link syncOwnerAnchor}) runs FIRST, so the snapshot
|
|
2771
2920
|
* built here judges `CadrePeer` rows against the anchor as the `OwnerKey` table now
|
|
@@ -2840,15 +2989,20 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2840
2989
|
// commit is what would authorize it), and the gate re-judges at push time.
|
|
2841
2990
|
//
|
|
2842
2991
|
// NOTE: every refresh re-arms every connected peer that is not yet caught
|
|
2843
|
-
// up
|
|
2844
|
-
// connection that is admitted but never authorized — a configured
|
|
2992
|
+
// up. A connection that is admitted but never authorized — a configured
|
|
2845
2993
|
// bootstrap or relay peer, the steady state for a NAT'd node — is
|
|
2846
|
-
// therefore re-judged on every
|
|
2847
|
-
//
|
|
2848
|
-
//
|
|
2849
|
-
//
|
|
2850
|
-
//
|
|
2994
|
+
// therefore re-judged on every refresh, forever: every local membership
|
|
2995
|
+
// write, every replicated membership change (the gate watch), and every
|
|
2996
|
+
// reconcile pass, down to the 2-minute backstop on an idle party. Each
|
|
2997
|
+
// such pass is one timer and a lookup in the snapshot just set (the gate
|
|
2998
|
+
// reads no control database). If a node ever holds many such connections,
|
|
2999
|
+
// or refreshes get frequent, skip re-arming peers whose last pass was
|
|
3000
|
+
// denied under the SAME membership snapshot instead of re-arming
|
|
3001
|
+
// unconditionally.
|
|
2851
3002
|
void this.controlBackfill?.scheduleConnectedPeers();
|
|
3003
|
+
// Same join order for the growth edge: a sibling that identified before this commit
|
|
3004
|
+
// authorized it becomes reachable here.
|
|
3005
|
+
this.updateControlSiblingReachability();
|
|
2852
3006
|
await this.hangUpRevokedControlPeers(rows, loadTombstones);
|
|
2853
3007
|
await this.noteOwnRemoval(rows, loadTombstones);
|
|
2854
3008
|
} catch (error) {
|
|
@@ -3035,9 +3189,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3035
3189
|
}
|
|
3036
3190
|
|
|
3037
3191
|
/**
|
|
3038
|
-
* What the strand watcher
|
|
3192
|
+
* What the strand watcher reads: this party's control rows plus a row per cross-party
|
|
3039
3193
|
* join (see {@link joinedStrands}), so a joined strand is offered and relaunched on
|
|
3040
|
-
* the same path as the party's own.
|
|
3194
|
+
* the same path as the party's own. It runs on a change to either table; the
|
|
3195
|
+
* machine-local joins wake it through {@link JoinedStrandSession}'s change callback.
|
|
3041
3196
|
*/
|
|
3042
3197
|
private createStrandQueryable(): StrandQueryable {
|
|
3043
3198
|
return {
|
|
@@ -3049,7 +3204,14 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3049
3204
|
log('Querying strands from control database');
|
|
3050
3205
|
const control = await this.controlDatabase.queryStrands();
|
|
3051
3206
|
return this.joinedStrands ? await this.joinedStrands.withControlRows(control) : control;
|
|
3052
|
-
}
|
|
3207
|
+
},
|
|
3208
|
+
watchChanges: (onChange, options) => {
|
|
3209
|
+
if (!this.controlDatabase) {
|
|
3210
|
+
throw new Error('CadreNode must initialize its control database before watching its strand tables');
|
|
3211
|
+
}
|
|
3212
|
+
return this.controlDatabase.watchTables(['CadreControl.Strand', 'CadreControl.JoinedStrand'], onChange, options);
|
|
3213
|
+
},
|
|
3214
|
+
lastReadPartial: () => this.joinedStrands?.lastReadPartial() ?? false
|
|
3053
3215
|
};
|
|
3054
3216
|
}
|
|
3055
3217
|
|
|
@@ -3065,10 +3227,6 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3065
3227
|
return this.controlDatabase;
|
|
3066
3228
|
};
|
|
3067
3229
|
return {
|
|
3068
|
-
// NOTE: one more control read per watcher poll (5 s), and for a party that never
|
|
3069
|
-
// joined anything it reads a never-written block, which consults the cohort. If poll
|
|
3070
|
-
// cost shows up in a device profile, cache the list and re-read it on the reconcile
|
|
3071
|
-
// cadence.
|
|
3072
3230
|
list: () => database().queryJoinedStrands(),
|
|
3073
3231
|
names: async (strandId) => (await database().queryStrand(strandId)) !== null
|
|
3074
3232
|
|| (await database().queryJoinedStrand(strandId)) !== null,
|
|
@@ -3155,26 +3313,29 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3155
3313
|
* after start (non-blocking). {@link registerSelf} is idempotent and safely
|
|
3156
3314
|
* no-ops when it cannot yet sign/insert (e.g. owner key not installed),
|
|
3157
3315
|
* so the timer is harmless even when registration only becomes possible later.
|
|
3316
|
+
* A record this process already put in place that is still current (a founding owner's
|
|
3317
|
+
* explicit publish, a redeemed invitation's admission, a restart's first-growth re-touch) is
|
|
3318
|
+
* not written again ({@link publishSelfRecordUnlessCurrent}). Whether the node is connected
|
|
3319
|
+
* yet does not matter: a device that redeems within a second of starting is.
|
|
3158
3320
|
*/
|
|
3159
3321
|
private scheduleSelfRegistration(): void {
|
|
3160
3322
|
this.selfRegistrationTimer = setTimeout(() => {
|
|
3161
3323
|
void (async () => {
|
|
3162
3324
|
try {
|
|
3163
|
-
await this.
|
|
3325
|
+
await this.publishSelfRecordUnlessCurrent();
|
|
3164
3326
|
} catch (error) {
|
|
3165
|
-
// Background task — a failed publish must not crash the node. A
|
|
3166
|
-
//
|
|
3167
|
-
//
|
|
3168
|
-
//
|
|
3327
|
+
// Background task — a failed publish must not crash the node. A publish that loses
|
|
3328
|
+
// its row to another writer — the owner's re-touch, or a restarted machine's own row
|
|
3329
|
+
// replicating in while the update runs — is re-run by the write funnel over the row
|
|
3330
|
+
// it finds (`ControlDatabase.updateSelfPeerRecord`); one that lost past the retry
|
|
3331
|
+
// budget fails here, and the next connected reconcile pass is the fallback
|
|
3169
3332
|
// (publishFirstSelfRecord).
|
|
3170
3333
|
log('Self-registration failed: %o', error);
|
|
3171
3334
|
}
|
|
3172
|
-
// Wire the ongoing refresh
|
|
3173
|
-
// an eager reconcile pass
|
|
3174
|
-
// interval was just armed inside startRecordRefresh).
|
|
3335
|
+
// Wire the ongoing refresh and the control-cohort reconcile schedule, then
|
|
3336
|
+
// run an eager reconcile pass now that the node has settled.
|
|
3175
3337
|
this.startRecordRefresh();
|
|
3176
|
-
|
|
3177
|
-
log('Control-cohort reconcile (start) failed: %o', error));
|
|
3338
|
+
this.controlReconcileRuns?.trigger(0);
|
|
3178
3339
|
})();
|
|
3179
3340
|
}, 1000); // Small delay to ensure node is fully started
|
|
3180
3341
|
// Node-only: don't keep the event loop alive solely for this timer.
|
|
@@ -3249,16 +3410,16 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3249
3410
|
return 'skipped';
|
|
3250
3411
|
}
|
|
3251
3412
|
// Collected only past that skip, which a node waiting for its row replicated in takes on
|
|
3252
|
-
//
|
|
3413
|
+
// reconcile pass made while a sibling is reachable ({@link publishFirstSelfRecord}).
|
|
3253
3414
|
const addrs = await this.collectSelfAddrs();
|
|
3254
3415
|
|
|
3255
3416
|
if (!existing && ownerService) {
|
|
3256
3417
|
// First-time row: requires an owner signature (the node is its own
|
|
3257
3418
|
// owner). insertSelfPeerRecord throws if no owner key is present.
|
|
3258
3419
|
const record = this.signSelfRecord(peerId, signingKey, addrs, null);
|
|
3420
|
+
const aloneBefore = this.committedAlone();
|
|
3259
3421
|
if (await ownerService.insertSelfPeerRecord(record)) {
|
|
3260
|
-
|
|
3261
|
-
this.noteSelfRecordPublished();
|
|
3422
|
+
this.noteSelfPeerWritten(record, aloneBefore);
|
|
3262
3423
|
log('registerSelf: inserted own CadrePeer record (owner-signed, updatedAt=%d, %d addrs, sig=%s…)', record.updatedAt, addrs.length, record.sig.slice(0, 16));
|
|
3263
3424
|
return 'inserted';
|
|
3264
3425
|
}
|
|
@@ -3273,44 +3434,102 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3273
3434
|
log('registerSelf: own CadrePeer row appeared mid-publish (concurrent authorize); self-updating to carry the signature');
|
|
3274
3435
|
}
|
|
3275
3436
|
|
|
3276
|
-
//
|
|
3277
|
-
//
|
|
3278
|
-
// the
|
|
3279
|
-
//
|
|
3280
|
-
//
|
|
3281
|
-
//
|
|
3282
|
-
|
|
3283
|
-
|
|
3284
|
-
|
|
3285
|
-
|
|
3286
|
-
if (!current) {
|
|
3437
|
+
// The update body reads the row under the write lock and signs over what it finds, on
|
|
3438
|
+
// every attempt — so neither `existing` (a read that predates this publish) nor a first
|
|
3439
|
+
// attempt that lost the row to another writer decides the `UpdatedAt` the strictly-greater
|
|
3440
|
+
// self-update rule is measured against, and a removal landing mid-publish re-reads as
|
|
3441
|
+
// absent instead of matching no rows under a 'refreshed'. The record it returns is the
|
|
3442
|
+
// one in the row.
|
|
3443
|
+
const aloneBefore = this.committedAlone();
|
|
3444
|
+
const record = await this.controlDatabase.updateSelfPeerRecord(peerId,
|
|
3445
|
+
(current) => this.signSelfRecord(peerId, signingKey, addrs, current));
|
|
3446
|
+
if (record === null) {
|
|
3287
3447
|
// Two causes, and the second is a steady state rather than a race: the row really
|
|
3288
3448
|
// vanished mid-publish (next refresh re-inserts), or this node was REVOKED — its row
|
|
3289
3449
|
// is physically present, so insertCadrePeer's in-lock existence check no-op'd above,
|
|
3290
|
-
// but
|
|
3291
|
-
//
|
|
3292
|
-
//
|
|
3450
|
+
// but the body reads it as absent because its stamp is retired. The revoked case
|
|
3451
|
+
// repeats every heartbeat and never re-inserts; that is intended (a revoked node has
|
|
3452
|
+
// nothing to publish), so this stays a skip, not an error.
|
|
3293
3453
|
log('registerSelf: own CadrePeer row not readable (vanished mid-publish, or this node is revoked and its stamp retired); skipping');
|
|
3294
3454
|
return 'skipped';
|
|
3295
3455
|
}
|
|
3296
|
-
|
|
3297
|
-
await this.controlDatabase.updateSelfPeerRecord(record);
|
|
3298
|
-
if (this.committedAlone()) this.pendingSelfPeerWrite = true;
|
|
3299
|
-
this.noteSelfRecordPublished();
|
|
3456
|
+
this.noteSelfPeerWritten(record, aloneBefore);
|
|
3300
3457
|
log('registerSelf: refreshed own CadrePeer record (updatedAt=%d, %d addrs, sig=%s…)', record.updatedAt, addrs.length, record.sig.slice(0, 16));
|
|
3301
3458
|
return 'refreshed';
|
|
3302
3459
|
}
|
|
3303
3460
|
|
|
3304
3461
|
/**
|
|
3305
|
-
*
|
|
3306
|
-
*
|
|
3307
|
-
*
|
|
3462
|
+
* Bookkeeping for a committed write of this node's own `CadrePeer` row by {@link publishSelfRecord}:
|
|
3463
|
+
* `record` is the one the row holds (for the update path, the one its body signed on the
|
|
3464
|
+
* attempt that committed, not the first it signed). Queued for the drain's re-touch
|
|
3465
|
+
* ({@link pendingSelfPeerWrite}) when no sibling was reachable when the write started or when
|
|
3466
|
+
* it ended: a sibling that identifies while the write is out may do so after the write's cohort
|
|
3467
|
+
* was chosen, and the first-growth drain skips a row holding {@link lastSignedSelfRecord} unless
|
|
3468
|
+
* this flag is set. Both fields are set here together so no write can leave the record without
|
|
3469
|
+
* the flag.
|
|
3308
3470
|
*/
|
|
3309
|
-
private
|
|
3310
|
-
this.
|
|
3471
|
+
private noteSelfPeerWritten(record: PeerAddressRecord, aloneBefore: boolean): void {
|
|
3472
|
+
if (aloneBefore || this.committedAlone()) this.pendingSelfPeerWrite = true;
|
|
3473
|
+
this.lastSignedSelfRecord = record;
|
|
3474
|
+
this.noteSelfRecordPublished();
|
|
3475
|
+
}
|
|
3476
|
+
|
|
3477
|
+
/**
|
|
3478
|
+
* Record that this node's own address record is in place as of `at`, and re-arm the
|
|
3479
|
+
* stale-record warning. Called for BOTH publishing paths of {@link publishSelfRecord}
|
|
3480
|
+
* ({@link noteSelfPeerWritten}), and by {@link publishSelfRecordUnlessCurrent} with the stamp
|
|
3481
|
+
* of a record already in place — a `skipped` publish wrote nothing and found nothing, so it
|
|
3482
|
+
* must not refresh the stamp.
|
|
3483
|
+
*/
|
|
3484
|
+
private noteSelfRecordPublished(at = Date.now()): void {
|
|
3485
|
+
this.lastSelfRecordPublishAt = at;
|
|
3311
3486
|
this.selfRecordStaleWarned = false;
|
|
3312
3487
|
}
|
|
3313
3488
|
|
|
3489
|
+
/**
|
|
3490
|
+
* {@link registerSelf}, unless the own row already holds the record this process last signed
|
|
3491
|
+
* ({@link lastSignedSelfRecord}) and that record is current ({@link isCurrentSelfRecord});
|
|
3492
|
+
* then nothing is written and the record counts as published at its own stamp. For a publish
|
|
3493
|
+
* that puts a record in place (the boot timer, the reconcile pass's first publish, the
|
|
3494
|
+
* first-growth drain), never the heartbeat, which exists to re-stamp freshness. `'current'`
|
|
3495
|
+
* is this method's own answer; {@link registerSelf} always writes.
|
|
3496
|
+
*/
|
|
3497
|
+
private async publishSelfRecordUnlessCurrent(): Promise<SelfRegistrationOutcome | 'current'> {
|
|
3498
|
+
if (this.registerSelfInFlight) {
|
|
3499
|
+
return this.registerSelfInFlight;
|
|
3500
|
+
}
|
|
3501
|
+
const record = this.lastSignedSelfRecord;
|
|
3502
|
+
if (record !== null && await this.isCurrentSelfRecord(record)) {
|
|
3503
|
+
this.noteSelfRecordPublished(record.updatedAt);
|
|
3504
|
+
log('registerSelf: own CadrePeer record already current (updatedAt=%d); not written again', record.updatedAt);
|
|
3505
|
+
return 'current';
|
|
3506
|
+
}
|
|
3507
|
+
return this.registerSelf();
|
|
3508
|
+
}
|
|
3509
|
+
|
|
3510
|
+
/**
|
|
3511
|
+
* Is `record` still the one to publish: signed less than half a heartbeat ago, with this
|
|
3512
|
+
* node's profile, over the addresses it would publish now, and the signature the own row
|
|
3513
|
+
* holds? The age bound keeps a record that is not re-written fresh for resolvers: at most half
|
|
3514
|
+
* a heartbeat old here and re-stamped by the heartbeat within one more, so never older than
|
|
3515
|
+
* 1.5 heartbeats (11.25 min) against {@link DEFAULT_PEER_RECORD_MAX_AGE_MS}. Cheap checks
|
|
3516
|
+
* first: the row comes from the membership view, the addresses from the app's resolver.
|
|
3517
|
+
*/
|
|
3518
|
+
private async isCurrentSelfRecord(record: PeerAddressRecord): Promise<boolean> {
|
|
3519
|
+
if (Date.now() - record.updatedAt >= DEFAULT_PEER_RECORD_HEARTBEAT_MS / 2 || record.profile !== this.config.profile) {
|
|
3520
|
+
return false;
|
|
3521
|
+
}
|
|
3522
|
+
const row = await this.controlDatabase?.queryPeerRecord(record.peerId);
|
|
3523
|
+
if (row?.sig !== record.sig) {
|
|
3524
|
+
return false;
|
|
3525
|
+
}
|
|
3526
|
+
// Sanitized, as the record a redemption carries is (`signRedeemRequest`); a published
|
|
3527
|
+
// record is not, so one whose list sanitizing would change (over 16 addresses) is never
|
|
3528
|
+
// current and is written as before.
|
|
3529
|
+
const addrs = sanitizeAddrs(await this.collectSelfAddrs());
|
|
3530
|
+
return addrs.length === record.addrs.length && addrs.every((addr, i) => addr === record.addrs[i]);
|
|
3531
|
+
}
|
|
3532
|
+
|
|
3314
3533
|
/**
|
|
3315
3534
|
* Report an abandoned control write to the embedding app.
|
|
3316
3535
|
*
|
|
@@ -3455,19 +3674,24 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3455
3674
|
* process does not learn it was removed ({@link noteOwnRemoval}), and pays that read on every
|
|
3456
3675
|
* pass.
|
|
3457
3676
|
*
|
|
3458
|
-
*
|
|
3459
|
-
*
|
|
3460
|
-
*
|
|
3461
|
-
* then never see rows written while it
|
|
3462
|
-
*
|
|
3463
|
-
*
|
|
3677
|
+
* Only while a sibling is reachable ({@link hasReachableControlSibling}): a row written
|
|
3678
|
+
* elsewhere arrives from a sibling, and an owner's publish committed alone is local-only and
|
|
3679
|
+
* forks its own history — a restarted owner whose embedder runs a pass before it reconnects
|
|
3680
|
+
* would write its row on the stale side of the fork, and then never see rows written while it
|
|
3681
|
+
* was down (`cadre-invite-any-member` runs that pass and fails every time without this
|
|
3682
|
+
* condition). A connection is not enough: until the sibling identifies, the write's cohort is
|
|
3683
|
+
* this node alone. The boot-time publish and the first-growth drain already try at the
|
|
3684
|
+
* node's lone and first-sibling moments; this retries on the passes after.
|
|
3685
|
+
*
|
|
3686
|
+
* A machine admitted by invitation finds its row already holding the record its redemption
|
|
3687
|
+
* carried, and writes nothing ({@link publishSelfRecordUnlessCurrent}).
|
|
3464
3688
|
*/
|
|
3465
3689
|
private async publishFirstSelfRecord(): Promise<void> {
|
|
3466
3690
|
if (this.lastSelfRecordPublishAt !== null || this.isAwaitingClaim() || this.removal !== null
|
|
3467
|
-
|| this.
|
|
3691
|
+
|| !this.hasReachableControlSibling()) {
|
|
3468
3692
|
return;
|
|
3469
3693
|
}
|
|
3470
|
-
const outcome = await this.
|
|
3694
|
+
const outcome = await this.publishSelfRecordUnlessCurrent().catch((error: unknown) => {
|
|
3471
3695
|
log('reconcileControlCohort: first self-record publish failed (continuing): %o', error);
|
|
3472
3696
|
return 'skipped' as const;
|
|
3473
3697
|
});
|
|
@@ -3506,17 +3730,24 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3506
3730
|
this.escalateIfSelfRecordStale(reason, error);
|
|
3507
3731
|
});
|
|
3508
3732
|
};
|
|
3509
|
-
const
|
|
3510
|
-
|
|
3511
|
-
|
|
3512
|
-
|
|
3733
|
+
const backstopMs = this.controlReconcileBackstopMs();
|
|
3734
|
+
// The pass's schedule. A run never joins a pass already in flight: that pass may have
|
|
3735
|
+
// read its rows before the change that woke this run, so joining it would drop the
|
|
3736
|
+
// change until the backstop. It waits for the pass, then starts its own.
|
|
3737
|
+
this.controlReconcileRuns = scheduleRuns(async () => {
|
|
3738
|
+
await this.reconcileControlCohortInFlight?.catch(() => undefined);
|
|
3739
|
+
await this.reconcileControlCohort().catch((error) =>
|
|
3740
|
+
log('Control-cohort reconcile (scheduled) failed: %o', error));
|
|
3741
|
+
}, { label: 'control-reconcile', backstopMs });
|
|
3513
3742
|
|
|
3514
3743
|
// Address churn re-publishes the self record AND re-checks the control cohort:
|
|
3515
3744
|
// a relay-reservation rotation / NAT change can drop a sibling connection, so
|
|
3516
|
-
// the next pass should re-observe and re-dial it.
|
|
3517
|
-
this
|
|
3745
|
+
// the next pass should re-observe and re-dial it. Only address churn: libp2p also
|
|
3746
|
+
// fires this for a protocol handler registered or removed (see selfAddressesChanged).
|
|
3747
|
+
this.selfPeerUpdateHandler = (event) => {
|
|
3748
|
+
if (!selfAddressesChanged(event.detail)) return;
|
|
3518
3749
|
republish('self:peer:update');
|
|
3519
|
-
|
|
3750
|
+
this.controlReconcileRuns?.trigger(0);
|
|
3520
3751
|
};
|
|
3521
3752
|
this.controlNode.addEventListener('self:peer:update', this.selfPeerUpdateHandler);
|
|
3522
3753
|
|
|
@@ -3538,15 +3769,32 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3538
3769
|
this.recordRefreshTimer = setInterval(() => republish('heartbeat'), DEFAULT_PEER_RECORD_HEARTBEAT_MS);
|
|
3539
3770
|
(this.recordRefreshTimer as { unref?: () => void } | null)?.unref?.();
|
|
3540
3771
|
|
|
3541
|
-
|
|
3542
|
-
|
|
3543
|
-
|
|
3772
|
+
log('Record refresh wired (heartbeat=%dms, cohortReconcileBackstop=%dms)',
|
|
3773
|
+
DEFAULT_PEER_RECORD_HEARTBEAT_MS, backstopMs);
|
|
3774
|
+
}
|
|
3775
|
+
|
|
3776
|
+
/** The reconcile schedule's backstop: `network.controlCohort.reconcileMs`, default `DEFAULT_CONTROL_COHORT_RECONCILE_MS`. */
|
|
3777
|
+
private controlReconcileBackstopMs(): number {
|
|
3778
|
+
return this.config.network?.controlCohort?.reconcileMs ?? DEFAULT_CONTROL_COHORT_RECONCILE_MS;
|
|
3779
|
+
}
|
|
3544
3780
|
|
|
3545
|
-
|
|
3546
|
-
|
|
3781
|
+
/**
|
|
3782
|
+
* The gate watch's run ({@link membershipGateWatch}): re-materialize the gate, so a
|
|
3783
|
+
* replicated revocation hangs up the removed machine now (`hangUpRevokedControlPeers`
|
|
3784
|
+
* ends every refresh), then start a reconcile pass for whatever else the change means
|
|
3785
|
+
* (a new sibling to dial, a tombstoned row to reap). A local membership write refreshes
|
|
3786
|
+
* the gate twice — once from the awaited membership listener, so the writer sees its
|
|
3787
|
+
* change before returning, and once here, 50 ms later. The listener's refresh runs under
|
|
3788
|
+
* the write lock, so its reads take the committed-read path and the view keeps nothing
|
|
3789
|
+
* from them; this refresh is the read that refills the view after the write, which some
|
|
3790
|
+
* later caller would otherwise pay. Accepted: one reload per change, not two.
|
|
3791
|
+
*/
|
|
3792
|
+
private async onMembershipGateTablesChanged(): Promise<void> {
|
|
3793
|
+
await this.refreshMembershipGate('table-change');
|
|
3794
|
+
this.controlReconcileRuns?.trigger(0);
|
|
3547
3795
|
}
|
|
3548
3796
|
|
|
3549
|
-
/** Tear down the self-record refresh timers
|
|
3797
|
+
/** Tear down the self-record refresh timers, the reconcile schedule and the listeners (see {@link cleanup}). */
|
|
3550
3798
|
private stopRecordRefresh(): void {
|
|
3551
3799
|
if (this.selfRegistrationTimer) {
|
|
3552
3800
|
clearTimeout(this.selfRegistrationTimer);
|
|
@@ -3556,24 +3804,30 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3556
3804
|
clearInterval(this.recordRefreshTimer);
|
|
3557
3805
|
this.recordRefreshTimer = null;
|
|
3558
3806
|
}
|
|
3559
|
-
if (this.
|
|
3560
|
-
|
|
3561
|
-
this.
|
|
3807
|
+
if (this.controlReconcileRuns) {
|
|
3808
|
+
this.controlReconcileRuns.dispose();
|
|
3809
|
+
this.controlReconcileRuns = null;
|
|
3562
3810
|
}
|
|
3811
|
+
if (this.membershipViewReloadTimer) {
|
|
3812
|
+
clearTimeout(this.membershipViewReloadTimer);
|
|
3813
|
+
this.membershipViewReloadTimer = null;
|
|
3814
|
+
}
|
|
3815
|
+
this.reconcileRedialStreak = 0;
|
|
3563
3816
|
if (this.controlNode) {
|
|
3564
3817
|
if (this.selfPeerUpdateHandler) {
|
|
3565
3818
|
this.controlNode.removeEventListener('self:peer:update', this.selfPeerUpdateHandler);
|
|
3566
3819
|
}
|
|
3567
|
-
if (this.
|
|
3568
|
-
this.controlNode.removeEventListener('
|
|
3820
|
+
if (this.controlPeerIdentifyHandler) {
|
|
3821
|
+
this.controlNode.removeEventListener('peer:identify', this.controlPeerIdentifyHandler);
|
|
3569
3822
|
}
|
|
3570
3823
|
if (this.controlConnectionCloseHandler) {
|
|
3571
3824
|
this.controlNode.removeEventListener('connection:close', this.controlConnectionCloseHandler);
|
|
3572
3825
|
}
|
|
3573
3826
|
}
|
|
3574
3827
|
this.selfPeerUpdateHandler = null;
|
|
3575
|
-
this.
|
|
3828
|
+
this.controlPeerIdentifyHandler = null;
|
|
3576
3829
|
this.controlConnectionCloseHandler = null;
|
|
3830
|
+
this.identifiedControlPeers.clear();
|
|
3577
3831
|
|
|
3578
3832
|
// Reset the write-while-alone re-replication state so a stop()→start() cycle
|
|
3579
3833
|
// re-arms the growth edge and re-runs BOTH one-shot passes. Any queued
|
|
@@ -3582,10 +3836,9 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3582
3836
|
// first-growth revocation sweep — which is why the sweep flag must reset too,
|
|
3583
3837
|
// or the second lifetime would skip the sweep and strand exactly the
|
|
3584
3838
|
// committed-alone tombstones it exists to carry.
|
|
3585
|
-
this.
|
|
3839
|
+
this.controlSiblingReachable = false;
|
|
3586
3840
|
this.reconstructedLocalOnlyWrites = false;
|
|
3587
3841
|
this.reissuedHeldRevocations = false;
|
|
3588
|
-
this.revocationLedgerOpened = false;
|
|
3589
3842
|
this.pendingPeerWrites.clear();
|
|
3590
3843
|
this.pendingRevocations.clear();
|
|
3591
3844
|
this.pendingSelfPeerWrite = false;
|
|
@@ -3594,6 +3847,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3594
3847
|
// from scratch, and carrying the previous lifetime's stamp forward would either
|
|
3595
3848
|
// suppress the stale-record warning or fire it on a node that just came up.
|
|
3596
3849
|
this.lastSelfRecordPublishAt = null;
|
|
3850
|
+
this.lastSignedSelfRecord = null;
|
|
3597
3851
|
this.selfRecordStaleWarned = false;
|
|
3598
3852
|
}
|
|
3599
3853
|
|
|
@@ -3783,26 +4037,37 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3783
4037
|
* Run one proactive control-cohort dial pass to keep this node connected to its
|
|
3784
4038
|
* cadre siblings (so the `CadreControl` collections form a replicating cohort).
|
|
3785
4039
|
* Public so the cohort-growth-driven re-replication path
|
|
3786
|
-
* (`control-write-ensure-replicated`)
|
|
3787
|
-
*
|
|
3788
|
-
*
|
|
4040
|
+
* (`control-write-ensure-replicated`), `addDrone`'s callers, `claimNode`,
|
|
4041
|
+
* `redeemCadreInvitation` and tests can run a pass on demand. On its own a pass
|
|
4042
|
+
* runs when something it acts on changes, and otherwise at a backstop
|
|
4043
|
+
* ({@link controlReconcileRuns}): once after start; on `self:peer:update`; when a
|
|
4044
|
+
* membership table changes, local commit or replicated ({@link membershipGateWatch});
|
|
4045
|
+
* when a control connection to an authorized sibling opens or closes
|
|
4046
|
+
* ({@link onControlSiblingConnectionChange}); on the redial and strand-address wakes
|
|
4047
|
+
* a pass arms for itself ({@link armReconcileWake}); and at the backstop,
|
|
4048
|
+
* `network.controlCohort.reconcileMs` (default `DEFAULT_CONTROL_COHORT_RECONCILE_MS`,
|
|
4049
|
+
* 2 minutes) after the last pass settled. A connection to a relay, a bootstrap node or
|
|
4050
|
+
* another party's machine starts no pass: the pass acts on none of them, beyond
|
|
4051
|
+
* counting them as "connected" for its write steps, which therefore run at the next
|
|
4052
|
+
* membership change or backstop on a one-machine party whose only connection is a
|
|
4053
|
+
* relay.
|
|
3789
4054
|
*
|
|
3790
4055
|
* Concurrent triggers collapse into a single in-flight pass
|
|
3791
4056
|
* (see {@link reconcileControlCohortInFlight}) so two passes never double-dial;
|
|
3792
|
-
* a call that joins an in-flight pass resolves to THAT pass's result
|
|
4057
|
+
* a call that joins an in-flight pass resolves to THAT pass's result (a scheduled
|
|
4058
|
+
* run waits for it instead, so the change that woke the run is seen).
|
|
3793
4059
|
* Best-effort throughout: a failure to resolve/dial any one sibling is logged
|
|
3794
4060
|
* and the pass continues; the whole pass is a no-op when the node is alone.
|
|
3795
4061
|
*
|
|
3796
4062
|
* Resolves to the peers the pass dialled ({@link ControlCohortReconcileResult}),
|
|
3797
4063
|
* so a caller can tell a link this pass opened from one something else opened —
|
|
3798
|
-
* the pass skips a peer that is already connected. The
|
|
3799
|
-
* discard it.
|
|
4064
|
+
* the pass skips a peer that is already connected. The scheduled runs discard it.
|
|
3800
4065
|
*/
|
|
3801
4066
|
async reconcileControlCohort(): Promise<ControlCohortReconcileResult> {
|
|
3802
4067
|
if (this.reconcileControlCohortInFlight) {
|
|
3803
4068
|
return this.reconcileControlCohortInFlight;
|
|
3804
4069
|
}
|
|
3805
|
-
const op = this.
|
|
4070
|
+
const op = this.runReconcileControlCohortAndArmWake();
|
|
3806
4071
|
this.reconcileControlCohortInFlight = op;
|
|
3807
4072
|
try {
|
|
3808
4073
|
return await op;
|
|
@@ -3811,38 +4076,77 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3811
4076
|
}
|
|
3812
4077
|
}
|
|
3813
4078
|
|
|
4079
|
+
/** One pass, then the wake it earns ({@link armReconcileWake}); the public result is the pass's `dialed`. */
|
|
4080
|
+
private async runReconcileControlCohortAndArmWake(): Promise<ControlCohortReconcileResult> {
|
|
4081
|
+
const startedAt = Date.now();
|
|
4082
|
+
const outcome = await this.runReconcileControlCohort();
|
|
4083
|
+
this.armReconcileWake(outcome, startedAt);
|
|
4084
|
+
return { dialed: outcome.dialed };
|
|
4085
|
+
}
|
|
4086
|
+
|
|
4087
|
+
/**
|
|
4088
|
+
* After every pass, explicit or scheduled: advance or reset the redial streak and
|
|
4089
|
+
* `trigger` the schedule for the earliest of the pass's own reasons to run again
|
|
4090
|
+
* (`nextReconcileWakeMs`): a redial for a sibling or bootstrap peer it left
|
|
4091
|
+
* unconnected, or a strand-address ask falling due before the backstop. Nothing else
|
|
4092
|
+
* the pass does needs a wake sooner than the backstop: delegate grants re-announce
|
|
4093
|
+
* every 15 minutes inside a 30-minute TTL, the FRET and formation address re-merges
|
|
4094
|
+
* guard a one-hour expiry, and the self record has its own heartbeat. A pass that
|
|
4095
|
+
* runs before the schedule exists (an explicit call during the first second after
|
|
4096
|
+
* start) still counts for the streak.
|
|
4097
|
+
*/
|
|
4098
|
+
private armReconcileWake(outcome: ControlCohortPassOutcome, startedAt: number): void {
|
|
4099
|
+
this.reconcileRedialStreak = outcome.unconnected ? this.reconcileRedialStreak + 1 : 0;
|
|
4100
|
+
const runs = this.controlReconcileRuns;
|
|
4101
|
+
if (!runs) {
|
|
4102
|
+
return;
|
|
4103
|
+
}
|
|
4104
|
+
const wakeMs = nextReconcileWakeMs({
|
|
4105
|
+
redialStreak: this.reconcileRedialStreak,
|
|
4106
|
+
passStartedAt: startedAt,
|
|
4107
|
+
strandAddrDueAt: this.strandAddrAskDueAt.values(),
|
|
4108
|
+
now: Date.now(),
|
|
4109
|
+
backstopMs: this.controlReconcileBackstopMs(),
|
|
4110
|
+
});
|
|
4111
|
+
if (wakeMs !== undefined) {
|
|
4112
|
+
log('reconcileControlCohort: next pass in %dms (redial streak %d)', wakeMs, this.reconcileRedialStreak);
|
|
4113
|
+
runs.trigger(wakeMs);
|
|
4114
|
+
}
|
|
4115
|
+
}
|
|
4116
|
+
|
|
3814
4117
|
/** Body of {@link reconcileControlCohort}; serialised by its single-flight guard. */
|
|
3815
|
-
private async runReconcileControlCohort(): Promise<
|
|
4118
|
+
private async runReconcileControlCohort(): Promise<ControlCohortPassOutcome> {
|
|
3816
4119
|
// Shutdown / not-yet-started guard (mirrors publishSelfRecord). A pass that
|
|
3817
4120
|
// fires after stop() began must early-return rather than touch a torn-down node.
|
|
3818
4121
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3819
|
-
return { dialed: [] };
|
|
3820
|
-
}
|
|
3821
|
-
//
|
|
3822
|
-
//
|
|
3823
|
-
//
|
|
3824
|
-
//
|
|
3825
|
-
// pass's length when that is longer (passes are single-flight, and one
|
|
3826
|
-
// unreachable sibling's dial alone can take `controlDialBudget().totalMs`).
|
|
4122
|
+
return { dialed: [], unconnected: false };
|
|
4123
|
+
}
|
|
4124
|
+
// Refresh the per-stream gate's materialized authorized set at the head of every
|
|
4125
|
+
// pass, so the sibling enumeration below judges rows the same way the gate does.
|
|
4126
|
+
// Changes reach the gate on their own through the gate watch; this refresh is the
|
|
4127
|
+
// backstop's, for a change this node's storage never heard of.
|
|
3827
4128
|
// NOTE: this refresh and the sibling enumeration below each run their own
|
|
3828
4129
|
// CadrePeer query (two reads per pass), plus a third from
|
|
3829
4130
|
// `refreshStrandPeerAddrs` on every pass where a strand is running AND this
|
|
3830
4131
|
// node holds a control connection (one read for the whole pass, not one per
|
|
3831
4132
|
// strand; it runs even when no sibling is due, because pruning departed
|
|
3832
4133
|
// siblings needs the current target set). Each also reads Revocation first.
|
|
3833
|
-
//
|
|
3834
|
-
//
|
|
3835
|
-
//
|
|
3836
|
-
//
|
|
3837
|
-
// control-founding-consult-budget.spec.ts).
|
|
3838
|
-
//
|
|
4134
|
+
// All of them are served from the control database's membership views
|
|
4135
|
+
// (`ControlDatabase.readMembershipRows`): an in-memory scan while the view is
|
|
4136
|
+
// current, storage only for the first read after the table changed or after
|
|
4137
|
+
// `MEMBERSHIP_VIEW_MAX_AGE_MS`, and an idle pass makes no cohort consult
|
|
4138
|
+
// (pinned in control-founding-consult-budget.spec.ts). Threading one row set
|
|
4139
|
+
// through the refresh, the reap, the enumeration and the strand-address targets
|
|
4140
|
+
// would save no storage read and was declined
|
|
4141
|
+
// (`control-reconcile-woken-by-changes-reads-once`); rows going stale during a
|
|
4142
|
+
// pass is harmless, since a change during the pass starts another one.
|
|
3839
4143
|
await this.refreshMembershipGate('reconcile');
|
|
3840
4144
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3841
|
-
return { dialed: [] };
|
|
4145
|
+
return { dialed: [], unconnected: false };
|
|
3842
4146
|
}
|
|
3843
4147
|
await this.publishFirstSelfRecord();
|
|
3844
4148
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3845
|
-
return { dialed: [] };
|
|
4149
|
+
return { dialed: [], unconnected: false };
|
|
3846
4150
|
}
|
|
3847
4151
|
// Refresh relay delegate grants BEFORE the sibling enumeration below: a
|
|
3848
4152
|
// solo cadre with a party relay has no siblings to dial but must still keep
|
|
@@ -3850,7 +4154,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3850
4154
|
// the reservation and faces the connection gate again).
|
|
3851
4155
|
await this.refreshDelegateGrants();
|
|
3852
4156
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3853
|
-
return { dialed: [] };
|
|
4157
|
+
return { dialed: [], unconnected: false };
|
|
3854
4158
|
}
|
|
3855
4159
|
// Then re-warm each running strand's own address book from its FRET address
|
|
3856
4160
|
// records and from whichever siblings are due an ask. Same reasoning as
|
|
@@ -3858,15 +4162,16 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3858
4162
|
// network, and every layer under cadre-core dials strand peers by bare peer id.
|
|
3859
4163
|
await this.refreshStrandPeerAddrs();
|
|
3860
4164
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3861
|
-
return { dialed: [] };
|
|
4165
|
+
return { dialed: [], unconnected: false };
|
|
3862
4166
|
}
|
|
3863
4167
|
const selfPeerId = this.controlNode.peerId.toString();
|
|
3864
4168
|
|
|
3865
4169
|
// Reap guarded rows this node still holds for incarnations an already-committed
|
|
3866
4170
|
// Revocation tombstone retires — the half of a removal replication cannot carry
|
|
3867
|
-
// (a delete of an already-absent row replays as nothing).
|
|
3868
|
-
//
|
|
3869
|
-
//
|
|
4171
|
+
// (a delete of an already-absent row replays as nothing). Only while a sibling is
|
|
4172
|
+
// reachable (hasReachableControlSibling, not merely connected): a reap is a write, and a
|
|
4173
|
+
// write committed alone is local-only and forks this node's own history, which is the
|
|
4174
|
+
// condition this work exists to stop creating
|
|
3870
4175
|
// (tickets/blocked/forked-control-collection-sync-livelocks.md). Nothing is urgent
|
|
3871
4176
|
// here — the stale row already reads as absent everywhere — so waiting for
|
|
3872
4177
|
// connectivity costs nothing and keeps the reap fork-CLOSING rather than
|
|
@@ -3888,7 +4193,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3888
4193
|
// avoid. Bounded by how long a sweep runs, which is bounded by the backlog of
|
|
3889
4194
|
// unreaped rows (0 in steady state). If that backlog can ever be large, move the
|
|
3890
4195
|
// check inside reapRevokedRows' loop via an injected predicate.
|
|
3891
|
-
if (this.
|
|
4196
|
+
if (this.hasReachableControlSibling()) {
|
|
3892
4197
|
try {
|
|
3893
4198
|
const reaped = await this.controlDatabase.reapRevokedRows(selfPeerId);
|
|
3894
4199
|
if (reaped > 0) {
|
|
@@ -3900,41 +4205,22 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3900
4205
|
log('reconcileControlCohort: reap pass failed (continuing): %o', error);
|
|
3901
4206
|
}
|
|
3902
4207
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3903
|
-
return { dialed: [] };
|
|
3904
|
-
}
|
|
3905
|
-
|
|
3906
|
-
// File the Revocation ledger marker once, so that table stops being a never-written
|
|
3907
|
-
// block, which the storage layer re-checks with the cohort on every read (and every
|
|
3908
|
-
// membership lookup and guarded insert reads it). Connected-only for the reap's
|
|
3909
|
-
// reason above: a marker committed alone is local-only, and one filed by a
|
|
3910
|
-
// disconnected owner while another machine creates the same collection is exactly
|
|
3911
|
-
// that fork. Owner-only, and at most once per process — see openRevocationLedgerIfDue.
|
|
3912
|
-
//
|
|
3913
|
-
// NOTE: accepted tradeoff — a solo founder does not file the marker until it holds a
|
|
3914
|
-
// control connection (normally its first sibling; a relay or bootstrap connection also
|
|
3915
|
-
// counts — see getControlConnectionCount — and is the reap's same residual), so until
|
|
3916
|
-
// then it keeps paying one local findCluster per read of
|
|
3917
|
-
// the missing Revocation block (0.009 ms each, measured upstream; no network work on
|
|
3918
|
-
// a cohort of one). Fork safety weighed over that and kept; revisit if findCluster
|
|
3919
|
-
// ever shows up as material in a device profile, or if a solo-founding marker can be
|
|
3920
|
-
// made fork-safe (e.g. filed inside the genesis transaction, before any other machine
|
|
3921
|
-
// can hold the party's collections).
|
|
3922
|
-
await this.openRevocationLedgerIfDue();
|
|
3923
|
-
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3924
|
-
return { dialed: [] };
|
|
4208
|
+
return { dialed: [], unconnected: false };
|
|
3925
4209
|
}
|
|
3926
4210
|
|
|
3927
4211
|
// Publish this machine's unpublished cross-party joins party-wide and remove the rows a
|
|
3928
|
-
// self-revocation queued (JoinedStrandSession.syncWithParty).
|
|
3929
|
-
//
|
|
3930
|
-
// a join
|
|
4212
|
+
// self-revocation queued (JoinedStrandSession.syncWithParty). Behind the reap's gate,
|
|
4213
|
+
// for its reason; owner-only by the session's own signer check. Nothing is urgent:
|
|
4214
|
+
// a join recorded locally starts no pass, so it reaches the party on the next pass
|
|
4215
|
+
// while a sibling is reachable — a membership change, a sibling connecting, or the
|
|
4216
|
+
// backstop, 2 minutes at most.
|
|
3931
4217
|
try {
|
|
3932
4218
|
await this.joinedStrands?.syncWithParty();
|
|
3933
4219
|
} catch (error) {
|
|
3934
4220
|
log('reconcileControlCohort: joined-strand sync failed (retrying next pass): %o', error);
|
|
3935
4221
|
}
|
|
3936
4222
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3937
|
-
return { dialed: [] };
|
|
4223
|
+
return { dialed: [], unconnected: false };
|
|
3938
4224
|
}
|
|
3939
4225
|
}
|
|
3940
4226
|
|
|
@@ -3949,11 +4235,11 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3949
4235
|
// because filling this table needs a connection and getting a connection
|
|
3950
4236
|
// needs this table. Fall back to the seed's bootstrap addresses; a solo
|
|
3951
4237
|
// cadre has none and this is a no-op. Must not throw or busy-loop.
|
|
3952
|
-
return
|
|
4238
|
+
return this.dialColdStartBootstrap();
|
|
3953
4239
|
}
|
|
3954
4240
|
// Re-guard after the await: a stop() may have raced the membership read.
|
|
3955
4241
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3956
|
-
return { dialed: [] };
|
|
4242
|
+
return { dialed: [], unconnected: false };
|
|
3957
4243
|
}
|
|
3958
4244
|
|
|
3959
4245
|
// 2. Classify backbone (always-on) members and select a bounded dial set.
|
|
@@ -3970,54 +4256,34 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3970
4256
|
// keeping what resolved for the dial loop below.
|
|
3971
4257
|
const resolved = await this.warmSiblingAddrBook(siblings);
|
|
3972
4258
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
3973
|
-
return { dialed: [] };
|
|
4259
|
+
return { dialed: [], unconnected: false };
|
|
3974
4260
|
}
|
|
3975
4261
|
|
|
3976
4262
|
// 4. Skip already-connected peers (no re-dial / churn for live connections).
|
|
3977
4263
|
const connected = new Set(this.controlNode.getConnections().map((c) => c.remotePeer.toString()));
|
|
3978
4264
|
|
|
3979
4265
|
// 5. Dial each selected, not-yet-connected sibling, best-effort, from the
|
|
3980
|
-
// addresses step 3 already resolved for it.
|
|
4266
|
+
// addresses step 3 already resolved for it. A failed dial earns the pass a
|
|
4267
|
+
// redial wake (armReconcileWake); a sibling with nothing to dial does not.
|
|
3981
4268
|
const dialed: string[] = [];
|
|
4269
|
+
let unconnected = false;
|
|
3982
4270
|
for (const sibling of dials) {
|
|
3983
4271
|
if (!this._running || !this.controlNode) {
|
|
3984
|
-
return { dialed };
|
|
4272
|
+
return { dialed, unconnected: false };
|
|
3985
4273
|
}
|
|
3986
4274
|
if (connected.has(sibling.peerId)) {
|
|
3987
4275
|
continue;
|
|
3988
4276
|
}
|
|
3989
|
-
|
|
4277
|
+
const outcome = await this.dialControlSibling(sibling, resolved.get(sibling.peerId) ?? []);
|
|
4278
|
+
if (outcome === 'connected') {
|
|
3990
4279
|
dialed.push(sibling.peerId);
|
|
4280
|
+
} else if (outcome === 'failed') {
|
|
4281
|
+
unconnected = true;
|
|
3991
4282
|
}
|
|
3992
4283
|
}
|
|
3993
|
-
log('reconcileControlCohort: pass complete (siblings=%d, selected=%d, dialed=%d)',
|
|
3994
|
-
siblings.length, dials.length, dialed.length);
|
|
3995
|
-
return { dialed };
|
|
3996
|
-
}
|
|
3997
|
-
|
|
3998
|
-
/**
|
|
3999
|
-
* The owner-only half of the reconcile pass's ledger-marker step: file the singleton
|
|
4000
|
-
* `Revocation` marker ({@link SeedBootstrapService.openRevocationLedger}) unless this
|
|
4001
|
-
* process has already seen it filed. The connectivity gate is the caller's
|
|
4002
|
-
* ({@link runReconcileControlCohort}); why the marker exists is on
|
|
4003
|
-
* {@link ControlDatabase.openRevocationLedger}.
|
|
4004
|
-
*
|
|
4005
|
-
* Sets {@link revocationLedgerOpened} on `'opened'` or `'already-open'`. A node that
|
|
4006
|
-
* cannot sign as an owner does nothing: it picks the owner's marker up the first time it
|
|
4007
|
-
* reads the table, like any other replicated row. Best-effort like every step of the
|
|
4008
|
-
* pass — a failure is logged and leaves the flag clear, so the next connected pass retries.
|
|
4009
|
-
*/
|
|
4010
|
-
private async openRevocationLedgerIfDue(): Promise<void> {
|
|
4011
|
-
if (this.revocationLedgerOpened || !this.seedBootstrapService?.canAuthorize()) {
|
|
4012
|
-
return;
|
|
4013
|
-
}
|
|
4014
|
-
try {
|
|
4015
|
-
const outcome = await this.seedBootstrapService.openRevocationLedger();
|
|
4016
|
-
this.revocationLedgerOpened = true;
|
|
4017
|
-
log('reconcileControlCohort: revocation ledger marker %s', outcome);
|
|
4018
|
-
} catch (error) {
|
|
4019
|
-
log('reconcileControlCohort: filing the revocation ledger marker failed (retrying next pass): %o', error);
|
|
4020
|
-
}
|
|
4284
|
+
log('reconcileControlCohort: pass complete (siblings=%d, selected=%d, dialed=%d, unconnected=%s)',
|
|
4285
|
+
siblings.length, dials.length, dialed.length, unconnected);
|
|
4286
|
+
return { dialed, unconnected };
|
|
4021
4287
|
}
|
|
4022
4288
|
|
|
4023
4289
|
/**
|
|
@@ -4043,8 +4309,9 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4043
4309
|
* current ({@link refreshDialHint}).
|
|
4044
4310
|
*
|
|
4045
4311
|
* NOTE: this resolves EVERY sibling serially before the dial loop below runs,
|
|
4046
|
-
* so it costs one record query per sibling per reconcile pass (
|
|
4047
|
-
*
|
|
4312
|
+
* so it costs one record query per sibling per reconcile pass (served from the
|
|
4313
|
+
* membership rows view while it is current) and each one delays the pass's first
|
|
4314
|
+
* dial. A cadre is a handful of devices, so today
|
|
4048
4315
|
* that is a few extra local reads; if cadres ever grow large, batch the records
|
|
4049
4316
|
* into one query, merge only on change, or move the warm pass after the dials.
|
|
4050
4317
|
*/
|
|
@@ -4180,26 +4447,28 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4180
4447
|
}
|
|
4181
4448
|
|
|
4182
4449
|
/**
|
|
4183
|
-
* Dial one sibling from its already-resolved addresses, best-effort.
|
|
4184
|
-
* whether the dial resolved (false when no address resolves or the dial fails).
|
|
4450
|
+
* Dial one sibling from its already-resolved addresses, best-effort.
|
|
4185
4451
|
*
|
|
4186
4452
|
* A per-peer failure (NAT, offline, relay down, connection-gater denial, or
|
|
4187
4453
|
* the {@link controlDialBudget} expiring) is logged and swallowed so one
|
|
4188
4454
|
* unreachable sibling never aborts the pass — exactly like
|
|
4189
4455
|
* {@link SeedBootstrapService.applySeed}'s owner-dial loop. A failed dial is
|
|
4190
|
-
*
|
|
4191
|
-
*
|
|
4192
|
-
*
|
|
4193
|
-
|
|
4194
|
-
|
|
4456
|
+
* retried on the pass's redial ladder ({@link armReconcileWake}: 15 s, 30 s, 60 s,
|
|
4457
|
+
* then every backstop) and on any pass a change starts sooner. A sibling with no
|
|
4458
|
+
* dialable address, or whose every address relays through this node
|
|
4459
|
+
* ({@link SelfRelayOnlyError}), is not dialed at all and is `undialable`: it gets a
|
|
4460
|
+
* one-line log and no redial, since only a change — its record replicating in, or
|
|
4461
|
+
* the sibling connecting — can make it dialable, and each of those starts a pass.
|
|
4462
|
+
*/
|
|
4463
|
+
private async dialControlSibling(sibling: CohortPeerRow, resolved: Multiaddr[]): Promise<SiblingDialOutcome> {
|
|
4195
4464
|
const controlNode = this.controlNode;
|
|
4196
4465
|
if (!controlNode) {
|
|
4197
|
-
return
|
|
4466
|
+
return 'undialable';
|
|
4198
4467
|
}
|
|
4199
4468
|
const addrs = await this.resolveControlDialAddrs(sibling.peerId, resolved);
|
|
4200
4469
|
if (addrs.length === 0) {
|
|
4201
4470
|
log('reconcileControlCohort: no dialable control address for sibling %s; skipping', sibling.peerId);
|
|
4202
|
-
return
|
|
4471
|
+
return 'undialable';
|
|
4203
4472
|
}
|
|
4204
4473
|
try {
|
|
4205
4474
|
log('reconcileControlCohort: dialing sibling %s (%d addr(s))', sibling.peerId, addrs.length);
|
|
@@ -4210,14 +4479,14 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4210
4479
|
this.controlDialBudget(),
|
|
4211
4480
|
`reconcileControlCohort dial of sibling ${sibling.peerId}`
|
|
4212
4481
|
);
|
|
4213
|
-
return
|
|
4482
|
+
return 'connected';
|
|
4214
4483
|
} catch (error) {
|
|
4215
4484
|
if (error instanceof SelfRelayOnlyError) {
|
|
4216
4485
|
log('reconcileControlCohort: sibling %s is reachable only by relaying through this node; waiting for it to reconnect', sibling.peerId);
|
|
4217
|
-
|
|
4218
|
-
log('reconcileControlCohort: dial of sibling %s failed (continuing): %o', sibling.peerId, error);
|
|
4486
|
+
return 'undialable';
|
|
4219
4487
|
}
|
|
4220
|
-
|
|
4488
|
+
log('reconcileControlCohort: dial of sibling %s failed (continuing): %o', sibling.peerId, error);
|
|
4489
|
+
return 'failed';
|
|
4221
4490
|
}
|
|
4222
4491
|
}
|
|
4223
4492
|
|
|
@@ -4272,7 +4541,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4272
4541
|
* These addresses are never merged into the libp2p address book: they are
|
|
4273
4542
|
* unverified, and the address book takes only verified ones (see
|
|
4274
4543
|
* `mergePeerAddrs`). Layers that dial by bare peer id use the connection this
|
|
4275
|
-
* dial opens, and after it drops the
|
|
4544
|
+
* dial opens, and after it drops the reconcile pass the close starts dials again.
|
|
4276
4545
|
*
|
|
4277
4546
|
* NOTE: an entry is consulted here only for a current sibling, so a peer revoked
|
|
4278
4547
|
* by another owner is not dialed from it, even though the entry stays (only this
|
|
@@ -4327,8 +4596,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4327
4596
|
* Called for every seed this node accepts, on BOTH intake paths — the
|
|
4328
4597
|
* {@link applySeed} wrapper (which may run on a throwaway service, so the
|
|
4329
4598
|
* service itself is the wrong place to keep this) and the inbound
|
|
4330
|
-
* `/sereus/seed/1.0.0` handler via `onSeedApplied`.
|
|
4331
|
-
* skipped: there is nothing to dial.
|
|
4599
|
+
* `/sereus/seed/1.0.0` handler via `onSeedApplied`.
|
|
4332
4600
|
*
|
|
4333
4601
|
* `preferred` is the seed's own claim about its peers, exactly as
|
|
4334
4602
|
* `SeedBootstrapService.applySeed` uses it to choose its dial targets. That is
|
|
@@ -4336,14 +4604,15 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4336
4604
|
* signature-checked against a trust-anchored signer before this runs, and the
|
|
4337
4605
|
* flag only *selects a dial target*; a dial grants no authority.
|
|
4338
4606
|
*
|
|
4339
|
-
*
|
|
4607
|
+
* The peers retained are `seedDialTargets` (addressless peers and self dropped), the same
|
|
4608
|
+
* ones the service dials; they are returned for the `seed:applied` payload.
|
|
4340
4609
|
*/
|
|
4341
|
-
private recordSeedBootstrapPeers(seed: ControlNetworkSeed):
|
|
4342
|
-
|
|
4343
|
-
|
|
4344
|
-
|
|
4345
|
-
}
|
|
4610
|
+
private recordSeedBootstrapPeers(seed: ControlNetworkSeed): SeedDialTarget[] {
|
|
4611
|
+
const targets = seedDialTargets(seed, this.controlNode?.peerId.toString());
|
|
4612
|
+
for (const { peerId, multiaddrs } of targets) {
|
|
4613
|
+
this.retainDialTarget(peerId, multiaddrs, 'recordSeedBootstrapPeers');
|
|
4346
4614
|
}
|
|
4615
|
+
return targets;
|
|
4347
4616
|
}
|
|
4348
4617
|
|
|
4349
4618
|
/**
|
|
@@ -4358,9 +4627,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4358
4627
|
* failing a seed or an add that has already been accepted.
|
|
4359
4628
|
*
|
|
4360
4629
|
* Two peers are never retained. One with no address: there is nothing to dial.
|
|
4361
|
-
* And self: `
|
|
4362
|
-
*
|
|
4363
|
-
* could be handed this node's own id. Retaining self would make the cold-start
|
|
4630
|
+
* And self: `addDrone` could be handed this node's own id (seed intake arrives
|
|
4631
|
+
* already filtered by `seedDialTargets`). Retaining self would make the cold-start
|
|
4364
4632
|
* pass dial this node forever (the steady-state pass filters self out of its
|
|
4365
4633
|
* sibling list for the same reason).
|
|
4366
4634
|
*/
|
|
@@ -4395,33 +4663,35 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4395
4663
|
* help: it dials only siblings enumerated from that very table. Retrying here
|
|
4396
4664
|
* is the only way back in.
|
|
4397
4665
|
*
|
|
4398
|
-
* Unbounded
|
|
4399
|
-
*
|
|
4400
|
-
*
|
|
4401
|
-
* a
|
|
4402
|
-
*
|
|
4403
|
-
*
|
|
4404
|
-
*
|
|
4405
|
-
*
|
|
4406
|
-
*
|
|
4666
|
+
* Unbounded: the branch is already gated by "the control database has no
|
|
4667
|
+
* siblings", so it stops the moment the node is actually in the party, and a node
|
|
4668
|
+
* that is stranded MUST keep trying — a give-up rule would turn a transient outage
|
|
4669
|
+
* into a permanent one. A failed dial earns the pass a redial wake on the ladder
|
|
4670
|
+
* {@link armReconcileWake} runs (15 s, 30 s, 60 s, then every backstop, 2 minutes
|
|
4671
|
+
* by default), so the steady cost is one dial per bootstrap peer per backstop (a
|
|
4672
|
+
* seed prefers its minter and the party's always-on machines, a few at most), each
|
|
4673
|
+
* failing fast against an unreachable address.
|
|
4674
|
+
* NOTE: the ladder is per pass, not per peer; if seeds ever carry many preferred
|
|
4675
|
+
* peers, add per-peer backoff here.
|
|
4407
4676
|
*
|
|
4408
4677
|
* Best-effort per peer, exactly like {@link dialControlSibling}: one dead
|
|
4409
4678
|
* address never aborts the pass.
|
|
4410
4679
|
*/
|
|
4411
|
-
private async dialColdStartBootstrap(): Promise<
|
|
4680
|
+
private async dialColdStartBootstrap(): Promise<ControlCohortPassOutcome> {
|
|
4412
4681
|
const controlNode = this.controlNode;
|
|
4413
4682
|
// Snapshot up front: the store hands back a copy, so a seed applied mid-pass
|
|
4414
4683
|
// cannot mutate what we are iterating.
|
|
4415
4684
|
const targets = this.bootstrapPeerStore?.all();
|
|
4416
4685
|
if (!controlNode || !targets || targets.size === 0) {
|
|
4417
|
-
return [];
|
|
4686
|
+
return { dialed: [], unconnected: false };
|
|
4418
4687
|
}
|
|
4419
4688
|
// Same skip rule as step 3 of the steady-state pass: no churn on live links.
|
|
4420
4689
|
const connected = new Set(controlNode.getConnections().map((c) => c.remotePeer.toString()));
|
|
4421
4690
|
const dialed: string[] = [];
|
|
4691
|
+
let unconnected = false;
|
|
4422
4692
|
for (const [peerId, entry] of targets) {
|
|
4423
4693
|
if (!this._running || !this.controlNode) {
|
|
4424
|
-
return dialed;
|
|
4694
|
+
return { dialed, unconnected: false };
|
|
4425
4695
|
}
|
|
4426
4696
|
// NOTE: a connection this node still holds in `status: 'open'` after the remote
|
|
4427
4697
|
// has already aborted it counts as connected here, so the cold-start retry stays
|
|
@@ -4435,11 +4705,13 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4435
4705
|
}
|
|
4436
4706
|
if (await this.dialBootstrapPeer(peerId, entry.addrs)) {
|
|
4437
4707
|
dialed.push(peerId);
|
|
4708
|
+
} else {
|
|
4709
|
+
unconnected = true;
|
|
4438
4710
|
}
|
|
4439
4711
|
}
|
|
4440
|
-
log('reconcileControlCohort: cold-start pass complete (bootstrap=%d, connected=%d, dialed=%d)',
|
|
4441
|
-
targets.size, connected.size, dialed.length);
|
|
4442
|
-
return dialed;
|
|
4712
|
+
log('reconcileControlCohort: cold-start pass complete (bootstrap=%d, connected=%d, dialed=%d, unconnected=%s)',
|
|
4713
|
+
targets.size, connected.size, dialed.length, unconnected);
|
|
4714
|
+
return { dialed, unconnected };
|
|
4443
4715
|
}
|
|
4444
4716
|
|
|
4445
4717
|
/**
|
|
@@ -4458,32 +4730,36 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4458
4730
|
return this.normalizeDialAddrs(this.parseMultiaddrs(addrs), peerId);
|
|
4459
4731
|
}
|
|
4460
4732
|
|
|
4461
|
-
/**
|
|
4462
|
-
|
|
4733
|
+
/**
|
|
4734
|
+
* Dial one peer from its retained dial target's addresses, best-effort: a cold-start bootstrap
|
|
4735
|
+
* peer, or a just-claimed node ({@link redialClaimedNode}), which `context` names in the logs.
|
|
4736
|
+
* Returns whether it connected.
|
|
4737
|
+
*/
|
|
4738
|
+
private async dialBootstrapPeer(peerId: string, addrs: string[], context = 'reconcileControlCohort(cold-start)'): Promise<boolean> {
|
|
4463
4739
|
const controlNode = this.controlNode;
|
|
4464
4740
|
if (!controlNode) {
|
|
4465
4741
|
return false;
|
|
4466
4742
|
}
|
|
4467
4743
|
const parsed = this.bootstrapDialAddrs(peerId, addrs);
|
|
4468
4744
|
if (parsed.length === 0) {
|
|
4469
|
-
log('
|
|
4745
|
+
log('%s: no dialable address for bootstrap peer %s; skipping', context, peerId);
|
|
4470
4746
|
return false;
|
|
4471
4747
|
}
|
|
4472
4748
|
try {
|
|
4473
|
-
log('
|
|
4749
|
+
log('%s: dialing bootstrap peer %s (%d addr(s))', context, peerId, parsed.length);
|
|
4474
4750
|
await dialPeerAddrs(
|
|
4475
4751
|
controlNode,
|
|
4476
4752
|
peerId,
|
|
4477
4753
|
parsed,
|
|
4478
4754
|
this.controlDialBudget(),
|
|
4479
|
-
|
|
4755
|
+
`${context} dial of bootstrap peer ${peerId}`
|
|
4480
4756
|
);
|
|
4481
4757
|
return true;
|
|
4482
4758
|
} catch (error) {
|
|
4483
4759
|
if (error instanceof SelfRelayOnlyError) {
|
|
4484
|
-
log('
|
|
4760
|
+
log('%s: bootstrap peer %s is reachable only by relaying through this node; waiting for it to reconnect', context, peerId);
|
|
4485
4761
|
} else {
|
|
4486
|
-
log('
|
|
4762
|
+
log('%s: dial of bootstrap peer %s failed (continuing): %o', context, peerId, error);
|
|
4487
4763
|
}
|
|
4488
4764
|
return false;
|
|
4489
4765
|
}
|
|
@@ -4494,32 +4770,34 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4494
4770
|
//
|
|
4495
4771
|
// Optimystic's coordinator commits a control write WITHOUT broadcasting when the
|
|
4496
4772
|
// block's cluster has ≤1 member. An owner that authorizes/removes a peer or
|
|
4497
|
-
// (re)publishes its own record while no sibling is
|
|
4773
|
+
// (re)publishes its own record while no sibling is reachable therefore writes a
|
|
4498
4774
|
// row that exists ONLY in its local control DB — a sibling that connects later
|
|
4499
4775
|
// never observes it (pull-on-read routes to the block's cluster, which never
|
|
4500
4776
|
// learned of the write). reconcileControlCohort shrinks the alone window but
|
|
4501
|
-
// cannot close it (a write in the instant before any sibling
|
|
4502
|
-
// local-only). The remedy: detect "wrote while alone" (
|
|
4503
|
-
//
|
|
4504
|
-
//
|
|
4777
|
+
// cannot close it (a write in the instant before any sibling identifies is still
|
|
4778
|
+
// local-only). The remedy: detect "wrote while alone" (no identified sibling the
|
|
4779
|
+
// cohort would admit: committedAlone), queue the affected row, and RE-ISSUE the write
|
|
4780
|
+
// once one is reachable, as an idempotent monotonic update that now broadcasts.
|
|
4505
4781
|
// See docs/architecture.md (Control Network Convergence → "Writes made while alone").
|
|
4506
4782
|
// ============================================================================
|
|
4507
4783
|
|
|
4508
4784
|
/**
|
|
4509
|
-
* Whether a control write happening now would commit local-only
|
|
4510
|
-
*
|
|
4511
|
-
*
|
|
4512
|
-
*
|
|
4513
|
-
*
|
|
4514
|
-
*
|
|
4515
|
-
*
|
|
4516
|
-
* is acceptable as the agreed first cut.
|
|
4785
|
+
* Whether a control write happening now would commit local-only: no sibling is reachable
|
|
4786
|
+
* ({@link hasReachableControlSibling}, the public form, so the two cannot drift). A relay, a
|
|
4787
|
+
* bootstrap node or a peer that has not identified yet holds a connection without being in the
|
|
4788
|
+
* write's cohort, so a write made with only those connected is queued too. One made alone at
|
|
4789
|
+
* the revision the cohort reached meanwhile is a fork no later consult sees, because a consult
|
|
4790
|
+
* compares revision numbers only; the re-issue on the growth edge is what conflicts with the
|
|
4791
|
+
* cohort's copy and pulls it in.
|
|
4517
4792
|
*
|
|
4518
|
-
*
|
|
4519
|
-
* the
|
|
4793
|
+
* Still a proxy for the precise signal (the block's own cohort): a reachable sibling outside
|
|
4794
|
+
* this block's cohort leaves the write local-only, caught by the cohort's pull-on-read. On the
|
|
4795
|
+
* control network every block's cohort is the whole party, so that gap is a party larger than
|
|
4796
|
+
* `CONTROL_REPLICATION_BREADTH`. Re-issuing a replicated row is harmless (an idempotent
|
|
4797
|
+
* monotonic bump).
|
|
4520
4798
|
*/
|
|
4521
4799
|
private committedAlone(): boolean {
|
|
4522
|
-
return this.
|
|
4800
|
+
return !this.hasReachableControlSibling();
|
|
4523
4801
|
}
|
|
4524
4802
|
|
|
4525
4803
|
/**
|
|
@@ -4576,7 +4854,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4576
4854
|
/**
|
|
4577
4855
|
* Drain the write-while-alone re-replication queue: re-issue the writes that
|
|
4578
4856
|
* committed local-only now that the cohort can broadcast them. Public so the
|
|
4579
|
-
*
|
|
4857
|
+
* growth trigger ({@link updateControlSiblingReachability}) and tests can drive
|
|
4580
4858
|
* it; concurrent calls collapse into one in-flight drain
|
|
4581
4859
|
* ({@link drainControlReplicationInFlight}) so two growth signals never
|
|
4582
4860
|
* double-issue. Best-effort throughout — a per-row failure leaves that entry
|
|
@@ -4618,13 +4896,15 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4618
4896
|
// periodic-refresh property.
|
|
4619
4897
|
await this.drainPendingRevocations();
|
|
4620
4898
|
|
|
4621
|
-
// 2. Self rows.
|
|
4622
|
-
//
|
|
4623
|
-
//
|
|
4899
|
+
// 2. Self rows. A self write made while alone is always re-touched: the re-touch
|
|
4900
|
+
// is what carries it to the cohort. Otherwise the first growth re-touches a row
|
|
4901
|
+
// a prior process may have committed local-only, unless the row already holds a
|
|
4902
|
+
// current record this process signed (a redeemed invitation's admission).
|
|
4624
4903
|
// registerSelf is idempotent + single-flight, so this never races a
|
|
4625
4904
|
// duplicate publish with the heartbeat republish.
|
|
4626
4905
|
if (firstGrowth || this.pendingSelfPeerWrite) {
|
|
4627
|
-
|
|
4906
|
+
const publish = this.pendingSelfPeerWrite ? this.registerSelf() : this.publishSelfRecordUnlessCurrent();
|
|
4907
|
+
await publish.catch((error) => log('drain: registerSelf failed: %o', error));
|
|
4628
4908
|
this.pendingSelfPeerWrite = false;
|
|
4629
4909
|
}
|
|
4630
4910
|
if (firstGrowth || this.pendingSelfDeviceWrite) {
|
|
@@ -4782,8 +5062,11 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4782
5062
|
|
|
4783
5063
|
/**
|
|
4784
5064
|
* Body of {@link reconstructAuthoredMembership}'s sweep: re-issue each candidate
|
|
4785
|
-
* owner-authored row, returning how many were re-touched. A
|
|
4786
|
-
*
|
|
5065
|
+
* owner-authored row, returning how many were re-touched. A row the sweep already holds as
|
|
5066
|
+
* self-published is skipped without a call; the re-touch itself decides again over the row
|
|
5067
|
+
* it reads under the lock, so a row that vanished or was self-published since the sweep's
|
|
5068
|
+
* read is reported, not written. A per-row failure is logged and skipped; a teardown
|
|
5069
|
+
* mid-sweep stops it.
|
|
4787
5070
|
*/
|
|
4788
5071
|
private async reissueAuthoredMembershipRows(selfPeerId: string, members: CadrePeerRow[]): Promise<number> {
|
|
4789
5072
|
let touched = 0;
|
|
@@ -4791,17 +5074,15 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4791
5074
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
4792
5075
|
return touched;
|
|
4793
5076
|
}
|
|
4794
|
-
|
|
5077
|
+
// Self, queued for the drain (step 4 re-touches it once), or peer-owned (self-Sig
|
|
5078
|
+
// present) → not this sweep's to bump.
|
|
5079
|
+
if (member.peerId === selfPeerId || this.pendingPeerWrites.has(member.peerId) || member.sig) {
|
|
4795
5080
|
continue;
|
|
4796
5081
|
}
|
|
4797
5082
|
try {
|
|
4798
|
-
|
|
4799
|
-
|
|
4800
|
-
if (!record || record.sig) {
|
|
4801
|
-
continue;
|
|
5083
|
+
if (await this.reissuePeerAuthorize(member.peerId) === 'retouched') {
|
|
5084
|
+
touched++;
|
|
4802
5085
|
}
|
|
4803
|
-
await this.reissuePeerAuthorize(member.peerId, record.updatedAt);
|
|
4804
|
-
touched++;
|
|
4805
5086
|
} catch (error) {
|
|
4806
5087
|
log('drain: reconstruction re-touch of %s failed (continuing): %o', member.peerId, error);
|
|
4807
5088
|
}
|
|
@@ -4836,9 +5117,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4836
5117
|
}
|
|
4837
5118
|
|
|
4838
5119
|
/**
|
|
4839
|
-
* Body of {@link drainPendingPeerWrites}' loop. An entry is cleared
|
|
4840
|
-
*
|
|
4841
|
-
* the
|
|
5120
|
+
* Body of {@link drainPendingPeerWrites}' loop. An entry is cleared on every outcome the
|
|
5121
|
+
* re-touch reports — re-touched, or nothing left to re-issue because the row was removed
|
|
5122
|
+
* since or the peer self-published; the re-touch decides which over the row it reads under
|
|
5123
|
+
* the lock. A failure is logged and the entry stays queued for the next cohort growth.
|
|
4842
5124
|
*/
|
|
4843
5125
|
private async reissuePendingPeerWrites(): Promise<void> {
|
|
4844
5126
|
for (const peerId of [...this.pendingPeerWrites.keys()]) {
|
|
@@ -4846,13 +5128,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4846
5128
|
return;
|
|
4847
5129
|
}
|
|
4848
5130
|
try {
|
|
4849
|
-
const
|
|
4850
|
-
|
|
4851
|
-
// Removed since, or peer self-published — nothing for the owner to re-issue.
|
|
4852
|
-
this.pendingPeerWrites.delete(peerId);
|
|
4853
|
-
continue;
|
|
4854
|
-
}
|
|
4855
|
-
await this.reissuePeerAuthorize(peerId, record.updatedAt);
|
|
5131
|
+
const outcome = await this.reissuePeerAuthorize(peerId);
|
|
5132
|
+
log('drain: re-issue of %s (authorize): %s', peerId, outcome);
|
|
4856
5133
|
this.pendingPeerWrites.delete(peerId);
|
|
4857
5134
|
} catch (error) {
|
|
4858
5135
|
log('drain: re-issue of %s (authorize) failed; leaving queued for next growth: %o', peerId, error);
|
|
@@ -4865,60 +5142,202 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4865
5142
|
* `UpdatedAt` strictly above the stored value (and the wall clock) and re-sign via
|
|
4866
5143
|
* the owner branch of `CadrePeer.AuthorizedUpdate`. The row already exists
|
|
4867
5144
|
* locally (it committed there, just local-only), so this is an UPDATE, not the
|
|
4868
|
-
* original INSERT.
|
|
5145
|
+
* original INSERT. The bump, and whether the row is still the owner's to re-issue, are
|
|
5146
|
+
* decided over the row as read inside the write ({@link SeedBootstrapService.reauthorizePeer}'s
|
|
5147
|
+
* re-broadcast mode).
|
|
4869
5148
|
*/
|
|
4870
|
-
private async reissuePeerAuthorize(peerId: string
|
|
5149
|
+
private async reissuePeerAuthorize(peerId: string): Promise<CadrePeerRetouchOutcome> {
|
|
4871
5150
|
if (!this.seedBootstrapService) {
|
|
4872
|
-
|
|
5151
|
+
throw new Error('Seed bootstrap service not initialized. Call initializeSeedBootstrap() first.');
|
|
4873
5152
|
}
|
|
4874
|
-
|
|
4875
|
-
await this.seedBootstrapService.reauthorizePeer(peerId, updatedAt);
|
|
5153
|
+
return await this.seedBootstrapService.reauthorizePeer(peerId, 'rebroadcast');
|
|
4876
5154
|
}
|
|
4877
5155
|
|
|
4878
5156
|
/**
|
|
4879
|
-
*
|
|
4880
|
-
* write-while-alone re-replication queue (single-flight;
|
|
4881
|
-
* not on
|
|
4882
|
-
*
|
|
5157
|
+
* Re-evaluate {@link hasReachableControlSibling} and act on its edges: on false→true, drain
|
|
5158
|
+
* the write-while-alone re-replication queue (single-flight; best-effort, a drain failure is
|
|
5159
|
+
* logged, not thrown); on true→false, re-arm, so the next reachable sibling drains again.
|
|
5160
|
+
* Called when a peer identifies, when a connection closes, and after every membership refresh,
|
|
5161
|
+
* where a sibling that connected before this node authorized it (the production join order)
|
|
5162
|
+
* becomes reachable. A device counted in for its admission ({@link admittingDevices}) is not
|
|
5163
|
+
* a trigger: the admission's own writes are in flight, and its row landing refreshes
|
|
5164
|
+
* membership.
|
|
4883
5165
|
*/
|
|
4884
|
-
private
|
|
5166
|
+
private updateControlSiblingReachability(): void {
|
|
4885
5167
|
if (!this._running || !this.controlNode) {
|
|
4886
5168
|
return;
|
|
4887
5169
|
}
|
|
4888
|
-
const
|
|
4889
|
-
if (
|
|
5170
|
+
const reachable = this.hasReachableControlSibling();
|
|
5171
|
+
if (reachable === this.controlSiblingReachable) {
|
|
5172
|
+
return;
|
|
5173
|
+
}
|
|
5174
|
+
this.controlSiblingReachable = reachable;
|
|
5175
|
+
if (!reachable) {
|
|
4890
5176
|
return;
|
|
4891
5177
|
}
|
|
4892
|
-
|
|
4893
|
-
|
|
4894
|
-
|
|
4895
|
-
.catch((error) => log('Control re-replication drain (connection:open) failed: %o', error));
|
|
5178
|
+
log('Control cohort grew (a sibling is reachable); draining write-while-alone re-replication queue');
|
|
5179
|
+
void this.drainPendingControlReplication('sibling-reachable')
|
|
5180
|
+
.catch((error) => log('Control re-replication drain (sibling-reachable) failed: %o', error));
|
|
4896
5181
|
}
|
|
4897
5182
|
|
|
4898
|
-
/**
|
|
4899
|
-
|
|
4900
|
-
|
|
5183
|
+
/**
|
|
5184
|
+
* A connected peer identified as serving the control database: count it
|
|
5185
|
+
* ({@link identifiedControlPeers}), re-evaluate the growth edge, and run the sibling-open
|
|
5186
|
+
* reactions, once per connected period. libp2p re-emits `peer:identify` on every
|
|
5187
|
+
* identify-push from a connected peer, and those repeats start nothing.
|
|
5188
|
+
*/
|
|
5189
|
+
private onControlPeerIdentified(peerId: PeerId): void {
|
|
5190
|
+
const id = peerId.toString();
|
|
5191
|
+
// An identify that reports after its connection closed counts nobody.
|
|
5192
|
+
if (this.identifiedControlPeers.has(id) || !this.controlNode || this.controlNode.getConnections(peerId).length === 0) {
|
|
4901
5193
|
return;
|
|
4902
5194
|
}
|
|
4903
|
-
|
|
4904
|
-
|
|
5195
|
+
this.identifiedControlPeers.add(id);
|
|
5196
|
+
this.updateControlSiblingReachability();
|
|
5197
|
+
this.onControlSiblingConnectionChange(id, 'open');
|
|
5198
|
+
}
|
|
5199
|
+
|
|
5200
|
+
/**
|
|
5201
|
+
* A control connection closed: `remotePeer` stops counting once it has none left, and the
|
|
5202
|
+
* growth edge re-arms if no reachable sibling remains. The sibling-close reaction runs on every
|
|
5203
|
+
* close. libp2p's connection manager drops the closed connection from `getConnections` before
|
|
5204
|
+
* this listener runs: libp2p dispatches each event on its internal bus, where the connection
|
|
5205
|
+
* manager listens, before re-dispatching it to the node's own listeners.
|
|
5206
|
+
*/
|
|
5207
|
+
private onControlConnectionClosed(remotePeer: PeerId): void {
|
|
5208
|
+
const id = remotePeer.toString();
|
|
5209
|
+
if (!this.controlNode || this.controlNode.getConnections(remotePeer).length === 0) {
|
|
5210
|
+
this.identifiedControlPeers.delete(id);
|
|
4905
5211
|
}
|
|
5212
|
+
this.updateControlSiblingReachability();
|
|
5213
|
+
this.onControlSiblingConnectionChange(id, 'close');
|
|
4906
5214
|
}
|
|
4907
5215
|
|
|
4908
5216
|
/**
|
|
4909
|
-
*
|
|
4910
|
-
*
|
|
4911
|
-
*
|
|
4912
|
-
*
|
|
5217
|
+
* Count the peers that identified before {@link wireControlConnectionListeners} ran, whose
|
|
5218
|
+
* `peer:identify` has already fired, from their peer-store records. Updates the growth edge
|
|
5219
|
+
* but runs no sibling-open reaction, as a connection that predates the listeners never did. A
|
|
5220
|
+
* peer not yet in the peer store is still identifying, and its own event counts it.
|
|
5221
|
+
*/
|
|
5222
|
+
private async noteIdentifiedControlPeers(): Promise<void> {
|
|
5223
|
+
const node = this.controlNode;
|
|
5224
|
+
if (!node) {
|
|
5225
|
+
return;
|
|
5226
|
+
}
|
|
5227
|
+
const connected = new Map<string, PeerId>();
|
|
5228
|
+
for (const connection of node.getConnections()) {
|
|
5229
|
+
connected.set(connection.remotePeer.toString(), connection.remotePeer);
|
|
5230
|
+
}
|
|
5231
|
+
const partyId = this.config.controlNetwork.partyId;
|
|
5232
|
+
await Promise.all([...connected.values()].map(async (peerId) => {
|
|
5233
|
+
try {
|
|
5234
|
+
const { protocols } = await node.peerStore.get(peerId);
|
|
5235
|
+
// Still wired to this node: a stop while the read was out cleared the set.
|
|
5236
|
+
const wired = this.controlPeerIdentifyHandler !== null && this.controlNode === node;
|
|
5237
|
+
if (wired && servesControlDatabase(partyId, protocols) && node.getConnections(peerId).length > 0) {
|
|
5238
|
+
this.identifiedControlPeers.add(peerId.toString());
|
|
5239
|
+
}
|
|
5240
|
+
} catch (error) {
|
|
5241
|
+
if ((error as Error).name !== 'NotFoundError') {
|
|
5242
|
+
log('Peer store read for connected control peer %s failed — left to its identify: %o', peerId.toString(), error);
|
|
5243
|
+
}
|
|
5244
|
+
}
|
|
5245
|
+
}));
|
|
5246
|
+
this.updateControlSiblingReachability();
|
|
5247
|
+
}
|
|
5248
|
+
|
|
5249
|
+
/**
|
|
5250
|
+
* An AUTHORIZED sibling became reachable (its identify, once per connected period:
|
|
5251
|
+
* {@link onControlPeerIdentified}) or one of its connections closed: start a reconcile
|
|
5252
|
+
* pass. On open the pass asks the new sibling for strand addresses and re-arms the
|
|
5253
|
+
* delegate grants; on close it re-dials. First, on open, drop the membership rows
|
|
5254
|
+
* views: a machine that was offline while a sibling committed learns of the commit
|
|
5255
|
+
* only when it next reads the block from storage, and dropping the views makes the
|
|
5256
|
+
* pass's reads do that now rather than after the views' maximum age. Those reads can
|
|
5257
|
+
* fall inside the read-repair window, so the open also arms a second drop past it
|
|
5258
|
+
* ({@link armMembershipViewReload}). Any other
|
|
5259
|
+
* peer — a relay, a bootstrap node, another party's machine — starts nothing.
|
|
5260
|
+
*
|
|
5261
|
+
* On identify rather than `connection:open` because, until the sibling identifies,
|
|
5262
|
+
* Optimystic's cohort for every control block is this node alone: a pass started at the
|
|
5263
|
+
* open writes (the first self-record publish) and reads on a cohort of one.
|
|
5264
|
+
*/
|
|
5265
|
+
private onControlSiblingConnectionChange(remotePeer: string, change: 'open' | 'close'): void {
|
|
5266
|
+
if (!this.authorizedControlPeers.has(remotePeer)) {
|
|
5267
|
+
return;
|
|
5268
|
+
}
|
|
5269
|
+
if (change === 'open') {
|
|
5270
|
+
// NOTE: a machine removed from the cadre that has not learned of it keeps redialing
|
|
5271
|
+
// its siblings, each of which admits it provisionally and hangs up at the deadline
|
|
5272
|
+
// (28.5 s at the default link), and every such open drops the views here and once
|
|
5273
|
+
// more past the read-repair window: two storage reloads, with their cohort consults
|
|
5274
|
+
// refused, per cycle until it learns. If a removed machine's churn ever shows up,
|
|
5275
|
+
// skip both drops for a peer whose previous connection the remote closed inside the
|
|
5276
|
+
// provisional deadline.
|
|
5277
|
+
this.controlDatabase?.invalidateMembershipViews();
|
|
5278
|
+
this.armMembershipViewReload();
|
|
5279
|
+
}
|
|
5280
|
+
// NOTE: a close starts a pass at once, and a sibling in its bring-up quiet period lets the
|
|
5281
|
+
// dial resolve and then drops the connection (`createMembershipConnectionGater`'s deny
|
|
5282
|
+
// timing), so a restarting sibling costs one dial-connect-close pass after another for its
|
|
5283
|
+
// whole bring-up (5-6 passes in ~200 ms against a claimed node restarting locally). If a
|
|
5284
|
+
// slow device's bring-up ever makes that churn show up, delay the pass a close starts when
|
|
5285
|
+
// the previous connection to that sibling lived only a few milliseconds.
|
|
5286
|
+
log('Control connection to sibling %s %s; reconcile pass triggered', remotePeer, change);
|
|
5287
|
+
this.controlReconcileRuns?.trigger(0);
|
|
5288
|
+
}
|
|
5289
|
+
|
|
5290
|
+
/**
|
|
5291
|
+
* Drop the membership views again {@link MEMBERSHIP_VIEW_RECONNECT_RELOAD_MS} after a
|
|
5292
|
+
* sibling connects, and start a pass whose reads reload them. This node's reads before
|
|
5293
|
+
* the connection stamped Optimystic's read-repair window on the control blocks, so the
|
|
5294
|
+
* reload the open itself starts is served from the local copy with no cohort consult,
|
|
5295
|
+
* and the view would keep rows that predate the sibling's commits until its maximum age.
|
|
5296
|
+
* The reload past the window starts the consults; one that brings a newer revision
|
|
5297
|
+
* raises the table change that drops the view and refreshes the gate. Re-armed by every
|
|
5298
|
+
* open, so it runs one window after the latest.
|
|
5299
|
+
*
|
|
5300
|
+
* NOTE: a consult Optimystic defers (16 already in flight) starts nothing and leaves the block
|
|
5301
|
+
* unstamped, so the view keeps its rows until the first read past its maximum age, which on an
|
|
5302
|
+
* idle node is the reconcile backstop. If `cluster-tx:read-repair-deferred` lines show at a
|
|
5303
|
+
* reconnect, re-arm this reload once more instead of waiting for the backstop.
|
|
5304
|
+
*/
|
|
5305
|
+
private armMembershipViewReload(): void {
|
|
5306
|
+
if (this.membershipViewReloadTimer) {
|
|
5307
|
+
clearTimeout(this.membershipViewReloadTimer);
|
|
5308
|
+
}
|
|
5309
|
+
this.membershipViewReloadTimer = setTimeout(() => {
|
|
5310
|
+
this.membershipViewReloadTimer = null;
|
|
5311
|
+
log('Read-repair window passed since a sibling connected; reloading the membership views');
|
|
5312
|
+
this.controlDatabase?.invalidateMembershipViews();
|
|
5313
|
+
this.controlReconcileRuns?.trigger(0);
|
|
5314
|
+
}, MEMBERSHIP_VIEW_RECONNECT_RELOAD_MS);
|
|
5315
|
+
// Node-only: don't keep the event loop alive solely for this timer.
|
|
5316
|
+
(this.membershipViewReloadTimer as { unref?: () => void } | null)?.unref?.();
|
|
5317
|
+
}
|
|
5318
|
+
|
|
5319
|
+
/**
|
|
5320
|
+
* Wire the control-connection listeners, which keep {@link identifiedControlPeers}: the
|
|
5321
|
+
* growth edge that drives write-while-alone re-replication, and the sibling reconcile
|
|
5322
|
+
* trigger ({@link onControlSiblingConnectionChange}). Wired in {@link start} (not the
|
|
5323
|
+
* delayed refresh path) so no early identify is missed, and reads back the peers that
|
|
5324
|
+
* identified before it ran ({@link noteIdentifiedControlPeers}); torn down in
|
|
5325
|
+
* {@link stopRecordRefresh}. Idempotent.
|
|
4913
5326
|
*/
|
|
4914
5327
|
private wireControlConnectionListeners(): void {
|
|
4915
|
-
if (!this.controlNode || this.
|
|
5328
|
+
if (!this.controlNode || this.controlPeerIdentifyHandler) {
|
|
4916
5329
|
return;
|
|
4917
5330
|
}
|
|
4918
|
-
|
|
4919
|
-
this.
|
|
4920
|
-
|
|
5331
|
+
const partyId = this.config.controlNetwork.partyId;
|
|
5332
|
+
this.controlPeerIdentifyHandler = (event) => {
|
|
5333
|
+
if (servesControlDatabase(partyId, event.detail.protocols)) {
|
|
5334
|
+
this.onControlPeerIdentified(event.detail.peerId);
|
|
5335
|
+
}
|
|
5336
|
+
};
|
|
5337
|
+
this.controlConnectionCloseHandler = (event) => this.onControlConnectionClosed(event.detail.remotePeer);
|
|
5338
|
+
this.controlNode.addEventListener('peer:identify', this.controlPeerIdentifyHandler);
|
|
4921
5339
|
this.controlNode.addEventListener('connection:close', this.controlConnectionCloseHandler);
|
|
5340
|
+
void this.noteIdentifiedControlPeers();
|
|
4922
5341
|
}
|
|
4923
5342
|
|
|
4924
5343
|
// ============================================================================
|
|
@@ -5249,6 +5668,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
5249
5668
|
if (sAppId !== undefined) {
|
|
5250
5669
|
this.replicaSAppIds.set(strandId, sAppId);
|
|
5251
5670
|
log('Storage replica %s serves sApp %s', strandId, sAppId);
|
|
5671
|
+
this.strandWatcher?.sAppIdLearned(strandId);
|
|
5252
5672
|
}
|
|
5253
5673
|
} catch (error) {
|
|
5254
5674
|
log('Could not read the sApp id of storage replica %s (left unknown): %o', strandId, error);
|
|
@@ -5371,6 +5791,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
5371
5791
|
this.strandLaunchRefusals.clear();
|
|
5372
5792
|
this.discoveredStrands.clear();
|
|
5373
5793
|
this.replicaSAppIds.clear();
|
|
5794
|
+
this.formedPartyKeys.clear();
|
|
5374
5795
|
|
|
5375
5796
|
// Drop delegate-admission state: the grants are scoped to the session that
|
|
5376
5797
|
// recorded them, so a stop()/start() cycle on this object must not keep
|
|
@@ -5394,6 +5815,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
5394
5815
|
// Detach the membership hub first: nothing this teardown does should drive a
|
|
5395
5816
|
// gate refresh, and a notification arriving after this point is a no-op.
|
|
5396
5817
|
if (this.controlDatabase) {
|
|
5818
|
+
// The database's registry would dispose the gate watch too; disposing it here, before
|
|
5819
|
+
// the listener goes, keeps a closing commit from scheduling a gate refresh.
|
|
5820
|
+
this.membershipGateWatch?.dispose();
|
|
5821
|
+
this.membershipGateWatch = null;
|
|
5397
5822
|
this.controlDatabase.setMembershipChangeListener(null);
|
|
5398
5823
|
this.controlDatabase.setGuardedDeleteListener(null);
|
|
5399
5824
|
this.controlDatabase.setControlWriteAbandonedListener(null);
|
|
@@ -5753,8 +6178,9 @@ export class CadreNode implements SAppIdLookup {
|
|
|
5753
6178
|
* registered, deliberately: both an explicit retry and the {@link StrandWatcher}'s
|
|
5754
6179
|
* automatic relaunch need it. A failed launch here hands the strand back to that
|
|
5755
6180
|
* relaunch — `StrandWatcher.forgetStrand` drops the id from the watcher's
|
|
5756
|
-
* `knownStrands
|
|
5757
|
-
*
|
|
6181
|
+
* `knownStrands`, records the backoff and wakes the watcher when it expires, so the
|
|
6182
|
+
* strand is re-attempted on the same ladder a watcher-driven failure gets
|
|
6183
|
+
* (`min(strandWatchInterval, 5 s) * 2^(failures-1)`, capped at 5 minutes). Because the
|
|
5758
6184
|
* config stays registered, the retry takes `handleStrandAdded`'s auto-launch branch:
|
|
5759
6185
|
* what the app sees is `strand:error` per failed retry and `strand:started` when one
|
|
5760
6186
|
* succeeds, never a second `strand:discovered`. {@link detachStrand} (reached via
|
|
@@ -5797,6 +6223,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
5797
6223
|
// A claim also overrides any earlier `stopStrand` of this id: the watcher must track
|
|
5798
6224
|
// the strand again, or a party-wide removal would never stop it here.
|
|
5799
6225
|
this.strandWatcher?.unsuppressStrand(strandRow.Id);
|
|
6226
|
+
this.strandWatcher?.sAppIdLearned(strandRow.Id);
|
|
5800
6227
|
log('Registered sAppConfig for strand %s (sApp: %s, founder: %s)',
|
|
5801
6228
|
strandRow.Id, sAppConfig.id, founder ?? 'derived');
|
|
5802
6229
|
|
|
@@ -5822,7 +6249,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
5822
6249
|
// Hand the strand back to the watcher's retry ladder. Harmless on the founder path
|
|
5823
6250
|
// (`foundStrand` → `publishStrand` + this call), where the watcher may never have
|
|
5824
6251
|
// offered the strand: the `knownStrands` delete is a no-op and the recorded backoff
|
|
5825
|
-
// only delays the watcher's own first attempt by one
|
|
6252
|
+
// only delays the watcher's own first attempt by one ladder step, which is what
|
|
5826
6253
|
// should happen right after an attempt that just failed.
|
|
5827
6254
|
//
|
|
5828
6255
|
// NOTE: the retry relaunches from the CONTROL row (or, for a strand joined from
|
|
@@ -5906,7 +6333,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
5906
6333
|
* (the key exists nowhere else). No app-data wipe is needed for either.
|
|
5907
6334
|
*
|
|
5908
6335
|
* The insert is signed through this node's owner signer ({@link getOwnerSigner}), in one
|
|
5909
|
-
* approval with a closed strand's party
|
|
6336
|
+
* approval and one control transaction with a closed strand's party key
|
|
6337
|
+
* ({@link ControlDatabase.insertFoundingRows}). The signer's key is persisted on the row
|
|
5910
6338
|
* as `FounderOwnerKey` (the schema pins the column to the verified signer), and this node's
|
|
5911
6339
|
* peer id as `FounderPeerId` (bound into the signature), which is what later lets any
|
|
5912
6340
|
* launch derive "this machine is the founder" from the row alone.
|
|
@@ -5917,7 +6345,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
5917
6345
|
*
|
|
5918
6346
|
* A CLOSED strand's publish also seats this party's own membership identity key —
|
|
5919
6347
|
* the `StrandPartyKey` row {@link ensureStrandPartyKey} documents — which is what the
|
|
5920
|
-
* founder bootstrap derives `Member.Key`/`Manager.MemberKey` from
|
|
6348
|
+
* founder bootstrap derives `Member.Key`/`Manager.MemberKey` from; a key the party already
|
|
6349
|
+
* holds for the id is kept. The
|
|
5921
6350
|
* `memberPrivateKey` ARGUMENT stays the strand-wide read secret formation hands to
|
|
5922
6351
|
* joiners; it derives nobody's identity.
|
|
5923
6352
|
*
|
|
@@ -5937,23 +6366,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
5937
6366
|
*/
|
|
5938
6367
|
async publishStrand(strandId: string, type: 'o' | 'c' = 'o', memberPrivateKey?: string, sign?: OwnerSign): Promise<StrandRow> {
|
|
5939
6368
|
const signer = this.requireOwnerSigner(`publish strand ${strandId}`);
|
|
5940
|
-
|
|
5941
|
-
|
|
5942
|
-
const trimmed = requireNonBlank(strandId, 'strand id');
|
|
5943
|
-
// Refuse an unusable id HERE, not only at launch. `addStrand` already asserts, but it
|
|
5944
|
-
// runs after this row is written, and this row replicates: an id no node can turn into
|
|
5945
|
-
// a storage scope key would otherwise reach the whole party's control database and be
|
|
5946
|
-
// declined once per member. Asserted after the trim so the id checked is the id stored.
|
|
5947
|
-
assertStrandScopeKey(trimmed);
|
|
5948
|
-
// FounderPeerId records THIS machine as the founder, which is what later lets a relaunch
|
|
5949
|
-
// derive founder-ness; ControlDatabase.insertStrand writes the same id (its own node's).
|
|
5950
|
-
const desired: StrandRow = {
|
|
5951
|
-
Id: trimmed,
|
|
5952
|
-
Type: type,
|
|
5953
|
-
MemberPrivateKey: memberPrivateKey ?? null,
|
|
5954
|
-
FounderOwnerKey: signer.publicKey,
|
|
5955
|
-
FounderPeerId: this.controlNode!.peerId.toString(),
|
|
5956
|
-
};
|
|
6369
|
+
const desired = this.founderStrandRow(strandId, type, memberPrivateKey, signer);
|
|
6370
|
+
const trimmed = desired.Id;
|
|
5957
6371
|
|
|
5958
6372
|
// Read first rather than leaning on the insert's collision for BOTH readings. The
|
|
5959
6373
|
// ordinary resume (an app killed after its own publish committed) then needs no error
|
|
@@ -5974,53 +6388,158 @@ export class CadreNode implements SAppIdLookup {
|
|
|
5974
6388
|
log('publishStrand(%s): already published (type %s) with identical content — no-op', trimmed, type);
|
|
5975
6389
|
// A closed strand THIS machine published must hold a party key (its own membership
|
|
5976
6390
|
// identity — see ensureStrandPartyKey). Insert-if-absent, so the ordinary repeat
|
|
5977
|
-
// is a read-only no-op and a
|
|
5978
|
-
//
|
|
6391
|
+
// is a read-only no-op and a closed row stored without its key heals here. A row
|
|
6392
|
+
// another machine published is that machine's to mint for.
|
|
5979
6393
|
if (type === 'c' && this.isSelfFoundedRow(matched)) {
|
|
5980
6394
|
await this.ensureStrandPartyKey(trimmed, undefined, sign);
|
|
5981
6395
|
}
|
|
5982
6396
|
return matched;
|
|
5983
6397
|
}
|
|
5984
6398
|
|
|
5985
|
-
return await
|
|
5986
|
-
(approved) => this.insertStrandRow(desired, signer, approved));
|
|
6399
|
+
return (await this.writeFoundingRows(desired, undefined, signer, sign)).strandRow;
|
|
5987
6400
|
}
|
|
5988
6401
|
|
|
5989
6402
|
/**
|
|
5990
|
-
* The
|
|
5991
|
-
*
|
|
5992
|
-
*
|
|
6403
|
+
* The `Strand` row this machine publishes for `strandId`, as {@link publishStrand} and
|
|
6404
|
+
* {@link foundStrand} write it: the id trimmed and checked, this node's owner key and peer id
|
|
6405
|
+
* as its founder.
|
|
5993
6406
|
*/
|
|
5994
|
-
private
|
|
5995
|
-
|
|
6407
|
+
private founderStrandRow(strandId: string, type: 'o' | 'c', memberPrivateKey: string | undefined, signer: OwnerSigner): StrandRow {
|
|
6408
|
+
// Trim/reject here so the id that lands matches the one unpublishStrand looks up: it
|
|
6409
|
+
// trims too, and an untrimmed row would be unreachable by the same string.
|
|
6410
|
+
const trimmed = requireNonBlank(strandId, 'strand id');
|
|
6411
|
+
// Refuse an unusable id HERE, not only at launch. `addStrand` already asserts, but it
|
|
6412
|
+
// runs after this row is written, and this row replicates: an id no node can turn into
|
|
6413
|
+
// a storage scope key would otherwise reach the whole party's control database and be
|
|
6414
|
+
// declined once per member. Asserted after the trim so the id checked is the id stored.
|
|
6415
|
+
assertStrandScopeKey(trimmed);
|
|
6416
|
+
// FounderPeerId records THIS machine as the founder, which is what later lets a relaunch
|
|
6417
|
+
// derive founder-ness; ControlDatabase.insertFoundingRows writes the same id (its own node's).
|
|
6418
|
+
return {
|
|
6419
|
+
Id: trimmed,
|
|
6420
|
+
Type: type,
|
|
6421
|
+
MemberPrivateKey: memberPrivateKey ?? null,
|
|
6422
|
+
FounderOwnerKey: signer.publicKey,
|
|
6423
|
+
FounderPeerId: this.controlNode!.peerId.toString(),
|
|
6424
|
+
};
|
|
6425
|
+
}
|
|
6426
|
+
|
|
6427
|
+
/**
|
|
6428
|
+
* The write half of {@link publishStrand} and {@link foundStrand}: under one owner approval,
|
|
6429
|
+
* insert `desired`, a closed strand's party key and the requested invitation in one control
|
|
6430
|
+
* transaction ({@link ControlDatabase.insertFoundingRows}), adopting a matching row a
|
|
6431
|
+
* concurrent founder landed first. Reads nothing first: the caller has just read the
|
|
6432
|
+
* `Strand` row, and a stored key or row is resolved from the insert's own rejection.
|
|
6433
|
+
*/
|
|
6434
|
+
private async writeFoundingRows(
|
|
6435
|
+
desired: StrandRow,
|
|
6436
|
+
invite: FoundingInvite | undefined,
|
|
6437
|
+
signer: OwnerSigner,
|
|
6438
|
+
sign?: OwnerSign
|
|
6439
|
+
): Promise<FoundingWrite> {
|
|
6440
|
+
const action = invite
|
|
6441
|
+
? `Publish the strand ${desired.Id} to the cadre, with an invitation to the app ${invite.sAppId}`
|
|
6442
|
+
: `Publish the strand ${desired.Id} to the cadre`;
|
|
6443
|
+
return await withinOwnerApproval(signer, { partyId: this.partyId, action }, sign,
|
|
6444
|
+
(approved) => this.insertOrAdoptFounding(desired, invite, signer, approved));
|
|
6445
|
+
}
|
|
6446
|
+
|
|
6447
|
+
/** {@link writeFoundingRows} inside its approval. */
|
|
6448
|
+
private async insertOrAdoptFounding(
|
|
6449
|
+
desired: StrandRow,
|
|
6450
|
+
invite: FoundingInvite | undefined,
|
|
6451
|
+
signer: OwnerSigner,
|
|
6452
|
+
sign: OwnerSign
|
|
6453
|
+
): Promise<FoundingWrite> {
|
|
6454
|
+
let partyKey: string | undefined;
|
|
5996
6455
|
try {
|
|
5997
|
-
await this.
|
|
6456
|
+
partyKey = await this.seatFoundingRows(desired, invite, signer, sign);
|
|
5998
6457
|
} catch (error) {
|
|
5999
6458
|
// Only the uniqueness collision is a candidate for the idempotent reading; every
|
|
6000
6459
|
// other rejection (unauthorized signer, retired stamp) must keep surfacing.
|
|
6001
6460
|
if (!isStrandIdConflict(error, 'Strand')) {
|
|
6002
6461
|
throw error;
|
|
6003
6462
|
}
|
|
6004
|
-
const
|
|
6005
|
-
|
|
6006
|
-
|
|
6007
|
-
|
|
6008
|
-
|
|
6463
|
+
const strandRow = await this.adoptConcurrentFounding(desired, error);
|
|
6464
|
+
// The rejection rolled the invitation back with the row; a caller asking for one gets one.
|
|
6465
|
+
if (invite) {
|
|
6466
|
+
await this.publishFoundingInviteIfAbsent(strandRow.Id, invite, sign);
|
|
6467
|
+
}
|
|
6468
|
+
return { strandRow };
|
|
6469
|
+
}
|
|
6470
|
+
log('Published strand %s (type %s) to control DB under owner %s', desired.Id, desired.Type, signer.publicKey);
|
|
6471
|
+
if (invite) {
|
|
6472
|
+
this.registerPublishedInvitation(invite.token, invite.expiresAtMs);
|
|
6473
|
+
}
|
|
6474
|
+
return { strandRow: desired, partyKey };
|
|
6475
|
+
}
|
|
6476
|
+
|
|
6477
|
+
/**
|
|
6478
|
+
* Insert `desired`'s founding rows, minting a closed strand's party key — this party's own
|
|
6479
|
+
* membership identity, which the founder bootstrap derives `Member`/`Manager` from, distinct
|
|
6480
|
+
* from the row's shared `MemberPrivateKey` read secret. A key already stored for the id with
|
|
6481
|
+
* no strand row (one {@link ensureStrandPartyKey} seated, which it would adopt too) refuses
|
|
6482
|
+
* that transaction whole; the rows are then written again without a key.
|
|
6483
|
+
*
|
|
6484
|
+
* @returns The party key this seated; undefined when it seated none, so the launch reads the stored one.
|
|
6485
|
+
*/
|
|
6486
|
+
private async seatFoundingRows(
|
|
6487
|
+
desired: StrandRow,
|
|
6488
|
+
invite: FoundingInvite | undefined,
|
|
6489
|
+
signer: OwnerSigner,
|
|
6490
|
+
sign: OwnerSign
|
|
6491
|
+
): Promise<string | undefined> {
|
|
6492
|
+
const controlDatabase = this.controlDatabase!;
|
|
6493
|
+
const strand = { id: desired.Id, type: desired.Type, memberPrivateKey: desired.MemberPrivateKey ?? undefined };
|
|
6494
|
+
if (desired.Type !== 'c') {
|
|
6495
|
+
await controlDatabase.insertFoundingRows({ strand, formationInvite: invite }, signer.publicKey, sign);
|
|
6496
|
+
return undefined;
|
|
6497
|
+
}
|
|
6498
|
+
const partyKey = await generateStrandMemberKey();
|
|
6499
|
+
try {
|
|
6500
|
+
await controlDatabase.insertFoundingRows({ strand, partyKey, formationInvite: invite }, signer.publicKey, sign);
|
|
6501
|
+
return partyKey;
|
|
6502
|
+
} catch (error) {
|
|
6503
|
+
if (!isStrandIdConflict(error, 'StrandPartyKey')) {
|
|
6009
6504
|
throw error;
|
|
6010
6505
|
}
|
|
6011
|
-
log('publishStrand(%s):
|
|
6012
|
-
|
|
6506
|
+
log('publishStrand(%s): this party already holds a key for the id; publishing the row and adopting that key', desired.Id);
|
|
6507
|
+
await controlDatabase.insertFoundingRows({ strand, formationInvite: invite }, signer.publicKey, sign);
|
|
6508
|
+
return undefined;
|
|
6013
6509
|
}
|
|
6014
|
-
|
|
6015
|
-
|
|
6016
|
-
|
|
6017
|
-
|
|
6018
|
-
|
|
6019
|
-
|
|
6020
|
-
|
|
6021
|
-
|
|
6510
|
+
}
|
|
6511
|
+
|
|
6512
|
+
/**
|
|
6513
|
+
* Resolve a founding refused on `Strand.Id`: re-read the row that landed and adopt it when it
|
|
6514
|
+
* matches `desired` — a concurrent founder, such as two machines of one party on one id —
|
|
6515
|
+
* or throw naming the columns that differ.
|
|
6516
|
+
*/
|
|
6517
|
+
private async adoptConcurrentFounding(desired: StrandRow, error: unknown): Promise<StrandRow> {
|
|
6518
|
+
const landed = await this.controlDatabase!.queryStrand(desired.Id);
|
|
6519
|
+
if (!landed) {
|
|
6520
|
+
// Collided, yet no row is readable: not the race this branch handles (a row
|
|
6521
|
+
// deleted between the rejection and the re-read, or a stale local view). Surface
|
|
6522
|
+
// the original rather than guessing.
|
|
6523
|
+
throw error;
|
|
6022
6524
|
}
|
|
6023
|
-
|
|
6525
|
+
log('publishStrand(%s): lost a concurrent founding race; re-read the landed row', desired.Id);
|
|
6526
|
+
return requireMatchingStrandRow(landed, desired, 'landed concurrently from another founder');
|
|
6527
|
+
}
|
|
6528
|
+
|
|
6529
|
+
/**
|
|
6530
|
+
* Publish a founding's invitation on its own, bound to `strandId`, for a founding that adopted
|
|
6531
|
+
* a stored row rather than writing its own — unless the invitation is already stored, bound to
|
|
6532
|
+
* this strand, as a repeat of a founding whose transaction committed leaves it.
|
|
6533
|
+
*
|
|
6534
|
+
* @param sign - The open approval's `sign`; omitted, the publish asks for its own.
|
|
6535
|
+
*/
|
|
6536
|
+
private async publishFoundingInviteIfAbsent(strandId: string, invite: FoundingInvite, sign?: OwnerSign): Promise<void> {
|
|
6537
|
+
if ((await this.controlDatabase!.queryFormationInvite(invite.token))?.strandId === strandId) {
|
|
6538
|
+
log('foundStrand(%s): the invitation is already published', strandId);
|
|
6539
|
+
return;
|
|
6540
|
+
}
|
|
6541
|
+
const { token, sAppId, ...options } = invite;
|
|
6542
|
+
await this.publishFormationInvite(token, sAppId, { ...options, strandId }, sign);
|
|
6024
6543
|
}
|
|
6025
6544
|
|
|
6026
6545
|
/**
|
|
@@ -6028,6 +6547,11 @@ export class CadreNode implements SAppIdLookup {
|
|
|
6028
6547
|
* founder — in one resumable call. The single entry point for creating a strand; a
|
|
6029
6548
|
* caller joining one someone else founded uses {@link addStrand} alone.
|
|
6030
6549
|
*
|
|
6550
|
+
* The publish is one control transaction under one owner approval: the `Strand` row, a
|
|
6551
|
+
* closed strand's `StrandPartyKey` row, and the invitation in
|
|
6552
|
+
* {@link FoundStrandConfig.formationInvite} when one is given. The start then uses the party
|
|
6553
|
+
* key the publish seated rather than reading it back.
|
|
6554
|
+
*
|
|
6031
6555
|
* Safe to re-run from any point of interruption, which hand-rolling
|
|
6032
6556
|
* {@link publishStrand} + {@link addStrand} is not. Either half can already have
|
|
6033
6557
|
* happened:
|
|
@@ -6109,18 +6633,19 @@ export class CadreNode implements SAppIdLookup {
|
|
|
6109
6633
|
* derive whether this machine founds it. Every check that can refuse the founding is here,
|
|
6110
6634
|
* so both forms of `foundStrand` reject on it before returning.
|
|
6111
6635
|
*/
|
|
6112
|
-
private async publishFoundingRow(config: FoundStrandConfig): Promise<
|
|
6113
|
-
const { strandId, type = 'o', memberPrivateKey } = config;
|
|
6636
|
+
private async publishFoundingRow(config: FoundStrandConfig): Promise<FoundingPublish> {
|
|
6637
|
+
const { strandId, type = 'o', memberPrivateKey, formationInvite, sAppConfig } = config;
|
|
6114
6638
|
if (!this._running || !this.controlDatabase) {
|
|
6115
6639
|
throw new Error(`CadreNode must be started before attempting to found strand ${strandId}`);
|
|
6116
6640
|
}
|
|
6117
6641
|
const controlDatabase = this.controlDatabase;
|
|
6118
6642
|
const trimmed = requireNonBlank(strandId, 'strand id');
|
|
6643
|
+
const invite: FoundingInvite | undefined = formationInvite && { ...formationInvite, sAppId: sAppConfig.id };
|
|
6119
6644
|
const timed = <T>(step: string, op: () => Promise<T>) => timedStep('foundStrand', trimmed, step, op);
|
|
6120
6645
|
const published = await timed('queryStrand', () => controlDatabase.queryStrand(trimmed));
|
|
6121
|
-
const strandRow = published
|
|
6122
|
-
? this.
|
|
6123
|
-
: await timed('publishStrand', () => this.
|
|
6646
|
+
const { strandRow, partyKey } = published
|
|
6647
|
+
? await this.resumeFounding(published, type, memberPrivateKey, invite)
|
|
6648
|
+
: await timed('publishStrand', () => this.publishFoundingStrand(trimmed, type, memberPrivateKey, invite));
|
|
6124
6649
|
// Derived, not hardcoded `true`: adopting a row another machine published means that
|
|
6125
6650
|
// machine runs the bootstrap — this one must attach or it would write a second Header.
|
|
6126
6651
|
const founded = this.isSelfFoundedRow(strandRow);
|
|
@@ -6129,20 +6654,52 @@ export class CadreNode implements SAppIdLookup {
|
|
|
6129
6654
|
'(FounderPeerId is not this node\'s peer id) — attaching as a joiner; ' +
|
|
6130
6655
|
'the founder bootstrap runs on that machine', trimmed);
|
|
6131
6656
|
}
|
|
6132
|
-
return { strandRow, founded };
|
|
6657
|
+
return { strandRow, founded, partyKey };
|
|
6658
|
+
}
|
|
6659
|
+
|
|
6660
|
+
/** {@link publishFoundingRow} when no row is stored: write the founding rows under this node's owner. */
|
|
6661
|
+
private publishFoundingStrand(
|
|
6662
|
+
strandId: string,
|
|
6663
|
+
type: 'o' | 'c',
|
|
6664
|
+
memberPrivateKey: string | undefined,
|
|
6665
|
+
invite: FoundingInvite | undefined
|
|
6666
|
+
): Promise<FoundingWrite> {
|
|
6667
|
+
const signer = this.requireOwnerSigner(`publish strand ${strandId}`);
|
|
6668
|
+
return this.writeFoundingRows(this.founderStrandRow(strandId, type, memberPrivateKey, signer), invite, signer);
|
|
6669
|
+
}
|
|
6670
|
+
|
|
6671
|
+
/**
|
|
6672
|
+
* {@link publishFoundingRow} over a stored row: adopt it ({@link adoptPublishedStrand}), then
|
|
6673
|
+
* publish the requested invitation on its own, under its own approval.
|
|
6674
|
+
*/
|
|
6675
|
+
private async resumeFounding(
|
|
6676
|
+
published: StrandRow,
|
|
6677
|
+
type: 'o' | 'c',
|
|
6678
|
+
memberPrivateKey: string | undefined,
|
|
6679
|
+
invite: FoundingInvite | undefined
|
|
6680
|
+
): Promise<FoundingWrite> {
|
|
6681
|
+
const strandRow = this.adoptPublishedStrand(published, type, memberPrivateKey);
|
|
6682
|
+
if (invite) {
|
|
6683
|
+
await timedStep('foundStrand', strandRow.Id, 'publishFormationInvite',
|
|
6684
|
+
() => this.publishFoundingInviteIfAbsent(strandRow.Id, invite));
|
|
6685
|
+
}
|
|
6686
|
+
return { strandRow };
|
|
6133
6687
|
}
|
|
6134
6688
|
|
|
6135
6689
|
/**
|
|
6136
6690
|
* The start half of {@link foundStrand}. Its timing lines (`addStrand: start`, then the
|
|
6137
6691
|
* duration or `failed after`) are logged when the start settles, also in the background
|
|
6138
6692
|
* form, so a start that hangs still shows as a start line with no end line.
|
|
6693
|
+
*
|
|
6694
|
+
* A party key the publish half just seated is handed to the launch, which would otherwise
|
|
6695
|
+
* read back the row the same founding wrote.
|
|
6139
6696
|
*/
|
|
6140
6697
|
private attachFoundingRow(
|
|
6141
|
-
{ strandRow, founded }:
|
|
6698
|
+
{ strandRow, founded, partyKey }: FoundingPublish,
|
|
6142
6699
|
{ sAppConfig, awaitFirstSync }: FoundStrandConfig
|
|
6143
6700
|
): Promise<StrandInstance> {
|
|
6144
6701
|
return timedStep('foundStrand', strandRow.Id, 'addStrand',
|
|
6145
|
-
() => this.addStrand({ strandRow, sAppConfig, founder: founded, awaitFirstSync }));
|
|
6702
|
+
() => this.addStrand({ strandRow, sAppConfig, founder: founded, awaitFirstSync, partyMemberPrivateKey: partyKey }));
|
|
6146
6703
|
}
|
|
6147
6704
|
|
|
6148
6705
|
/**
|
|
@@ -6178,9 +6735,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
6178
6735
|
* inverse of {@link publishStrand}.
|
|
6179
6736
|
*
|
|
6180
6737
|
* Party-wide, unlike {@link stopStrand}, which only stops the strand on THIS node: every
|
|
6181
|
-
* cadre node watching the table
|
|
6182
|
-
*
|
|
6183
|
-
*
|
|
6738
|
+
* cadre node watching the table hears the row's removal and stops its own instance (a
|
|
6739
|
+
* machine outside the `Strand` block's cohort, on a cadre larger than 16, only at its
|
|
6740
|
+
* watcher's backstop read, default 5 minutes). This node converges immediately — the
|
|
6741
|
+
* method forces a watcher run and stops any still-running local instance before resolving. Removing the row
|
|
6184
6742
|
* removes OUR party's participation only: other parties in the strand keep their own
|
|
6185
6743
|
* rows and the strand network carries on, so no cross-party sign-off is involved — this
|
|
6186
6744
|
* is an owner-signed control-plane write like every other, not "destroy the network".
|
|
@@ -6267,16 +6825,16 @@ export class CadreNode implements SAppIdLookup {
|
|
|
6267
6825
|
// deadline rather than dropping it (a binding cleared late is still worth more than
|
|
6268
6826
|
// one never cleared).
|
|
6269
6827
|
await this.strandManager.clearOwnMemberPeerBinding(trimmed);
|
|
6270
|
-
// Converge locally
|
|
6271
|
-
// onStrandRemoved for a strand it tracked; the explicit teardown below
|
|
6272
|
-
// whose strandFilter never admitted this strand (the watcher never knew
|
|
6273
|
-
// never fire). StrandInstanceManager.stopStrand no-ops on an unknown id, so the two
|
|
6828
|
+
// Converge locally before resolving rather than when the write wakes the watcher. The
|
|
6829
|
+
// watcher fires onStrandRemoved for a strand it tracked; the explicit teardown below
|
|
6830
|
+
// covers a node whose strandFilter never admitted this strand (the watcher never knew
|
|
6831
|
+
// it, so it will never fire). StrandInstanceManager.stopStrand no-ops on an unknown id, so the two
|
|
6274
6832
|
// paths cannot double-stop into an error.
|
|
6275
6833
|
//
|
|
6276
6834
|
// detachStrand, NOT stopStrand: this is a party-wide removal, not a local abandonment
|
|
6277
6835
|
// of a strand that still exists, so it must not SUPPRESS the id in the watcher. The
|
|
6278
6836
|
// row is already gone — there is nothing to suppress — and an id re-published before
|
|
6279
|
-
// the next
|
|
6837
|
+
// the next run's suppression cleanup would then never be offered again this session.
|
|
6280
6838
|
// (Same reasoning as handleStrandRemoved, which also avoids stopStrand's `_running`
|
|
6281
6839
|
// guard; here the guard is already satisfied by requireOwnerSigner above.)
|
|
6282
6840
|
await this.strandWatcher?.forcePoll();
|
|
@@ -6300,9 +6858,11 @@ export class CadreNode implements SAppIdLookup {
|
|
|
6300
6858
|
* as `MemberPrivateKey`; docs/strands.md → "Closed-Strand Member Key Handling") and is
|
|
6301
6859
|
* never put on the formation wire.
|
|
6302
6860
|
*
|
|
6303
|
-
*
|
|
6304
|
-
*
|
|
6305
|
-
*
|
|
6861
|
+
* A founding seats the key in the same transaction as the strand row, not through here.
|
|
6862
|
+
* Called internally by {@link publishStrand}'s repeat of a published closed strand and by a
|
|
6863
|
+
* founder launch that finds no row, which heal a closed strand row stored without its key
|
|
6864
|
+
* (one seated by `ControlDatabase.insertStrand` directly, or whose key was removed); public
|
|
6865
|
+
* so a test harness — or a joiner flow that persists a formation-issued identity — can seat
|
|
6306
6866
|
* a specific key deliberately.
|
|
6307
6867
|
*
|
|
6308
6868
|
* @param strandId - The strand the key is this party's identity for.
|
|
@@ -6353,8 +6913,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
6353
6913
|
* Resolve the party membership key a closed strand's launch threads into the founder
|
|
6354
6914
|
* bootstrap: the explicit attach-time key when given, else the party's persisted
|
|
6355
6915
|
* `StrandPartyKey` row, else — on the one machine the row names as founder (`FounderPeerId`)
|
|
6356
|
-
* — a freshly minted-and-persisted key, which seats the identity for a
|
|
6357
|
-
*
|
|
6916
|
+
* — a freshly minted-and-persisted key, which seats the identity for a closed row stored
|
|
6917
|
+
* without one (see {@link ensureStrandPartyKey}). Everyone else resolves undefined: non-founding
|
|
6358
6918
|
* machines never mint (no mint race between a party's machines), and only a founder
|
|
6359
6919
|
* bootstrap needs the key at all.
|
|
6360
6920
|
*
|
|
@@ -6374,6 +6934,11 @@ export class CadreNode implements SAppIdLookup {
|
|
|
6374
6934
|
if (explicitKey !== undefined) {
|
|
6375
6935
|
return explicitKey;
|
|
6376
6936
|
}
|
|
6937
|
+
const formed = this.formedPartyKeys.get(strand.Id);
|
|
6938
|
+
if (formed !== undefined) {
|
|
6939
|
+
this.formedPartyKeys.delete(strand.Id);
|
|
6940
|
+
return formed;
|
|
6941
|
+
}
|
|
6377
6942
|
if (!this.controlDatabase) {
|
|
6378
6943
|
return undefined;
|
|
6379
6944
|
}
|
|
@@ -6398,8 +6963,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
6398
6963
|
* Counterpart to {@link createOpenInvitation}, which only mints the
|
|
6399
6964
|
* out-of-band {@link OpenInvitation} envelope: persisting the matching
|
|
6400
6965
|
* `FormationInvite` row is what makes the token *redeemable* — the host's
|
|
6401
|
-
* {@link ControlFormationUsageRecorder}
|
|
6402
|
-
*
|
|
6966
|
+
* {@link ControlFormationUsageRecorder} reads it (`readToken`) to check a
|
|
6967
|
+
* redemption. A host minting a closed-strand invite does both (mint +
|
|
6403
6968
|
* publish), exactly as the integration harness's `createInvitation` does.
|
|
6404
6969
|
*
|
|
6405
6970
|
* Signs through the same owner signer as {@link publishStrand}, whose key must be an
|
|
@@ -6422,16 +6987,21 @@ export class CadreNode implements SAppIdLookup {
|
|
|
6422
6987
|
const signer = this.requireOwnerSigner(`publish formation invite ${token}`);
|
|
6423
6988
|
await withinOwnerApproval(signer, { partyId: this.partyId, action: `Publish an invitation to the app ${sAppId}` }, sign,
|
|
6424
6989
|
(approved) => this.controlDatabase!.insertFormationInvite(token, sAppId, signer.publicKey, approved, options));
|
|
6425
|
-
|
|
6426
|
-
// it locally opens the connection gate's formation exemption immediately
|
|
6427
|
-
// rather than waiting for the durable row to become readable.
|
|
6428
|
-
this.strandSolicitationService?.registerMintedInvitation(
|
|
6429
|
-
token,
|
|
6430
|
-
options.expiresAtMs ?? Number.POSITIVE_INFINITY
|
|
6431
|
-
);
|
|
6990
|
+
this.registerPublishedInvitation(token, options.expiresAtMs);
|
|
6432
6991
|
log('Published formation invite %s (sApp %s) under owner %s', token, sAppId, signer.publicKey);
|
|
6433
6992
|
}
|
|
6434
6993
|
|
|
6994
|
+
/**
|
|
6995
|
+
* Register an invitation this node just published with the strand solicitation service. A
|
|
6996
|
+
* host may publish one whose token was minted elsewhere; registering it locally opens the
|
|
6997
|
+
* connection gate's formation exemption immediately rather than waiting for the durable row
|
|
6998
|
+
* to become readable. A token {@link createOpenInvitation} minted here is already registered,
|
|
6999
|
+
* and this records the same expiry again.
|
|
7000
|
+
*/
|
|
7001
|
+
private registerPublishedInvitation(token: string, expiresAtMs: number | undefined): void {
|
|
7002
|
+
this.strandSolicitationService?.registerMintedInvitation(token, expiresAtMs ?? Number.POSITIVE_INFINITY);
|
|
7003
|
+
}
|
|
7004
|
+
|
|
6435
7005
|
/**
|
|
6436
7006
|
* Enroll an approver public key allowed to sign off on `ValidationUrl` redemptions.
|
|
6437
7007
|
*
|
|
@@ -6547,7 +7117,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
6547
7117
|
* fan-out, so a rediscovery costs nothing beyond the map lookup plus the (pure)
|
|
6548
7118
|
* founder derivation. NOT a silent no-op any more when the launch resolves as a
|
|
6549
7119
|
* FOUNDER: the tracked instance may have been launched first as a joiner (an app
|
|
6550
|
-
* attach, or this node's own watcher
|
|
7120
|
+
* attach, or this node's own watcher run winning the `resolveCohortSeed` window
|
|
6551
7121
|
* below), which used to drop the founder request and leave the strand headerless —
|
|
6552
7122
|
* now the tracked instance is founded in place
|
|
6553
7123
|
* ({@link StrandInstanceManager.foundExistingStrand}), waking it first if quiesced
|
|
@@ -6559,9 +7129,9 @@ export class CadreNode implements SAppIdLookup {
|
|
|
6559
7129
|
*
|
|
6560
7130
|
* An absent `sAppConfig` launches a storage replica ({@link CadreNodeConfig.hostUnclaimedStrands}),
|
|
6561
7131
|
* which is ALWAYS a joiner whatever the row says: the founder bootstrap writes the sApp
|
|
6562
|
-
* into `Strand.Header`, and a replica has none. So a self-founded row whose watcher
|
|
7132
|
+
* into `Strand.Header`, and a replica has none. So a self-founded row whose watcher run
|
|
6563
7133
|
* wins the race against its app's `addStrand` after a restart comes up as a replica — and
|
|
6564
|
-
* so does a `foundStrand` whose publish a watcher
|
|
7134
|
+
* so does a `foundStrand` whose publish a watcher run saw before its attach. A claim
|
|
6565
7135
|
* that finds a tracked replica upgrades it in place, over the same node and store
|
|
6566
7136
|
* ({@link StrandInstanceManager.attachSApp}), BEFORE any founding: the founder bootstrap
|
|
6567
7137
|
* needs the sApp the attach supplies.
|
|
@@ -6604,8 +7174,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
6604
7174
|
|
|
6605
7175
|
// A closed strand's launch carries the party's OWN membership identity key: the
|
|
6606
7176
|
// explicit attach-time key, else the control-layer StrandPartyKey row — minted here
|
|
6607
|
-
// when this machine is the row-derived founder and no row exists yet (a
|
|
6608
|
-
//
|
|
7177
|
+
// when this machine is the row-derived founder and no row exists yet (a closed row
|
|
7178
|
+
// stored without its key). A joiner with no persisted key threads undefined;
|
|
6609
7179
|
// only a FOUNDER bootstrap needs the key, and that path throws loudly without one
|
|
6610
7180
|
// (StrandDatabase).
|
|
6611
7181
|
const partyMemberPrivateKey = strand.Type === 'c'
|
|
@@ -6948,7 +7518,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
6948
7518
|
*
|
|
6949
7519
|
* Timestamps are recorded OPTIMISTICALLY, whatever `collectStrandAddrs` reports
|
|
6950
7520
|
* per peer: recording only on success would re-announce to an unreachable relay
|
|
6951
|
-
* on every reconcile
|
|
7521
|
+
* on every reconcile pass, one dial timeout each. A failed INITIAL announce costs
|
|
6952
7522
|
* the strand supervisor its first attempt only (the relay denies the
|
|
6953
7523
|
* reservation; every re-drive re-announces first through
|
|
6954
7524
|
* {@link announceDelegateToRelay}); a failed REFRESH retries within
|
|
@@ -7092,9 +7662,11 @@ export class CadreNode implements SAppIdLookup {
|
|
|
7092
7662
|
*
|
|
7093
7663
|
* Due times are per (sibling, strand) and set from each sibling's own outcome, so
|
|
7094
7664
|
* a sibling that connects late — a phone joining after the party's always-on
|
|
7095
|
-
* machines — is asked on the
|
|
7096
|
-
* (its view of the membership may not include us yet) is retried
|
|
7097
|
-
* {@link STRAND_PEER_ADDR_RETRY_MS} rather than
|
|
7665
|
+
* machines — is asked on the pass its connection starts, and one that could not
|
|
7666
|
+
* answer or refused (its view of the membership may not include us yet) is retried
|
|
7667
|
+
* within {@link STRAND_PEER_ADDR_RETRY_MS} rather than
|
|
7668
|
+
* {@link STRAND_PEER_ADDR_REFRESH_MS}; the pass that stamped the retry arms a wake
|
|
7669
|
+
* for it ({@link armReconcileWake}).
|
|
7098
7670
|
*
|
|
7099
7671
|
* Distinct from {@link refreshDelegateGrants}, deliberately: that pass covers
|
|
7100
7672
|
* RELAYS on a `DELEGATE_GRANT_TTL_MS / 2` throttle to keep circuit-relay
|
|
@@ -7144,15 +7716,17 @@ export class CadreNode implements SAppIdLookup {
|
|
|
7144
7716
|
*
|
|
7145
7717
|
* `connectedSiblingTargets`' membership read is unbounded, and with zero
|
|
7146
7718
|
* connections its answer is empty whatever the table holds, so decide it from the
|
|
7147
|
-
* connection list instead of paying for the read once a
|
|
7719
|
+
* connection list instead of paying for the read once a pass.
|
|
7148
7720
|
*
|
|
7149
|
-
* NOTE: otherwise the read runs on EVERY
|
|
7721
|
+
* NOTE: otherwise the read runs on EVERY pass with a running strand, not only when
|
|
7150
7722
|
* some (sibling, strand) is due: pruning a departed sibling needs the current
|
|
7151
7723
|
* target set, and whether anyone is due cannot be told without it (a connected
|
|
7152
|
-
* non-member never gets a due time to compare). One more `CadrePeer` read per
|
|
7153
|
-
*
|
|
7154
|
-
*
|
|
7155
|
-
*
|
|
7724
|
+
* non-member never gets a due time to compare). One more `CadrePeer` read per
|
|
7725
|
+
* pass, on top of the two `runReconcileControlCohort` already makes, all served
|
|
7726
|
+
* from the membership rows view: an in-memory scan while the view is current,
|
|
7727
|
+
* storage only after the table changed or the view's maximum age passed. Sharing
|
|
7728
|
+
* one row set across the pass would save no storage read and was declined (see
|
|
7729
|
+
* the NOTE there).
|
|
7156
7730
|
*
|
|
7157
7731
|
* NOTE: one strand-addr RPC per (running strand × due sibling) — each a tiny
|
|
7158
7732
|
* request/response on an already-open control connection. If a node ever runs
|
|
@@ -7243,8 +7817,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
7243
7817
|
*
|
|
7244
7818
|
* NOTE: every pass serialises the whole FRET table (`exportTable()`), verifies one
|
|
7245
7819
|
* signature per record and makes one `peerStore.merge` per peer, per running strand
|
|
7246
|
-
* per
|
|
7247
|
-
* nothing in libp2p's persistent peer store. If many strands or large tables make it
|
|
7820
|
+
* per pass — every change-driven pass and the 2-minute backstop. Fine at strand table
|
|
7821
|
+
* sizes, and an unchanged address set writes nothing in libp2p's persistent peer store. If many strands or large tables make it
|
|
7248
7822
|
* show up, read only the entries whose address stamp is near the one-hour mark.
|
|
7249
7823
|
*
|
|
7250
7824
|
* @returns the ids of the peers the table holds a usable record for.
|
|
@@ -7550,8 +8124,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
7550
8124
|
* The stop + emit are skipped when the strand manager holds no instance for `strandId` —
|
|
7551
8125
|
* e.g. a party owner that published a strand's row but never ran it locally, or an
|
|
7552
8126
|
* explicit {@link stopStrand} for an id this node never started. `hibernationManager`
|
|
7553
|
-
* untrack and the `sAppConfigs` / {@link strandLaunchRefusals} / {@link discoveredStrands}
|
|
7554
|
-
* deletes stay unconditional (all no-ops when there is nothing to remove), so a launch
|
|
8127
|
+
* untrack and the `sAppConfigs` / {@link strandLaunchRefusals} / {@link discoveredStrands} /
|
|
8128
|
+
* {@link formedPartyKeys} deletes stay unconditional (all no-ops when there is nothing to remove), so a launch
|
|
7555
8129
|
* that failed before an instance was ever tracked still gets its stray entries cleared.
|
|
7556
8130
|
*
|
|
7557
8131
|
* Dropping the {@link discoveredStrands} entry here covers both callers, and both
|
|
@@ -7568,6 +8142,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
7568
8142
|
this.sAppConfigs.delete(strandId);
|
|
7569
8143
|
this.strandLaunchRefusals.delete(strandId);
|
|
7570
8144
|
this.discoveredStrands.delete(strandId);
|
|
8145
|
+
this.formedPartyKeys.delete(strandId);
|
|
7571
8146
|
if (!this.strandManager.hasStrand(strandId)) {
|
|
7572
8147
|
log('detachStrand: strand %s not tracked locally — no-op (no stop, no strand:stopped)', strandId);
|
|
7573
8148
|
return;
|
|
@@ -7956,21 +8531,37 @@ export class CadreNode implements SAppIdLookup {
|
|
|
7956
8531
|
}
|
|
7957
8532
|
|
|
7958
8533
|
/**
|
|
7959
|
-
* Open control-network connections right now
|
|
7960
|
-
*
|
|
7961
|
-
*
|
|
7962
|
-
*
|
|
7963
|
-
* It approximates the precise signal (the block's cluster size), and a caller that
|
|
7964
|
-
* samples it AFTER a write samples a slightly wider window than `committedAlone`
|
|
7965
|
-
* does inside the write itself. It exists so an embedder can warn an owner before a
|
|
7966
|
-
* control write that this machine currently sees none of its siblings, and report
|
|
7967
|
-
* after one that the write may not have travelled — a warning shown unconditionally
|
|
7968
|
-
* is a warning people learn to ignore.
|
|
8534
|
+
* Open control-network connections right now, of every kind: siblings, relays, bootstrap
|
|
8535
|
+
* nodes and peers still identifying. A report of connectivity, not of replication reach: a
|
|
8536
|
+
* control write can commit local-only with connections open, and whether one would is
|
|
8537
|
+
* {@link hasReachableControlSibling}.
|
|
7969
8538
|
*/
|
|
7970
8539
|
getControlConnectionCount(): number {
|
|
7971
8540
|
return this.controlNode?.getConnections().length ?? 0;
|
|
7972
8541
|
}
|
|
7973
8542
|
|
|
8543
|
+
/**
|
|
8544
|
+
* Whether a control write made now reaches another machine: a connected peer has identified
|
|
8545
|
+
* as serving the control database and is one this node sends control writes to (an
|
|
8546
|
+
* authorized member, or a device whose admission it is writing). Optimystic counts a peer
|
|
8547
|
+
* into a control block's cohort on those same terms; with none, a write commits on this node
|
|
8548
|
+
* alone and is queued for re-issue once one is reachable (docs/architecture.md → "Writes made
|
|
8549
|
+
* while alone").
|
|
8550
|
+
*
|
|
8551
|
+
* It exists so an embedder can warn an owner before a control write that this machine
|
|
8552
|
+
* currently reaches none of its siblings, and report after one that the write may not have
|
|
8553
|
+
* travelled; a warning shown unconditionally is a warning people learn to ignore. A caller
|
|
8554
|
+
* that samples it AFTER a write samples a slightly wider window than the write did.
|
|
8555
|
+
*/
|
|
8556
|
+
hasReachableControlSibling(): boolean {
|
|
8557
|
+
for (const peerId of this.identifiedControlPeers) {
|
|
8558
|
+
if (this.isControlCohortMember(peerId)) {
|
|
8559
|
+
return true;
|
|
8560
|
+
}
|
|
8561
|
+
}
|
|
8562
|
+
return false;
|
|
8563
|
+
}
|
|
8564
|
+
|
|
7974
8565
|
/**
|
|
7975
8566
|
* Force a poll of the strand watcher (for testing)
|
|
7976
8567
|
*/
|
|
@@ -8052,7 +8643,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
8052
8643
|
}), this.controlNode, this.controlDatabase);
|
|
8053
8644
|
log('Owner signer wired (owner key %s)', signer.publicKey);
|
|
8054
8645
|
// Embedders wire the owner key after start(), so the runner's first pass usually found
|
|
8055
|
-
// this machine not yet an owner;
|
|
8646
|
+
// this machine not yet an owner; wiring a signer writes no table it watches, so without
|
|
8647
|
+
// this it would wait for its backstop.
|
|
8056
8648
|
this.pendingJoinRunner?.kick();
|
|
8057
8649
|
}
|
|
8058
8650
|
|
|
@@ -8290,6 +8882,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
8290
8882
|
if (usage === undefined || invite === undefined) {
|
|
8291
8883
|
return false;
|
|
8292
8884
|
}
|
|
8885
|
+
// NOTE: accepted tradeoff — three ed25519 verifies per invitation-admitted row on every
|
|
8886
|
+
// refresh, CPU only since the rows come from the membership views; a memo of verdicts would
|
|
8887
|
+
// also have to follow the trusted-owner anchor, which syncOwnerAnchor rewrites. Revisit if
|
|
8888
|
+
// the gate's admission deadline ever shows it.
|
|
8293
8889
|
return verifyInvitationAdmission(this.partyId, row, usage, invite, key => this.trustedOwnerStore?.has(key) ?? false);
|
|
8294
8890
|
}
|
|
8295
8891
|
|
|
@@ -8313,7 +8909,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
8313
8909
|
* builds its snapshot against that anchor, which is also what its previous snapshot was
|
|
8314
8910
|
* judged against. Caught here rather than failing the refresh, because the membership read
|
|
8315
8911
|
* tolerates an unreachable `Revocation` block on an isolated node
|
|
8316
|
-
* (`ControlDatabase.
|
|
8912
|
+
* (`ControlDatabase.readRevocationRowsOrNone`) and `queryRevocations` does not: failing the
|
|
8317
8913
|
* whole refresh on that would freeze such a node's gate snapshot.
|
|
8318
8914
|
* NOTE: on such a node the sync is skipped at every refresh until the block is held; if an
|
|
8319
8915
|
* isolated node ever has to derive owner additions while cut off, give `queryRevocations`
|
|
@@ -8322,9 +8918,12 @@ export class CadreNode implements SAppIdLookup {
|
|
|
8322
8918
|
* NOTE: cost per membership refresh is one `OwnerKey` read and one `Revocation` read (the
|
|
8323
8919
|
* tombstone list, whose `OwnerKey` stamps are handed to `queryOwnerKeyRows` so it does not
|
|
8324
8920
|
* read the retired set again, and which the hang-up of removed peers shares through
|
|
8325
|
-
* `loadTombstones`), plus the owner-grant chain when some owner row is
|
|
8326
|
-
*
|
|
8327
|
-
*
|
|
8921
|
+
* `loadTombstones`), plus the owner-grant chain's two reads when some owner row is
|
|
8922
|
+
* grant-seated. All are served from the control database's membership views, so they reach
|
|
8923
|
+
* storage only after their table changed or the view's maximum age passed
|
|
8924
|
+
* (`control-founding-consult-budget.spec.ts` pins the counts). On a party founded before
|
|
8925
|
+
* genesis created every control collection, whose `Revocation` table has never been
|
|
8926
|
+
* written, each `Revocation` load is a cohort consult until the first tombstone.
|
|
8328
8927
|
*
|
|
8329
8928
|
* @param retry - forwarded to every read; `false` on the refresh path, for the reason on
|
|
8330
8929
|
* {@link refreshAuthorizedControlPeers}.
|
|
@@ -8345,9 +8944,6 @@ export class CadreNode implements SAppIdLookup {
|
|
|
8345
8944
|
const tombstones = await loadTombstones();
|
|
8346
8945
|
const retired = new Set(tombstones.filter(tombstone => tombstone.tableName === 'OwnerKey').map(tombstone => tombstone.stampId));
|
|
8347
8946
|
const rows = await controlDatabase.queryOwnerKeyRows(retry, retired);
|
|
8348
|
-
// NOTE: the grant chain is two whole-table reads per membership refresh while any owner
|
|
8349
|
-
// row is grant-seated, over tables that are append-only; the memo suggested on
|
|
8350
|
-
// loadInvitationChain would cover it too.
|
|
8351
8947
|
const grantChain = rows.some(isConsentSeated) ? await this.loadOwnerGrantChain(controlDatabase, retry) : null;
|
|
8352
8948
|
const derivation = deriveOwnerAnchor({ partyId: this.partyId, anchor: before, rows, tombstones, grantChain });
|
|
8353
8949
|
if (derivation.refusedRemovals.length > 0) {
|
|
@@ -8413,15 +9009,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
8413
9009
|
/**
|
|
8414
9010
|
* The usage and invitation rows {@link hasAnchoredProof} resolves an invitation-admitted
|
|
8415
9011
|
* `CadrePeer` row through, keyed for lookup. Both reads honour `retry` for
|
|
8416
|
-
* {@link listAuthorizedMembers}' reason.
|
|
8417
|
-
*
|
|
8418
|
-
* NOTE: once invitations are the only admission route (ticket
|
|
8419
|
-
* `cadre-invitations-redeemable-by-any-member`), every cadre with a member not yet
|
|
8420
|
-
* re-vouched pays these two whole-table reads and three ed25519 verifies per row on every
|
|
8421
|
-
* membership refresh. Both tables are append-only and a verified chain never changes (expiry,
|
|
8422
|
-
* use count and withdrawal are not re-checked), so if the gate's admission deadline ever
|
|
8423
|
-
* shows this, memoise verified (row stamp → usage stamp) pairs per node and skip the reads
|
|
8424
|
-
* when every admitted row is already memoised.
|
|
9012
|
+
* {@link listAuthorizedMembers}' reason. The rows come from the membership views, so they
|
|
9013
|
+
* reach storage only after the table changed or the view's maximum age passed.
|
|
8425
9014
|
*/
|
|
8426
9015
|
private async loadInvitationChain(controlDatabase: ControlDatabase, retry: boolean): Promise<InvitationChain> {
|
|
8427
9016
|
const [usages, invites] = await Promise.all([
|
|
@@ -8587,16 +9176,16 @@ export class CadreNode implements SAppIdLookup {
|
|
|
8587
9176
|
* write-driven refresh never fires for it — yet applying it can ANCHOR a new
|
|
8588
9177
|
* owner key, which flips rows already present from unauthorized to authorized.
|
|
8589
9178
|
* Hence the explicit refresh here; without it a peer the freshly anchored owner
|
|
8590
|
-
* vouched for stays denied until the next
|
|
9179
|
+
* vouched for stays denied until the next membership change or the backstop pass.
|
|
8591
9180
|
*/
|
|
8592
9181
|
private seedEventCallbacks(): SeedEventCallbacks {
|
|
8593
9182
|
return {
|
|
8594
9183
|
onSeedReceived: (partyId, peerId) => this.emit('seed:received', { partyId, peerId }),
|
|
8595
9184
|
onSeedApplied: (partyId, peersAdded, seed) => {
|
|
8596
|
-
this.emit('seed:applied', { partyId, peersAdded });
|
|
8597
9185
|
// A wire-delivered seed never passes through the `applySeed` wrapper, so
|
|
8598
9186
|
// this is where its preferred peers become cold-start bootstrap targets.
|
|
8599
|
-
this.recordSeedBootstrapPeers(seed);
|
|
9187
|
+
const dialTargets = this.recordSeedBootstrapPeers(seed);
|
|
9188
|
+
this.emit('seed:applied', { partyId, peersAdded, dialTargets });
|
|
8600
9189
|
void this.refreshMembershipGate('seed-applied');
|
|
8601
9190
|
},
|
|
8602
9191
|
onSeedError: (partyId, error) => this.emit('seed:error', { partyId, error }),
|
|
@@ -8622,13 +9211,14 @@ export class CadreNode implements SAppIdLookup {
|
|
|
8622
9211
|
* NO `CadrePeer` writer needs to call this: the control database notifies it
|
|
8623
9212
|
* after every committed member-row write (`ControlDatabase.mutateCadrePeer`,
|
|
8624
9213
|
* wired in {@link start}), which is exactly what makes the refresh automatic
|
|
8625
|
-
* rather than a caller obligation.
|
|
8626
|
-
*
|
|
8627
|
-
*
|
|
8628
|
-
*
|
|
8629
|
-
*
|
|
8630
|
-
*
|
|
8631
|
-
*
|
|
9214
|
+
* rather than a caller obligation. A membership row that arrived by REPLICATION
|
|
9215
|
+
* refreshes it too, through the gate watch {@link start} takes on the membership
|
|
9216
|
+
* tables (within its 50 ms window; a change this node's storage never hears of
|
|
9217
|
+
* waits for the reconcile pass's backstop). It stays public for the one change
|
|
9218
|
+
* that writes no row and raises no table event: a newly anchored trusted owner
|
|
9219
|
+
* key ({@link applySeed}), which flips rows ALREADY present from unauthorized to
|
|
9220
|
+
* authorized without touching them. A caller may also await it to observe a
|
|
9221
|
+
* refresh that began after its own change.
|
|
8632
9222
|
*
|
|
8633
9223
|
* Coalescing: marks the snapshot stale and resolves once a refresh that began
|
|
8634
9224
|
* after this call has completed, so a caller that awaits it always observes
|
|
@@ -8654,13 +9244,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
8654
9244
|
* connected peer it newly covers — the strand-side counterpart to
|
|
8655
9245
|
* {@link refreshMembershipGate}.
|
|
8656
9246
|
*
|
|
8657
|
-
*
|
|
8658
|
-
*
|
|
8659
|
-
*
|
|
8660
|
-
*
|
|
8661
|
-
* The gate therefore polls (default 30 s). An app that has just called
|
|
8662
|
-
* `revokeMember` or `leaveStrand` should follow the write with this call so
|
|
8663
|
-
* the cut is immediate rather than up to one poll interval late.
|
|
9247
|
+
* The gate already refreshes on its own when the strand's `Member` or
|
|
9248
|
+
* `MemberPeer` table changes: at once for a `revokeMember` or `leaveStrand`
|
|
9249
|
+
* written on this machine, and when a replicated removal reaches this node's
|
|
9250
|
+
* storage. Call this to await the cut — it resolves after the sweep.
|
|
8664
9251
|
*
|
|
8665
9252
|
* Quiet no-op for a strand that is not running, is quiesced, is open, or has
|
|
8666
9253
|
* the gate disabled. Never rejects; resolves once the sweep has finished.
|
|
@@ -8711,8 +9298,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
8711
9298
|
* CONCURRENT external write's refresh for the life of the scope: that writer's
|
|
8712
9299
|
* promise resolves before its peer is admitted, and admission lands at scope
|
|
8713
9300
|
* exit instead. Acceptable — scope exit is milliseconds away (bounded by the
|
|
8714
|
-
* drain), versus the
|
|
8715
|
-
*
|
|
9301
|
+
* drain), versus the reconcile pass that was the alternative before any of this
|
|
9302
|
+
* existed — but it is the reason these scopes stay short and rare.
|
|
8716
9303
|
*/
|
|
8717
9304
|
private async deferMembershipGateRefresh<T>(reason: string, body: () => Promise<T>): Promise<T> {
|
|
8718
9305
|
this.membershipGateDeferDepth++;
|
|
@@ -8854,7 +9441,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
8854
9441
|
throw new Error('CadreNode must be started before planning an owner key removal');
|
|
8855
9442
|
}
|
|
8856
9443
|
const key = requireEd25519PublicKeyB64(ownerPublicKeyB64, 'owner key');
|
|
8857
|
-
return
|
|
9444
|
+
return await this.planOwnerKeyRemovalWith(this.controlDatabase, key);
|
|
8858
9445
|
}
|
|
8859
9446
|
|
|
8860
9447
|
/**
|
|
@@ -8888,7 +9475,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
8888
9475
|
throw new Error('Seed bootstrap service not initialized. Call initializeSeedBootstrap() first.');
|
|
8889
9476
|
}
|
|
8890
9477
|
const key = requireEd25519PublicKeyB64(ownerPublicKeyB64, 'owner key');
|
|
8891
|
-
const
|
|
9478
|
+
const plan = await this.planOwnerKeyRemovalWith(this.controlDatabase, key);
|
|
8892
9479
|
if (plan.refusal !== null) {
|
|
8893
9480
|
throw new OwnerKeyRemovalRefusedError(key, plan.refusal);
|
|
8894
9481
|
}
|
|
@@ -8903,19 +9490,19 @@ export class CadreNode implements SAppIdLookup {
|
|
|
8903
9490
|
}
|
|
8904
9491
|
const action = `Remove the owner key ${namedOwnerKey(key, plan.ownerKey?.label ?? null)} from the cadre`;
|
|
8905
9492
|
const removed = await signer.approve({ partyId: this.partyId, action }, async (sign) => {
|
|
8906
|
-
await this.revouchDependents(service, revouch,
|
|
9493
|
+
await this.revouchDependents(service, revouch, sign);
|
|
8907
9494
|
return await service.removeOwner(key, sign);
|
|
8908
9495
|
});
|
|
8909
9496
|
log('removeOwner(%s): removed=%s revouched owners=%d peers=%d', key, removed, revouch.owners.length, revouch.peers.length);
|
|
8910
9497
|
return { removed, revouched: revouch };
|
|
8911
9498
|
}
|
|
8912
9499
|
|
|
8913
|
-
/** {@link planOwnerKeyRemoval}
|
|
8914
|
-
private async planOwnerKeyRemovalWith(controlDatabase: ControlDatabase, key: string): Promise<
|
|
9500
|
+
/** {@link planOwnerKeyRemoval} over `controlDatabase`, for a caller that already holds it. */
|
|
9501
|
+
private async planOwnerKeyRemovalWith(controlDatabase: ControlDatabase, key: string): Promise<OwnerKeyRemovalPlan> {
|
|
8915
9502
|
const [rows, owners, grants] = await Promise.all([
|
|
8916
9503
|
controlDatabase.queryCadrePeers(), controlDatabase.queryOwnerKeyRows(), controlDatabase.listOwnerGrantStatuses()
|
|
8917
9504
|
]);
|
|
8918
|
-
|
|
9505
|
+
return planOwnerKeyRemoval({
|
|
8919
9506
|
key,
|
|
8920
9507
|
rows,
|
|
8921
9508
|
owners,
|
|
@@ -8924,7 +9511,6 @@ export class CadreNode implements SAppIdLookup {
|
|
|
8924
9511
|
selfOwnerKey: this.ownerSignerEnrolledIn(new Set(owners.map((owner) => owner.key)))?.publicKey ?? null,
|
|
8925
9512
|
anchor: this.trustedOwnerStore?.all() ?? new Set(),
|
|
8926
9513
|
});
|
|
8927
|
-
return { plan, rows };
|
|
8928
9514
|
}
|
|
8929
9515
|
|
|
8930
9516
|
/**
|
|
@@ -8934,12 +9520,14 @@ export class CadreNode implements SAppIdLookup {
|
|
|
8934
9520
|
* the only rebinding `OwnerKey.NoUpdate` allows); peers through the owner re-touch
|
|
8935
9521
|
* (`reauthorizePeer`), which rewrites `VouchOwner`/`VouchSig` and clears `VouchUsage`.
|
|
8936
9522
|
*
|
|
8937
|
-
* The peer re-touch
|
|
8938
|
-
* `
|
|
8939
|
-
*
|
|
8940
|
-
*
|
|
8941
|
-
*
|
|
8942
|
-
*
|
|
9523
|
+
* The peer re-touch is the re-vouch mode of `reauthorizePeer`, which leaves the row's
|
|
9524
|
+
* `UpdatedAt` untouched and writes whether or not the machine has self-published. The owner
|
|
9525
|
+
* branch of `CadrePeer.AuthorizedUpdate` does not require a bump, and bumping a self-published
|
|
9526
|
+
* row would leave its `Sig` unverifiable, so unresolvable and unlisted as self-published, until
|
|
9527
|
+
* the machine's next heartbeat (the NOTE on that constraint); the write-while-alone drain's
|
|
9528
|
+
* re-broadcast mode bumps because it re-issues an otherwise unchanged row, whereas here the
|
|
9529
|
+
* voucher itself changes. A dependent whose row vanished since the plan is reported absent and
|
|
9530
|
+
* skipped.
|
|
8943
9531
|
*
|
|
8944
9532
|
* A failure rethrows and stops the call before the removal; re-vouches already made stay
|
|
8945
9533
|
* (the row is only re-signed by a current owner) and the caller retries. An owner re-vouch
|
|
@@ -8953,7 +9541,6 @@ export class CadreNode implements SAppIdLookup {
|
|
|
8953
9541
|
private async revouchDependents(
|
|
8954
9542
|
service: SeedBootstrapService,
|
|
8955
9543
|
revouch: OwnerKeyDependents,
|
|
8956
|
-
rows: readonly CadrePeerRow[],
|
|
8957
9544
|
sign: OwnerSign,
|
|
8958
9545
|
): Promise<void> {
|
|
8959
9546
|
for (const key of revouch.owners) {
|
|
@@ -8964,9 +9551,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
8964
9551
|
}
|
|
8965
9552
|
}
|
|
8966
9553
|
for (const dependent of revouch.peers) {
|
|
8967
|
-
|
|
8968
|
-
const updatedAt = row !== undefined && row.updatedAt > 0 ? row.updatedAt : Date.now();
|
|
8969
|
-
await service.reauthorizePeer(dependent, updatedAt, sign);
|
|
9554
|
+
await service.reauthorizePeer(dependent, 'revouch', sign);
|
|
8970
9555
|
}
|
|
8971
9556
|
}
|
|
8972
9557
|
|
|
@@ -8993,6 +9578,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
8993
9578
|
throw new Error('CadreNode must be started before planning a machine removal');
|
|
8994
9579
|
}
|
|
8995
9580
|
const machines = await this.readMachines(this.controlDatabase);
|
|
9581
|
+
// NOTE: the invitation and usage lists come from the membership views, so on a node that
|
|
9582
|
+
// heard of a row with no storage event (outside the block's cohort) an invitation issued
|
|
9583
|
+
// elsewhere in the last MEMBERSHIP_VIEW_MAX_AGE_MS can be missing and left unwithdrawn. If
|
|
9584
|
+
// removal must close that window, read both tables fresh here.
|
|
8996
9585
|
const [invites, usages, owners] = await Promise.all([
|
|
8997
9586
|
this.controlDatabase.listCadreInviteStatuses(),
|
|
8998
9587
|
this.controlDatabase.queryCadreInviteUsages(),
|
|
@@ -9224,12 +9813,13 @@ export class CadreNode implements SAppIdLookup {
|
|
|
9224
9813
|
* that connection from the addresses it was handed, on this launch or a later
|
|
9225
9814
|
* one.
|
|
9226
9815
|
*
|
|
9227
|
-
* Nothing is dialed here: the drone has not received the seed yet.
|
|
9228
|
-
*
|
|
9229
|
-
*
|
|
9230
|
-
*
|
|
9231
|
-
*
|
|
9232
|
-
*
|
|
9816
|
+
* Nothing is dialed here: the drone has not received the seed yet. Its `CadrePeer`
|
|
9817
|
+
* insert is a membership change, so a reconcile pass dials the retained addresses
|
|
9818
|
+
* within about 50 ms of this call returning, whether or not the caller runs one, and
|
|
9819
|
+
* a failed dial is retried on the redial ladder (15 s, 30 s, 60 s, then every
|
|
9820
|
+
* backstop). A caller that wants to await the connection calls
|
|
9821
|
+
* {@link reconcileControlCohort} itself, twice in a row: a call made while a pass is
|
|
9822
|
+
* in flight joins it, and that pass may have listed siblings before this add.
|
|
9233
9823
|
*/
|
|
9234
9824
|
async addDrone(options: AddDroneOptions): Promise<DroneInitResult> {
|
|
9235
9825
|
if (!this.seedBootstrapService) {
|
|
@@ -9275,12 +9865,19 @@ export class CadreNode implements SAppIdLookup {
|
|
|
9275
9865
|
* by the node (its anchored branch), the row insert is a no-op on an existing row
|
|
9276
9866
|
* (`ControlDatabase.insertCadrePeer`), and the dial target is replaced, not duplicated.
|
|
9277
9867
|
*
|
|
9278
|
-
*
|
|
9279
|
-
*
|
|
9280
|
-
*
|
|
9281
|
-
*
|
|
9282
|
-
*
|
|
9283
|
-
*
|
|
9868
|
+
* Once it has acknowledged, the node restarts into this party (cadre-cli's
|
|
9869
|
+
* `restartIntoClaimedParty`), closing the connection the claim used. The dial target puts
|
|
9870
|
+
* the address that connection was dialed on first, so every later dial tries the address
|
|
9871
|
+
* that just worked before the others (a QR code lists public addresses before LAN ones, and
|
|
9872
|
+
* a home router often drops a connection to its own public address from inside). A
|
|
9873
|
+
* reconcile pass is kicked (not awaited) now, and the vouch's `CadrePeer` insert starts
|
|
9874
|
+
* another about 50 ms later; for the first {@link CONTROL_COHORT_REDIAL_MS} after the
|
|
9875
|
+
* acknowledgement this node also retries the node every second while it restarts
|
|
9876
|
+
* ({@link redialClaimedNode}). A caller that wants to wait for the connection awaits
|
|
9877
|
+
* {@link reconcileControlCohort} itself, with `addDrone`'s caveat that a pass already in
|
|
9878
|
+
* flight is joined rather than restarted. The claimed node dials this one back too, as the
|
|
9879
|
+
* seed's minter, when this node has a listen or relay address; a phone with neither is
|
|
9880
|
+
* reached by its own dials alone.
|
|
9284
9881
|
*
|
|
9285
9882
|
* @throws {ClaimRefusedError} when the node was reached and refused the claim.
|
|
9286
9883
|
* @throws {PeerUnreachableError} when the node was never reached.
|
|
@@ -9295,6 +9892,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
9295
9892
|
if (!signer) {
|
|
9296
9893
|
throw new Error('Owner key required to create seeds');
|
|
9297
9894
|
}
|
|
9895
|
+
let dialAddrs = target.multiaddrs;
|
|
9298
9896
|
await signer.approve({ partyId: this.partyId, action: `Claim the node ${target.peerId} for the cadre` }, async (sign) => {
|
|
9299
9897
|
const seed = await service.createSeed(sign);
|
|
9300
9898
|
const proof = claimProof(secret, target.peerId, seed.signerKey, seedDigest(seed));
|
|
@@ -9303,14 +9901,91 @@ export class CadreNode implements SAppIdLookup {
|
|
|
9303
9901
|
if (!ack.accepted) {
|
|
9304
9902
|
throw new ClaimRefusedError(target.peerId, ack.reason ?? 'no reason given', ack.code);
|
|
9305
9903
|
}
|
|
9904
|
+
// Read before anything else is awaited: the node's restart closes the connection.
|
|
9905
|
+
dialAddrs = this.connectedAddrFirst(target.peerId, target.multiaddrs);
|
|
9306
9906
|
log('claimNode: %s accepted the claim; vouching it into the cadre', target.peerId);
|
|
9307
9907
|
// The wrapper queues the write for re-replication: a phone claiming its first
|
|
9308
9908
|
// always-on node has no control connection yet, so the insert commits local-only.
|
|
9909
|
+
// NOTE: accepted tradeoff — the node's own signed record is not carried in the
|
|
9910
|
+
// acknowledgement and written with this vouch: it would be signed before the node
|
|
9911
|
+
// restarts, so its addresses may not be the restarted node's (a new relay reservation),
|
|
9912
|
+
// and the write-while-alone drain re-sends only rows whose `Sig` is null
|
|
9913
|
+
// (drainPendingPeerWrites, reconstructAuthoredMembership), so a vouch carrying it would
|
|
9914
|
+
// never reach this cadre's other members. The node publishes its record once it has
|
|
9915
|
+
// restarted. Revisit if the drain ever re-sends signed rows.
|
|
9309
9916
|
await this.vouchPeer(target.peerId, target.multiaddrs, sign);
|
|
9310
9917
|
});
|
|
9311
|
-
this.retainDialTarget(target.peerId,
|
|
9918
|
+
this.retainDialTarget(target.peerId, dialAddrs, 'claimNode');
|
|
9312
9919
|
void this.reconcileControlCohort().catch((error) =>
|
|
9313
9920
|
log('claimNode: reconcile kick after claiming %s failed: %o', target.peerId, error));
|
|
9921
|
+
void this.redialClaimedNode(target.peerId).catch((error) =>
|
|
9922
|
+
log('claimNode: redialing %s while it restarts failed: %o', target.peerId, error));
|
|
9923
|
+
}
|
|
9924
|
+
|
|
9925
|
+
/**
|
|
9926
|
+
* `addrs` with the address this node's open connection to `peerId` was dialed on moved to
|
|
9927
|
+
* the front, when it is one of them; `addrs` unchanged otherwise. Compared without a
|
|
9928
|
+
* trailing `/p2p/<peerId>`, which a QR code's address may or may not carry ({@link addrKey}).
|
|
9929
|
+
*/
|
|
9930
|
+
private connectedAddrFirst(peerId: string, addrs: string[]): string[] {
|
|
9931
|
+
const controlNode = this.controlNode;
|
|
9932
|
+
if (!controlNode) {
|
|
9933
|
+
return addrs;
|
|
9934
|
+
}
|
|
9935
|
+
const remote = peerIdFromString(peerId);
|
|
9936
|
+
const connected = new Set(controlNode.getConnections(remote).map((c) => addrKey(c.remoteAddr.toString(), remote)));
|
|
9937
|
+
const index = addrs.findIndex((addr) => connected.has(addrKey(addr, remote)));
|
|
9938
|
+
return index <= 0 ? addrs : [addrs[index], ...addrs.slice(0, index), ...addrs.slice(index + 1)];
|
|
9939
|
+
}
|
|
9940
|
+
|
|
9941
|
+
/**
|
|
9942
|
+
* Retry a node this node has just claimed ({@link claimNode}) once every
|
|
9943
|
+
* {@link CLAIM_RESTART_REDIAL_STEP_MS} for {@link CONTROL_COHORT_REDIAL_MS} after it accepted.
|
|
9944
|
+
* The node restarts into this party right after it acknowledges, and while it does, a
|
|
9945
|
+
* reconcile pass's dial either fails or opens a connection the restarting node closes at
|
|
9946
|
+
* once. The first failure puts the pass's next redial 15 s out ({@link armReconcileWake}),
|
|
9947
|
+
* and an owner that cannot be dialed has no other way back to the node.
|
|
9948
|
+
*
|
|
9949
|
+
* Each step dials once from the retained dial target, unless a connection to the node is
|
|
9950
|
+
* open: the one the claim used until the restart closes it, or one to the restarted node
|
|
9951
|
+
* that has not identified yet. A dial that connects does not end the loop, because the
|
|
9952
|
+
* restarting node may still close it. The loop ends once the node has identified as serving
|
|
9953
|
+
* this party's control database ({@link identifiedControlPeers}, which the placeholder node
|
|
9954
|
+
* it ran before the claim never joins), at the window's end, when this node stops, or when
|
|
9955
|
+
* the target is forgotten ({@link removePeer}). Steps run one after another, so a dial
|
|
9956
|
+
* finishes on its own budget first. The redial streak is left alone: the ladder carries on
|
|
9957
|
+
* after the window. One loop per peer: a repeated claim while one runs starts none.
|
|
9958
|
+
*/
|
|
9959
|
+
private async redialClaimedNode(peerId: string): Promise<void> {
|
|
9960
|
+
if (this.claimRedialPeers.has(peerId)) {
|
|
9961
|
+
return;
|
|
9962
|
+
}
|
|
9963
|
+
this.claimRedialPeers.add(peerId);
|
|
9964
|
+
try {
|
|
9965
|
+
const endsAt = Date.now() + CONTROL_COHORT_REDIAL_MS;
|
|
9966
|
+
while (Date.now() < endsAt && await this.redialClaimedNodeStep(peerId)) {
|
|
9967
|
+
await new Promise<void>((resolve) => {
|
|
9968
|
+
const timer = setTimeout(resolve, CLAIM_RESTART_REDIAL_STEP_MS);
|
|
9969
|
+
// Node-only: a stop mid-window must not keep the event loop alive for the step.
|
|
9970
|
+
(timer as { unref?: () => void }).unref?.();
|
|
9971
|
+
});
|
|
9972
|
+
}
|
|
9973
|
+
} finally {
|
|
9974
|
+
this.claimRedialPeers.delete(peerId);
|
|
9975
|
+
}
|
|
9976
|
+
}
|
|
9977
|
+
|
|
9978
|
+
/** One step of {@link redialClaimedNode}; false ends the loop. */
|
|
9979
|
+
private async redialClaimedNodeStep(peerId: string): Promise<boolean> {
|
|
9980
|
+
const controlNode = this.controlNode;
|
|
9981
|
+
const target = this.bootstrapPeerStore?.all().get(peerId);
|
|
9982
|
+
if (!this._running || !controlNode || !target || this.identifiedControlPeers.has(peerId)) {
|
|
9983
|
+
return false;
|
|
9984
|
+
}
|
|
9985
|
+
if (controlNode.getConnections(peerIdFromString(peerId)).length === 0) {
|
|
9986
|
+
await this.dialBootstrapPeer(peerId, target.addrs, 'claimNode(redial)');
|
|
9987
|
+
}
|
|
9988
|
+
return true;
|
|
9314
9989
|
}
|
|
9315
9990
|
|
|
9316
9991
|
/**
|
|
@@ -9481,11 +10156,13 @@ export class CadreNode implements SAppIdLookup {
|
|
|
9481
10156
|
* Steps: refuse an invitation for another party; pin its `ownerKeys` into the node-local
|
|
9482
10157
|
* anchor (`'invite'` provenance — the pin sticks even if the redemption then fails, as it
|
|
9483
10158
|
* does for a seed invite, so a later attempt or seed from the same owner is anchored);
|
|
9484
|
-
* sign the request
|
|
9485
|
-
* acceptance verify the reply against the pinned
|
|
9486
|
-
* store, retain the answering member as a
|
|
9487
|
-
*
|
|
9488
|
-
* the connection it already holds, the same unified behaviour as after a seed.
|
|
10159
|
+
* sign the request, which carries this node's own signed address record; dial the listed
|
|
10160
|
+
* members in order (`redeemAtMembers`); on acceptance verify the reply against the pinned
|
|
10161
|
+
* keys, merge its dial hints into the peer store, retain the answering member as a
|
|
10162
|
+
* cold-start dial target and start a cohort reconcile pass. This node then syncs the control
|
|
10163
|
+
* database over the connection it already holds, the same unified behaviour as after a seed.
|
|
10164
|
+
* The member writes the record on this node's `CadrePeer` row when its signature verifies, so
|
|
10165
|
+
* the row resolves to this node's addresses from its admission.
|
|
9489
10166
|
*
|
|
9490
10167
|
* A device that is already a member (a restart with the invitation still configured) is
|
|
9491
10168
|
* answered as accepted by the member's idempotent redemption, with no second usage row.
|
|
@@ -9529,12 +10206,20 @@ export class CadreNode implements SAppIdLookup {
|
|
|
9529
10206
|
}
|
|
9530
10207
|
await this.trustOwnerKeys(invitation.ownerKeys, 'invite');
|
|
9531
10208
|
const pinned = new Set(invitation.ownerKeys);
|
|
10209
|
+
const selfPeerId = controlNode.peerId.toString();
|
|
10210
|
+
// The addresses a publish would carry, sanitized before signing: the member stores the
|
|
10211
|
+
// signature only over the exact list it stores.
|
|
10212
|
+
const record = this.signSelfRecord(selfPeerId, signer, sanitizeAddrs(await this.collectSelfAddrs()), null);
|
|
9532
10213
|
const request = signRedeemRequest(
|
|
9533
10214
|
invitation,
|
|
9534
10215
|
{ peerKey: signer.publicKeyB64, peerPrivateKey: signer.privateKeyB64 },
|
|
9535
|
-
generateStampId(
|
|
9536
|
-
|
|
10216
|
+
generateStampId(selfPeerId),
|
|
10217
|
+
record
|
|
9537
10218
|
);
|
|
10219
|
+
// Before dialing, not on acceptance: the member's commit can reach this node before its
|
|
10220
|
+
// reply does, and a reconcile pass that reads the row first must find it current. A record
|
|
10221
|
+
// no member wrote matches no row, so a failed redemption leaves nothing to undo.
|
|
10222
|
+
this.lastSignedSelfRecord = record;
|
|
9538
10223
|
const { reply, memberPeerId, memberAddr } = await redeemInPasses(controlNode, {
|
|
9539
10224
|
invitation,
|
|
9540
10225
|
request,
|
|
@@ -9546,7 +10231,6 @@ export class CadreNode implements SAppIdLookup {
|
|
|
9546
10231
|
});
|
|
9547
10232
|
log('Cadre invitation %s redeemed at %s', invitation.invite.key, memberAddr);
|
|
9548
10233
|
|
|
9549
|
-
const selfPeerId = controlNode.peerId.toString();
|
|
9550
10234
|
await mergeSeedPeers(controlNode, reply.peers.filter((peer) => peer.peerId !== selfPeerId));
|
|
9551
10235
|
if (memberPeerId !== null) {
|
|
9552
10236
|
// Every address the bundle gave the answering member, so a restart has the same choices.
|
|
@@ -9555,9 +10239,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
9555
10239
|
.map((addr) => addr.toString());
|
|
9556
10240
|
this.retainDialTarget(memberPeerId, memberAddrs, 'redeemCadreInvitation');
|
|
9557
10241
|
}
|
|
9558
|
-
//
|
|
9559
|
-
//
|
|
9560
|
-
await this.refreshMembershipGate('invitation-redeemed');
|
|
10242
|
+
// No gate refresh here: the rows about to replicate in are judged against the pins above
|
|
10243
|
+
// when they land, by the gate watch's refresh and the head refresh of this pass.
|
|
9561
10244
|
void this.reconcileControlCohort().catch((error) =>
|
|
9562
10245
|
log('redeemCadreInvitation: the post-redemption reconcile pass failed: %o', error));
|
|
9563
10246
|
return { peerId: memberPeerId, redeemedAt: new Date().toISOString() };
|
|
@@ -9737,25 +10420,31 @@ export class CadreNode implements SAppIdLookup {
|
|
|
9737
10420
|
}
|
|
9738
10421
|
|
|
9739
10422
|
/**
|
|
9740
|
-
* {@link redeemOwnerGrant}'s local path: the alone refusal, the seat
|
|
9741
|
-
* then one approval around the consent signature and the
|
|
9742
|
-
* the approval, so a prompting signer never asks for a
|
|
10423
|
+
* {@link redeemOwnerGrant}'s local path: the alone refusal, the seat (only when this node holds
|
|
10424
|
+
* no grant row), the liveness refusal, then one approval around the consent signature and the
|
|
10425
|
+
* write. Every refusal comes before the approval, so a prompting signer never asks for a
|
|
10426
|
+
* redemption this node would refuse. The held row is read once and carried into the liveness
|
|
10427
|
+
* check and the write, as the member handler carries it (`MemberRedemptionHandler`).
|
|
9743
10428
|
*/
|
|
9744
10429
|
private async redeemOwnerGrantLocally(controlDatabase: ControlDatabase, bundle: OwnerGrantBundle, signer: OwnerSigner): Promise<RedeemOwnerGrantResult> {
|
|
9745
10430
|
if (await this.ownerKeyWriteAlone()) {
|
|
9746
10431
|
throw new OwnerKeyAddRefusedError(signer.publicKey, 'alone');
|
|
9747
10432
|
}
|
|
9748
|
-
|
|
10433
|
+
const held = await controlDatabase.queryOwnerGrant(bundle.grant.key);
|
|
10434
|
+
if (held === null) {
|
|
10435
|
+
await refusingUnsyncedGrant(() => controlDatabase.seatOwnerGrant(bundle.grant));
|
|
10436
|
+
}
|
|
9749
10437
|
// The write answers a key that is already an owner without reading the grant, so only a
|
|
9750
10438
|
// key that is not one needs the grant live (the same permissive pre-check the handler's
|
|
9751
10439
|
// admission uses; the write enforces every condition).
|
|
9752
|
-
if (!(await controlDatabase.getOwnerKeys()).has(signer.publicKey) && !await controlDatabase.isOwnerGrantLive(bundle.grant)) {
|
|
10440
|
+
if (!(await controlDatabase.getOwnerKeys()).has(signer.publicKey) && !await controlDatabase.isOwnerGrantLive(bundle.grant, held)) {
|
|
9753
10441
|
throw new OwnerGrantRejectedError('grant-spent', 'The grant is expired, withdrawn, exhausted, or its issuer is no longer an owner');
|
|
9754
10442
|
}
|
|
9755
10443
|
const { alreadyOwner } = await signer.approve(ownerGrantApproval(bundle), async (sign) => {
|
|
9756
10444
|
const request = await this.signOwnerGrantRequest(bundle, signer, sign);
|
|
9757
10445
|
return await refusingUnsyncedGrant(() => controlDatabase.redeemOwnerGrant({
|
|
9758
10446
|
grantKey: bundle.grant.key,
|
|
10447
|
+
...(held ? { held } : {}),
|
|
9759
10448
|
seatedKey: signer.publicKey,
|
|
9760
10449
|
usageStampId: request.usageStampId,
|
|
9761
10450
|
grantSig: request.grantSig,
|
|
@@ -9950,8 +10639,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
9950
10639
|
// Overridden on the same grounds: only this node holds the running strand
|
|
9951
10640
|
// instance's database and this party's `StrandPartyKey` identity, and a wrong
|
|
9952
10641
|
// issuer here would admit joiners under someone else's authority.
|
|
9953
|
-
issueMembershipInvite: (
|
|
9954
|
-
this.issueStrandMembershipInvite(strandId, signal)
|
|
10642
|
+
issueMembershipInvite: (host, signal) => this.issueStrandMembershipInvite(host, signal)
|
|
9955
10643
|
});
|
|
9956
10644
|
|
|
9957
10645
|
const previous = this.strandSolicitationService;
|
|
@@ -10259,10 +10947,17 @@ export class CadreNode implements SAppIdLookup {
|
|
|
10259
10947
|
selfId: this.controlNode!.peerId.toString(),
|
|
10260
10948
|
linkRoundTripMs: this.config.network?.linkRoundTripMs,
|
|
10261
10949
|
isOwner: () => this.isOwner(),
|
|
10262
|
-
// NOTE: one owner-key read and one pending-join read (three tables) per pass (30 s); for a
|
|
10263
|
-
// party that never asked for a join the second reads never-written blocks, which consults
|
|
10264
|
-
// the cohort. If it shows up in a device profile, poll more slowly while the tables read empty.
|
|
10265
10950
|
readRows: () => database().queryPendingJoins(),
|
|
10951
|
+
// What `isOwner` and `queryPendingJoins` read: the owner keys, the joins and their
|
|
10952
|
+
// outcomes, and the tombstones both filter by. A dismissal writes its tombstone with the
|
|
10953
|
+
// request's delete, but the two can replicate separately.
|
|
10954
|
+
watchChanges: (onChange, options) => database().watchTables([
|
|
10955
|
+
'CadreControl.JoinRequest',
|
|
10956
|
+
'CadreControl.JoinSuccess',
|
|
10957
|
+
'CadreControl.JoinFailure',
|
|
10958
|
+
'CadreControl.OwnerKey',
|
|
10959
|
+
'CadreControl.Revocation',
|
|
10960
|
+
], onChange, options),
|
|
10266
10961
|
readRow: (id) => database().queryPendingJoin(id),
|
|
10267
10962
|
attempt: (join) => this.formStrand(this.pendingJoinInvitation(join), parseStoredDisclosure(join)),
|
|
10268
10963
|
record: (expected, outcome) => ownerWrite(`Record the outcome of the pending join ${expected.Id}`,
|
|
@@ -10328,6 +11023,13 @@ export class CadreNode implements SAppIdLookup {
|
|
|
10328
11023
|
* token is spent, and a join no store names would vanish at the next restart.
|
|
10329
11024
|
*/
|
|
10330
11025
|
private async rememberFormedStrand(result: FormStrandResult): Promise<void> {
|
|
11026
|
+
// NOTE: accepted tradeoff — the party-wide `JoinedStrand` row is not written here, in
|
|
11027
|
+
// formStrand's own approval, but later by an owner machine's connected reconcile pass
|
|
11028
|
+
// (`JoinedStrandSession.syncWithParty`). That publish is gated on a control connection so a
|
|
11029
|
+
// machine cut off from its own party never commits it alone, and during formStrand the
|
|
11030
|
+
// count always includes the connection to the other party's responder, so the gate would
|
|
11031
|
+
// always pass here. Non-owner and signer-less machines need this local path regardless.
|
|
11032
|
+
// Revisit if the gate can tell a connection to this party's own machines from any other.
|
|
10331
11033
|
try {
|
|
10332
11034
|
await this.joinedStrands!.remember({
|
|
10333
11035
|
Id: result.strandId,
|
|
@@ -10371,7 +11073,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
10371
11073
|
*/
|
|
10372
11074
|
private async adoptFormationMembershipInvite(strandId: string, invite: StrandMembershipInvite): Promise<void> {
|
|
10373
11075
|
try {
|
|
10374
|
-
await this.ensureStrandPartyKey(strandId);
|
|
11076
|
+
this.formedPartyKeys.set(strandId, await this.ensureStrandPartyKey(strandId));
|
|
10375
11077
|
} catch (error) {
|
|
10376
11078
|
throw new FormationPostApprovalError(
|
|
10377
11079
|
strandId,
|
|
@@ -10422,6 +11124,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
10422
11124
|
* implements): mint a single-use `Strand.Invite` against the LIVE host strand so a
|
|
10423
11125
|
* validated joiner can seat its own `Strand.Member` row.
|
|
10424
11126
|
*
|
|
11127
|
+
* `host` is the strand as the formation session resolved it from its `Strand` row, read
|
|
11128
|
+
* once there; this method reads no `Strand` row of its own. A row removed in between is
|
|
11129
|
+
* refused when the consent row is written (the `StrandExists` CHECK).
|
|
11130
|
+
*
|
|
10425
11131
|
* - Open host strand → `null` (no members, nothing to invite into).
|
|
10426
11132
|
* - Closed host strand whose founder launch was refused as pre-split
|
|
10427
11133
|
* ({@link strandLaunchRefusals}) → rethrow the recorded `PreSplitStrandIdentityError`;
|
|
@@ -10470,19 +11176,14 @@ export class CadreNode implements SAppIdLookup {
|
|
|
10470
11176
|
* not a retry.
|
|
10471
11177
|
*/
|
|
10472
11178
|
private async issueStrandMembershipInvite(
|
|
10473
|
-
|
|
11179
|
+
host: BoundHostStrand,
|
|
10474
11180
|
signal?: AbortSignal
|
|
10475
11181
|
): Promise<StrandMembershipInvite | null> {
|
|
11182
|
+
const { strandId } = host;
|
|
10476
11183
|
if (!this.controlDatabase) {
|
|
10477
11184
|
throw new Error(`Cannot issue a membership invitation for strand ${strandId}: control database unavailable`);
|
|
10478
11185
|
}
|
|
10479
|
-
|
|
10480
|
-
if (!row) {
|
|
10481
|
-
// The bound resolution saw this row moments ago; a vanished row is a concurrent
|
|
10482
|
-
// unpublish — reject rather than invite into a strand this party just removed.
|
|
10483
|
-
throw new Error(`Cannot issue a membership invitation for strand ${strandId}: its Strand row is gone`);
|
|
10484
|
-
}
|
|
10485
|
-
if (row.Type !== 'c') {
|
|
11186
|
+
if (host.strandType !== 'c') {
|
|
10486
11187
|
return null;
|
|
10487
11188
|
}
|
|
10488
11189
|
const refusal = this.strandLaunchRefusals.get(strandId);
|
|
@@ -10504,8 +11205,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
10504
11205
|
'not live on this responder (not launched, or still starting)'
|
|
10505
11206
|
);
|
|
10506
11207
|
}
|
|
10507
|
-
if (
|
|
10508
|
-
await assertNotPreSplitStrand(db, strandId, strandMemberKeyPair(
|
|
11208
|
+
if (host.memberPrivateKey) {
|
|
11209
|
+
await assertNotPreSplitStrand(db, strandId, strandMemberKeyPair(host.memberPrivateKey).publicKeyB64);
|
|
10509
11210
|
}
|
|
10510
11211
|
if (signal?.aborted) {
|
|
10511
11212
|
throw new Error(
|