@serfab/cadre-core 1.7.0 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -6
- package/dist/cadre-node.d.ts +527 -206
- package/dist/cadre-node.js +1084 -528
- package/dist/cadre-node.js.map +1 -1
- package/dist/control-authorization.d.ts +2 -2
- package/dist/control-authorization.js +1 -0
- package/dist/control-authorization.js.map +1 -1
- package/dist/control-cohort.d.ts +6 -0
- package/dist/control-cohort.js +6 -0
- package/dist/control-cohort.js.map +1 -1
- package/dist/control-database.d.ts +46 -8
- package/dist/control-database.js +97 -18
- package/dist/control-database.js.map +1 -1
- package/dist/control-formation-recorder.d.ts +37 -14
- package/dist/control-formation-recorder.js +60 -10
- package/dist/control-formation-recorder.js.map +1 -1
- package/dist/control-read-retry.d.ts +13 -12
- package/dist/control-read-retry.js +14 -12
- package/dist/control-read-retry.js.map +1 -1
- package/dist/control-schema.d.ts +1 -1
- package/dist/control-schema.js +67 -5
- package/dist/control-schema.js.map +1 -1
- package/dist/control-stream.d.ts +9 -1
- package/dist/control-stream.js +24 -1
- package/dist/control-stream.js.map +1 -1
- package/dist/control-write-retry.d.ts +12 -2
- package/dist/control-write-retry.js +13 -2
- package/dist/control-write-retry.js.map +1 -1
- package/dist/delegate-admission.d.ts +11 -8
- package/dist/delegate-admission.js +19 -11
- package/dist/delegate-admission.js.map +1 -1
- package/dist/diagnostics/connection-path.d.ts +0 -7
- package/dist/diagnostics/connection-path.js +0 -7
- package/dist/diagnostics/connection-path.js.map +1 -1
- package/dist/formation-approval.d.ts +7 -0
- package/dist/formation-approval.js +9 -3
- package/dist/formation-approval.js.map +1 -1
- package/dist/fs-atomic.d.ts +11 -2
- package/dist/fs-atomic.js +13 -3
- package/dist/fs-atomic.js.map +1 -1
- package/dist/hibernation-manager.d.ts +111 -8
- package/dist/hibernation-manager.js +234 -49
- package/dist/hibernation-manager.js.map +1 -1
- package/dist/index.d.ts +10 -10
- package/dist/index.js +25 -20
- package/dist/index.js.map +1 -1
- package/dist/joined-strand-store.d.ts +90 -25
- package/dist/joined-strand-store.js +223 -55
- package/dist/joined-strand-store.js.map +1 -1
- package/dist/key-store-file.js +14 -5
- package/dist/key-store-file.js.map +1 -1
- package/dist/link-budget.d.ts +367 -69
- package/dist/link-budget.js +375 -72
- package/dist/link-budget.js.map +1 -1
- package/dist/membership-connection-gater.d.ts +30 -43
- package/dist/membership-connection-gater.js +31 -42
- package/dist/membership-connection-gater.js.map +1 -1
- package/dist/node-local-snapshot.d.ts +6 -4
- package/dist/node-local-snapshot.js +7 -3
- package/dist/node-local-snapshot.js.map +1 -1
- package/dist/peer-dial.d.ts +24 -4
- package/dist/peer-dial.js +74 -5
- package/dist/peer-dial.js.map +1 -1
- package/dist/peer-join-backfill.d.ts +2 -10
- package/dist/peer-join-backfill.js +8 -8
- package/dist/peer-join-backfill.js.map +1 -1
- package/dist/push-fanout.d.ts +6 -0
- package/dist/push-fanout.js +6 -0
- package/dist/push-fanout.js.map +1 -1
- package/dist/relay-reservation.d.ts +3 -1
- package/dist/relay-reservation.js +3 -1
- package/dist/relay-reservation.js.map +1 -1
- package/dist/relay-server.d.ts +79 -0
- package/dist/relay-server.js +74 -0
- package/dist/relay-server.js.map +1 -0
- package/dist/seed-bootstrap.d.ts +57 -4
- package/dist/seed-bootstrap.js +107 -83
- package/dist/seed-bootstrap.js.map +1 -1
- package/dist/storage-scope.d.ts +33 -17
- package/dist/storage-scope.js +44 -23
- package/dist/storage-scope.js.map +1 -1
- package/dist/strand-addr-protocol.d.ts +58 -21
- package/dist/strand-addr-protocol.js +133 -85
- package/dist/strand-addr-protocol.js.map +1 -1
- package/dist/strand-database.d.ts +20 -2
- package/dist/strand-database.js +42 -5
- package/dist/strand-database.js.map +1 -1
- package/dist/strand-first-sync-gate.d.ts +59 -21
- package/dist/strand-first-sync-gate.js +58 -19
- package/dist/strand-first-sync-gate.js.map +1 -1
- package/dist/strand-formation-deadlines.d.ts +151 -0
- package/dist/strand-formation-deadlines.js +171 -0
- package/dist/strand-formation-deadlines.js.map +1 -0
- package/dist/strand-formation-manager.d.ts +63 -30
- package/dist/strand-formation-manager.js +67 -41
- package/dist/strand-formation-manager.js.map +1 -1
- package/dist/strand-formation-protocol.d.ts +45 -46
- package/dist/strand-formation-protocol.js +62 -126
- package/dist/strand-formation-protocol.js.map +1 -1
- package/dist/strand-fret-addrs.d.ts +54 -0
- package/dist/strand-fret-addrs.js +108 -0
- package/dist/strand-fret-addrs.js.map +1 -0
- package/dist/strand-instance-manager.d.ts +122 -44
- package/dist/strand-instance-manager.js +270 -114
- package/dist/strand-instance-manager.js.map +1 -1
- package/dist/strand-membership-writer.d.ts +6 -0
- package/dist/strand-membership-writer.js +11 -0
- package/dist/strand-membership-writer.js.map +1 -1
- package/dist/strand-network-state-file.d.ts +12 -0
- package/dist/strand-network-state-file.js +38 -0
- package/dist/strand-network-state-file.js.map +1 -0
- package/dist/strand-network-state.d.ts +111 -0
- package/dist/strand-network-state.js +191 -0
- package/dist/strand-network-state.js.map +1 -0
- package/dist/strand-revocation-enforcer.js +2 -1
- package/dist/strand-revocation-enforcer.js.map +1 -1
- package/dist/strand-solicitation.d.ts +36 -20
- package/dist/strand-solicitation.js +4 -4
- package/dist/strand-solicitation.js.map +1 -1
- package/dist/strand-wake-protocol.d.ts +80 -23
- package/dist/strand-wake-protocol.js +89 -64
- package/dist/strand-wake-protocol.js.map +1 -1
- package/dist/types.d.ts +148 -61
- package/dist/types.js +4 -3
- package/dist/types.js.map +1 -1
- package/package.json +11 -10
- package/src/cadre-node.ts +8400 -7776
- package/src/control-authorization.ts +2 -1
- package/src/control-cohort.ts +6 -0
- package/src/control-database.ts +122 -18
- package/src/control-formation-recorder.ts +354 -305
- package/src/control-read-retry.ts +14 -12
- package/src/control-schema.ts +67 -5
- package/src/control-stream.ts +25 -1
- package/src/control-write-retry.ts +13 -2
- package/src/delegate-admission.ts +23 -13
- package/src/diagnostics/connection-path.ts +0 -7
- package/src/formation-approval.ts +610 -604
- package/src/fs-atomic.ts +14 -4
- package/src/hibernation-manager.ts +638 -437
- package/src/index.ts +43 -65
- package/src/joined-strand-store.ts +247 -57
- package/src/key-store-file.ts +14 -5
- package/src/link-budget.ts +393 -74
- package/src/membership-connection-gater.ts +31 -45
- package/src/node-local-snapshot.ts +12 -7
- package/src/peer-dial.ts +81 -7
- package/src/peer-join-backfill.ts +10 -9
- package/src/push-fanout.ts +6 -0
- package/src/relay-reservation.ts +3 -1
- package/src/relay-server.ts +98 -0
- package/src/seed-bootstrap.ts +133 -91
- package/src/storage-scope.ts +44 -23
- package/src/strand-addr-protocol.ts +547 -458
- package/src/strand-database.ts +287 -242
- package/src/strand-first-sync-gate.ts +58 -19
- package/src/strand-formation-deadlines.ts +227 -0
- package/src/strand-formation-manager.ts +700 -647
- package/src/strand-formation-protocol.ts +106 -154
- package/src/strand-fret-addrs.ts +145 -0
- package/src/strand-instance-manager.ts +307 -140
- package/src/strand-membership-writer.ts +12 -0
- package/src/strand-network-state-file.ts +40 -0
- package/src/strand-network-state.ts +243 -0
- package/src/strand-revocation-enforcer.ts +2 -1
- package/src/strand-solicitation.ts +519 -500
- package/src/strand-wake-protocol.ts +401 -355
- package/src/types.ts +150 -61
- package/dist/strand-peer-book-file.d.ts +0 -12
- package/dist/strand-peer-book-file.js +0 -37
- package/dist/strand-peer-book-file.js.map +0 -1
- package/dist/strand-peer-book-protocol.d.ts +0 -159
- package/dist/strand-peer-book-protocol.js +0 -360
- package/dist/strand-peer-book-protocol.js.map +0 -1
- package/dist/strand-peer-book-swap.d.ts +0 -104
- package/dist/strand-peer-book-swap.js +0 -284
- package/dist/strand-peer-book-swap.js.map +0 -1
- package/dist/strand-peer-book.d.ts +0 -159
- package/dist/strand-peer-book.js +0 -354
- package/dist/strand-peer-book.js.map +0 -1
- package/dist/strand-peer-observer.d.ts +0 -88
- package/dist/strand-peer-observer.js +0 -180
- package/dist/strand-peer-observer.js.map +0 -1
- package/src/strand-peer-book-file.ts +0 -39
- package/src/strand-peer-book-protocol.ts +0 -463
- package/src/strand-peer-book-swap.ts +0 -338
- package/src/strand-peer-book.ts +0 -468
- package/src/strand-peer-observer.ts +0 -235
package/dist/cadre-node.js
CHANGED
|
@@ -11,16 +11,17 @@ import { ed25519KeyPairFromLibp2p, ed25519PublicKeyFromPrivate, requireEd25519Pu
|
|
|
11
11
|
import { strandTransportKey } from './strand-transport-key.js';
|
|
12
12
|
import { assertScopeKeyCharset, assertStrandScopeKey, controlStorageScope, InvalidStrandIdError, isValidStrandScopeKey } from './storage-scope.js';
|
|
13
13
|
import { generateStrandMemberKey, strandMemberKeyPair } from './strand-member-key.js';
|
|
14
|
-
import { assertNotPreSplitStrand, issueInvite, PreSplitStrandIdentityError } from './strand-membership-writer.js';
|
|
14
|
+
import { assertNotPreSplitStrand, issueInvite, PreSplitStrandIdentityError, readStrandHeaderSAppId } from './strand-membership-writer.js';
|
|
15
15
|
import { MEMBERSHIP_INVITE_TTL_MS } from './strand-formation-manager.js';
|
|
16
16
|
import { DEFAULT_IDENTITY_KEY_ID } from './key-store.js';
|
|
17
17
|
import { loadOrCreateIdentityKey } from './identity-key.js';
|
|
18
18
|
import { MemoryTrustedOwnerStore } from './trusted-owner-store.js';
|
|
19
19
|
import { MemoryBootstrapPeerStore } from './bootstrap-peer-store.js';
|
|
20
|
-
import {
|
|
20
|
+
import { MemoryStrandNetworkStateStore } from './strand-network-state.js';
|
|
21
21
|
import { MemoryEnrolledMachineStore } from './enrolled-machine-store.js';
|
|
22
|
-
import {
|
|
22
|
+
import { JoinedStrandSession, KeyStoreJoinedStrandStore, MemoryJoinedStrandStore } from './joined-strand-store.js';
|
|
23
23
|
import { mergePeerAddrs, groupAddrsByPeerId } from './peer-addr-book.js';
|
|
24
|
+
import { strandFretPeerAddrs } from './strand-fret-addrs.js';
|
|
24
25
|
import { verifyCadrePeerVoucher } from './peer-authorization.js';
|
|
25
26
|
import { ed25519PublicKeyB64FromPeerId } from './seed-bootstrap.js';
|
|
26
27
|
import { signPeerRecord, verifyPeerRecordSignature, isPeerRecordFresh, orderSignalingFirst, isSignalingAddr, trailingPeerId, withTrailingPeerId, currentMemberTrustPolicy, DEFAULT_PEER_RECORD_MAX_AGE_MS, DEFAULT_PEER_RECORD_HEARTBEAT_MS } from './peer-record.js';
|
|
@@ -30,8 +31,8 @@ import { StrandInstanceManager, liveStrandStatus } from './strand-instance-manag
|
|
|
30
31
|
import { PeerJoinBackfill } from './peer-join-backfill.js';
|
|
31
32
|
import { deriveCohortMembers } from './strand-cohort.js';
|
|
32
33
|
import { selectControlCohortDials, DEFAULT_CONTROL_COHORT_RECONCILE_MS, DEFAULT_CONTROL_COHORT_TARGET_DEGREE } from './control-cohort.js';
|
|
33
|
-
import { dialPeerAddrs, CONTROL_COHORT_DIAL_ADDRESS_ATTEMPTS } from './peer-dial.js';
|
|
34
|
-
import {
|
|
34
|
+
import { dialPeerAddrs, SelfRelayOnlyError, CONTROL_COHORT_DIAL_ADDRESS_ATTEMPTS } from './peer-dial.js';
|
|
35
|
+
import { ADMISSION_DECISION_TIMEOUT_MS, declaredCohortReadDeadlineMs, peerJoinPushBudget, relayAdmissionReserveDeadlineMs, relayReservationBudgetMs, relayedDialBudgetMs, resolveLinkRoundTripMs } from './link-budget.js';
|
|
35
36
|
import { EnrollmentService } from './enrollment.js';
|
|
36
37
|
import { HibernationManager } from './hibernation-manager.js';
|
|
37
38
|
import { ControlDatabase, isStrandIdConflict } from './control-database.js';
|
|
@@ -40,8 +41,9 @@ import { StrandSolicitationService } from './strand-solicitation.js';
|
|
|
40
41
|
import { createMembershipConnectionGater, DEFAULT_ENROLLMENT_WINDOW_MS, UnauthorizedReservationBudget } from './membership-connection-gater.js';
|
|
41
42
|
import { StrandWakeService, dialWake } from './strand-wake-protocol.js';
|
|
42
43
|
import { StrandAddrService, collectStrandAddrs } from './strand-addr-protocol.js';
|
|
43
|
-
import { DelegateAdmissionStore, extractCircuitRelayTargets, dueRelayAnnounces,
|
|
44
|
+
import { DelegateAdmissionStore, extractCircuitRelayTargets, dueRelayAnnounces, prunePeerStrandKeys, peerStrandKey } from './delegate-admission.js';
|
|
44
45
|
import { relayCircuitAddrs, resolveListenAddrs, resolveTransportOptions, RelayReservationFailedError } from './relay-addrs.js';
|
|
46
|
+
import { resolveRelayServer } from './relay-server.js';
|
|
45
47
|
import { replacesAdvertisedAddrs, resolveAnnounceAddrs } from './announce-addrs.js';
|
|
46
48
|
import { superviseRelayReservation, resolveRelayReservationState } from './relay-reservation.js';
|
|
47
49
|
import { PushFanoutService } from './push-fanout.js';
|
|
@@ -57,10 +59,10 @@ const timing = debug('sereus:cadre:timing');
|
|
|
57
59
|
*/
|
|
58
60
|
const TURN_CONSUME_WINDOW_MS = 1000;
|
|
59
61
|
/**
|
|
60
|
-
* How often a running strand re-
|
|
61
|
-
* and re-merges them into
|
|
62
|
-
* ({@link CadreNode.refreshStrandPeerAddrs}).
|
|
63
|
-
* `network.controlCohort.strandAddrRefreshMs`.
|
|
62
|
+
* How often a running strand re-asks each connected sibling that last ANSWERED
|
|
63
|
+
* (even with nothing) for its strand-network addresses, and re-merges them into
|
|
64
|
+
* its own libp2p address book ({@link CadreNode.refreshStrandPeerAddrs}).
|
|
65
|
+
* Overridable per node via `network.controlCohort.strandAddrRefreshMs`.
|
|
64
66
|
*
|
|
65
67
|
* Ten minutes sits comfortably inside the peerStore's one-hour address expiry
|
|
66
68
|
* (`MAX_ADDRESS_AGE`, see `peer-addr-book.ts`) with headroom for several missed
|
|
@@ -68,6 +70,16 @@ const TURN_CONSUME_WINDOW_MS = 1000;
|
|
|
68
70
|
* strand-addr RPC fan-out stays cheap.
|
|
69
71
|
*/
|
|
70
72
|
export const STRAND_PEER_ADDR_REFRESH_MS = 10 * 60 * 1000;
|
|
73
|
+
/**
|
|
74
|
+
* How soon {@link CadreNode.refreshStrandPeerAddrs} re-asks a sibling that did NOT
|
|
75
|
+
* answer — unreachable, `unavailable`, or `refused`. A refusal matters most: a
|
|
76
|
+
* phone asking before its `CadrePeer` row has replicated to the sibling is refused
|
|
77
|
+
* and its delegate grant goes unrecorded, and waiting the full
|
|
78
|
+
* {@link STRAND_PEER_ADDR_REFRESH_MS} for the next try is the delay this bounds.
|
|
79
|
+
* Four reconcile ticks, so a sibling that keeps failing costs one timed-out RPC a
|
|
80
|
+
* minute rather than one a tick.
|
|
81
|
+
*/
|
|
82
|
+
export const STRAND_PEER_ADDR_RETRY_MS = 60 * 1000;
|
|
71
83
|
/**
|
|
72
84
|
* A circuit relay as a strand-addr RPC target: dialed by peerId, with its
|
|
73
85
|
* direct addr riding along as the fallback for a relay we hold no connection to.
|
|
@@ -75,12 +87,20 @@ export const STRAND_PEER_ADDR_REFRESH_MS = 10 * 60 * 1000;
|
|
|
75
87
|
function relayStrandAddrPeer(relay) {
|
|
76
88
|
return { peerId: relay.relayPeerId, addrs: [multiaddr(relay.relayAddr)] };
|
|
77
89
|
}
|
|
90
|
+
/**
|
|
91
|
+
* Whether a sibling actually answered a strand-addr RPC — with addresses or
|
|
92
|
+
* without — as opposed to being unreachable, unavailable, or refusing us. Decides
|
|
93
|
+
* which of the refresh and retry intervals it waits before the next ask.
|
|
94
|
+
*/
|
|
95
|
+
function siblingAnswered(outcome) {
|
|
96
|
+
return outcome === 'answered' || outcome === 'empty';
|
|
97
|
+
}
|
|
78
98
|
/**
|
|
79
99
|
* `primary` followed by the entries of `extra` it does not already contain, de-duplicated
|
|
80
100
|
* and order-preserving.
|
|
81
101
|
*
|
|
82
|
-
* Used
|
|
83
|
-
* freshly-RPC'd sibling answers and the
|
|
102
|
+
* Used where a strand's discovery seed unions two sources of address strings — the
|
|
103
|
+
* freshly-RPC'd sibling answers and the addresses the strand's formation carried. The
|
|
84
104
|
* caller passes the FRESHER source as `primary`, so a stale entry can only ever be
|
|
85
105
|
* appended, never promoted ahead of a current one, and each source keeps whatever
|
|
86
106
|
* signaling-first ordering it arrived with.
|
|
@@ -169,6 +189,34 @@ function requireMatchingStrandRow(live, desired, situation) {
|
|
|
169
189
|
}
|
|
170
190
|
return live;
|
|
171
191
|
}
|
|
192
|
+
/**
|
|
193
|
+
* Await `work` unless `signal` aborts first: `true` once `work` resolves, `false` when the
|
|
194
|
+
* signal aborted before it did (already at the call, or during the wait). A rejection of
|
|
195
|
+
* `work` that lands before the abort propagates. `work` is NOT cancelled — a caller that
|
|
196
|
+
* gets `false` owns its late settlement. The abort listener is detached on both endings, so
|
|
197
|
+
* a long-lived signal does not collect listeners.
|
|
198
|
+
*/
|
|
199
|
+
async function resolvesBeforeAbort(work, signal) {
|
|
200
|
+
if (!signal) {
|
|
201
|
+
await work;
|
|
202
|
+
return true;
|
|
203
|
+
}
|
|
204
|
+
if (signal.aborted) {
|
|
205
|
+
return false;
|
|
206
|
+
}
|
|
207
|
+
let detach = () => { };
|
|
208
|
+
const abortedFirst = new Promise((resolve) => {
|
|
209
|
+
const onAbort = () => resolve(false);
|
|
210
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
211
|
+
detach = () => signal.removeEventListener('abort', onAbort);
|
|
212
|
+
});
|
|
213
|
+
try {
|
|
214
|
+
return await Promise.race([work.then(() => true), abortedFirst]);
|
|
215
|
+
}
|
|
216
|
+
finally {
|
|
217
|
+
detach();
|
|
218
|
+
}
|
|
219
|
+
}
|
|
172
220
|
/**
|
|
173
221
|
* CadreNode is the main entry point for a cadre member.
|
|
174
222
|
* It manages:
|
|
@@ -289,6 +337,19 @@ export class CadreNode {
|
|
|
289
337
|
* {@link getDiscoveredStrands}.
|
|
290
338
|
*/
|
|
291
339
|
this.discoveredStrands = new Map();
|
|
340
|
+
/**
|
|
341
|
+
* The sApp id each storage replica read from its own `Strand.Header` — {@link getSAppId}'s
|
|
342
|
+
* answer for a strand no local config claims, so an `sAppId` strand filter can reject a
|
|
343
|
+
* replica of some other app instead of admitting it provisionally forever.
|
|
344
|
+
*
|
|
345
|
+
* Cleared by {@link cleanup} only, deliberately NOT by {@link detachStrand}: the watcher's
|
|
346
|
+
* filter rejection stops a replica through that very method, and forgetting the id there
|
|
347
|
+
* would have the next poll find it unknown again, re-admit the strand provisionally and
|
|
348
|
+
* relaunch the replica — once every other poll, forever. A strand id's sApp never changes,
|
|
349
|
+
* and the map is bounded by the strands this party has published, like
|
|
350
|
+
* {@link discoveredStrands}.
|
|
351
|
+
*/
|
|
352
|
+
this.replicaSAppIds = new Map();
|
|
292
353
|
/**
|
|
293
354
|
* Most-recently pushed invite addresses (see {@link setInviteAddresses}).
|
|
294
355
|
* When non-null these take priority over `libp2pNode.getMultiaddrs()` when
|
|
@@ -365,46 +426,67 @@ export class CadreNode {
|
|
|
365
426
|
*/
|
|
366
427
|
this.delegateAnnounceAt = new Map();
|
|
367
428
|
/**
|
|
368
|
-
* When each
|
|
369
|
-
*
|
|
370
|
-
*
|
|
371
|
-
*
|
|
372
|
-
*
|
|
373
|
-
*
|
|
374
|
-
*
|
|
429
|
+
* When each connected sibling is next due a strand-addr RPC for each running
|
|
430
|
+
* strand, keyed `peerStrandKey(siblingControlPeerId, strandId)` → epoch ms; a
|
|
431
|
+
* missing key is due. Bounds {@link refreshStrandPeerAddrs}'s fan-out per
|
|
432
|
+
* (sibling, strand) rather than per strand, so a sibling that connects after a
|
|
433
|
+
* pass is asked on the next 15 s tick instead of waiting out a stamp another
|
|
434
|
+
* sibling's answer set. An answer (even an empty one) makes the sibling due again
|
|
435
|
+
* in {@link STRAND_PEER_ADDR_REFRESH_MS}; no answer, in
|
|
436
|
+
* {@link STRAND_PEER_ADDR_RETRY_MS}. Keys whose strand stopped running or whose
|
|
437
|
+
* sibling is no longer connected are pruned every pass, so a resumed strand and a
|
|
438
|
+
* reconnected sibling are both asked at once.
|
|
375
439
|
*/
|
|
376
|
-
this.
|
|
440
|
+
this.strandAddrAskDueAt = new Map();
|
|
377
441
|
/**
|
|
378
|
-
*
|
|
379
|
-
*
|
|
380
|
-
*
|
|
381
|
-
*
|
|
382
|
-
*
|
|
383
|
-
*
|
|
384
|
-
* store as `StartStrandConfig.strandPeerBook`) files the node's own signed entry
|
|
385
|
-
* plus every signed entry a strand peer sends it — the only writer whose entries
|
|
386
|
-
* carry a signature, and the only one that can refresh a peer's addresses without
|
|
387
|
-
* meeting that peer. Read by {@link resolveCohortSeed} (launch + hibernation resume) and
|
|
388
|
-
* re-merged by every {@link refreshStrandPeerAddrs} pass, which is what keeps the
|
|
389
|
-
* entries alive past the peerStore's one-hour address expiry.
|
|
442
|
+
* The strand-network addresses a formation carried back
|
|
443
|
+
* (`FormationResultMessage.strandAddrs`), by strand id: the responder's live strand
|
|
444
|
+
* node addresses, kept so the strand's first attach has something to dial. Written by
|
|
445
|
+
* {@link recordFormationStrandAddrs}, read by {@link resolveCohortSeed} (launch and
|
|
446
|
+
* hibernation resume) and by every refresh pass for the peers the strand node's FRET
|
|
447
|
+
* table holds no record for yet ({@link remergeUnrecordedFormationAddrs}).
|
|
390
448
|
*
|
|
391
|
-
* This is the
|
|
449
|
+
* This is the only cross-party input a FIRST attach has. The strand-addr RPC that
|
|
392
450
|
* resolves a strand's addresses is membership-gated and answers own-party siblings
|
|
393
|
-
* only, so without
|
|
394
|
-
*
|
|
395
|
-
*
|
|
396
|
-
*
|
|
397
|
-
*
|
|
451
|
+
* only, so without it a joiner's seed for a two-party strand is empty and the mesh
|
|
452
|
+
* never forms. Every later launch has the strand node's saved network state as well
|
|
453
|
+
* ({@link strandNetworkStateStore}), which is what a restart re-finds the other party
|
|
454
|
+
* from (gotchoices/sereus#18).
|
|
455
|
+
*
|
|
456
|
+
* A re-formation replaces the strand's list. An entry survives a {@link stopStrand}
|
|
457
|
+
* and a stop()→start() of this node instance ({@link cleanup} leaves the map alone),
|
|
458
|
+
* and is dropped where the saved network state is: {@link unpublishStrand},
|
|
459
|
+
* {@link forgetJoinedStrand} and self-revocation.
|
|
460
|
+
*
|
|
461
|
+
* NOTE: accepted tradeoff — in memory only, so a process restart between `formStrand`
|
|
462
|
+
* and the strand's first launch loses the carried addresses, and the strand then
|
|
463
|
+
* launches with no cross-party seed until it is re-formed. Persisting the list on the
|
|
464
|
+
* joined-strand record was weighed and declined: it mixes address state into the join
|
|
465
|
+
* record, and needs an "only until the first run" rule to stop dead addresses being
|
|
466
|
+
* dialled on every launch. Every embedder launches straight after forming, and once
|
|
467
|
+
* the strand has run the saved network state covers every later restart. Revisit if an
|
|
468
|
+
* embedder forms and launches in separate sessions.
|
|
469
|
+
*/
|
|
470
|
+
this.formationStrandAddrs = new Map();
|
|
471
|
+
/**
|
|
472
|
+
* Node-local strand network state (see `strand-network-state.ts`): per strand, the
|
|
473
|
+
* state db-p2p saves for the strand's libp2p node — its FRET routing table with each
|
|
474
|
+
* peer's signed address record. Handed to every strand launch as
|
|
475
|
+
* `StartStrandConfig.networkState`; the strand node saves into it on every connection
|
|
476
|
+
* and re-imports it when it is next built, which is what gives a restarted strand
|
|
477
|
+
* node addresses for the peers it was talking to.
|
|
478
|
+
*
|
|
479
|
+
* Constructed (or adopted from `config.strandNetworkState.store`) by
|
|
480
|
+
* {@link initializeStrandNetworkStateStore} during {@link start}, deliberately NOT
|
|
398
481
|
* cleared by {@link cleanup}, so it survives a stop()→start() cycle of the same node
|
|
399
482
|
* instance — same lifecycle as {@link bootstrapPeerStore}.
|
|
400
483
|
*
|
|
401
|
-
*
|
|
402
|
-
* {@link
|
|
403
|
-
*
|
|
404
|
-
*
|
|
405
|
-
* bounded by time and by the strands this node runs rather than by its history.
|
|
484
|
+
* A strand's state survives a {@link stopStrand} and a hibernation quiesce, and is
|
|
485
|
+
* forgotten by {@link unpublishStrand}, {@link forgetJoinedStrand} and
|
|
486
|
+
* self-revocation. A strand detached because the watcher saw its row gone keeps its
|
|
487
|
+
* state (`NOTE:` on `PersistentStrandNetworkStateStore`).
|
|
406
488
|
*/
|
|
407
|
-
this.
|
|
489
|
+
this.strandNetworkStateStore = null;
|
|
408
490
|
/**
|
|
409
491
|
* PENDING strand membership invitations learned at formation, keyed by strandId — the
|
|
410
492
|
* single-use `Strand.Invite` credential a closed-strand formation result carried back
|
|
@@ -432,7 +514,7 @@ export class CadreNode {
|
|
|
432
514
|
* NOTE: entries live for the node's lifetime (one small pair per formed closed
|
|
433
515
|
* strand); the number of keys is bounded only by the strands this node has ever
|
|
434
516
|
* formed, not by time — if a node ever forms strands at scale, evict on
|
|
435
|
-
* `unpublishStrand` / `forgetJoinedStrand` as
|
|
517
|
+
* `unpublishStrand` / `forgetJoinedStrand` as {@link formationStrandAddrs} is.
|
|
436
518
|
*/
|
|
437
519
|
this.pendingMembershipInvites = new Map();
|
|
438
520
|
/**
|
|
@@ -504,22 +586,22 @@ export class CadreNode {
|
|
|
504
586
|
*/
|
|
505
587
|
this.enrolledMachineStore = null;
|
|
506
588
|
/**
|
|
507
|
-
* Node-local record of the strands this node joined from ANOTHER party
|
|
508
|
-
* `joined-strand-store.ts`)
|
|
509
|
-
*
|
|
510
|
-
*
|
|
511
|
-
* and on self-revocation. Adopted from `config.joinedStrands.store`, else built over
|
|
589
|
+
* Node-local record of the strands this node joined from ANOTHER party and has not yet
|
|
590
|
+
* published party-wide (see `joined-strand-store.ts`). Written by {@link formStrand}
|
|
591
|
+
* and by an {@link addStrand} of a foreign row; drained once the party-wide
|
|
592
|
+
* `JoinedStrand` row is visible. Adopted from `config.joinedStrands.store`, else built over
|
|
512
593
|
* `config.keyStore`, else in-memory, by {@link initializeJoinedStrandStore}. Like
|
|
513
594
|
* {@link bootstrapPeerStore} it is deliberately NOT cleared by {@link cleanup}.
|
|
514
595
|
*/
|
|
515
596
|
this.joinedStrandStore = null;
|
|
516
597
|
/**
|
|
517
|
-
*
|
|
518
|
-
*
|
|
598
|
+
* This session's joined strands: {@link joinedStrandStore} and the party-wide
|
|
599
|
+
* `JoinedStrand` table, unioned with the control rows each watcher poll, published and
|
|
600
|
+
* removed by the reconcile pass. Per SESSION, unlike the store — rebuilt at every
|
|
519
601
|
* {@link start}, which is what ends a revoked join's "keep offering until this session
|
|
520
|
-
* ends" (see `
|
|
602
|
+
* ends" (see `JoinedStrandSession.forgetAfterThisSession`).
|
|
521
603
|
*/
|
|
522
|
-
this.
|
|
604
|
+
this.joinedStrands = null;
|
|
523
605
|
/** Initial self-registration timer (see {@link scheduleSelfRegistration}). */
|
|
524
606
|
this.selfRegistrationTimer = null;
|
|
525
607
|
/** TTL heartbeat that re-publishes the self record before it goes stale. */
|
|
@@ -644,24 +726,28 @@ export class CadreNode {
|
|
|
644
726
|
/** `connection:close` listener clearing a peer's TURN flag (teardown in {@link stopRecordRefresh}). */
|
|
645
727
|
this.turnConnectionCloseHandler = null;
|
|
646
728
|
this.config = config;
|
|
729
|
+
this.hostUnclaimedStrands = config.hostUnclaimedStrands ?? config.profile === 'storage';
|
|
647
730
|
this.strandManager = new StrandInstanceManager();
|
|
648
731
|
this.enrollmentService = new EnrollmentService();
|
|
649
|
-
this.
|
|
732
|
+
this.relayServer = resolveRelayServer(config.network, config.profile);
|
|
733
|
+
this.unauthorizedRelayReservations = new UnauthorizedReservationBudget(config.network?.unauthorizedRelayReservationCap, this.relayServer.init.reservations.reservationTtl);
|
|
650
734
|
// Create hibernation manager with callbacks
|
|
651
735
|
const hibernationCallbacks = {
|
|
652
736
|
onIdle: async (strandId) => this.handleStrandIdle(strandId),
|
|
653
737
|
onHibernate: async (strandId) => this.handleStrandHibernate(strandId),
|
|
654
738
|
onWake: async (strandId) => this.handleStrandWake(strandId),
|
|
655
|
-
onCheckIn: async (strandId) => this.handleStrandCheckIn(strandId)
|
|
739
|
+
onCheckIn: async (strandId) => this.handleStrandCheckIn(strandId),
|
|
740
|
+
isQuiescing: (strandId) => this.strandManager.isQuiescing(strandId)
|
|
656
741
|
};
|
|
657
742
|
this.hibernationManager = new HibernationManager(config.hibernation ?? { enabled: false }, hibernationCallbacks);
|
|
658
743
|
log('CadreNode created for party: %s', config.controlNetwork.partyId);
|
|
659
744
|
}
|
|
660
745
|
/**
|
|
661
|
-
* SAppIdLookup implementation - get sAppId for a strand
|
|
746
|
+
* SAppIdLookup implementation - get sAppId for a strand: the claiming config's, else
|
|
747
|
+
* the one a storage replica read from the strand's own `Strand.Header`.
|
|
662
748
|
*/
|
|
663
749
|
getSAppId(strandId) {
|
|
664
|
-
return this.sAppConfigs.get(strandId)?.id;
|
|
750
|
+
return this.sAppConfigs.get(strandId)?.id ?? this.replicaSAppIds.get(strandId);
|
|
665
751
|
}
|
|
666
752
|
/**
|
|
667
753
|
* Get the peer ID of this node (available after start)
|
|
@@ -777,6 +863,7 @@ export class CadreNode {
|
|
|
777
863
|
return;
|
|
778
864
|
}
|
|
779
865
|
log('Starting CadreNode for party: %s', this.config.controlNetwork.partyId);
|
|
866
|
+
this.logRelayServerSettings();
|
|
780
867
|
try {
|
|
781
868
|
const tTotal = performance.now();
|
|
782
869
|
// Config pre-flight, inside the guarded region: resolving the listen addrs it reads
|
|
@@ -810,8 +897,9 @@ export class CadreNode {
|
|
|
810
897
|
// beside the control rows. Before network bring-up, so a mis-scoped injected store
|
|
811
898
|
// fails closed like the two above.
|
|
812
899
|
this.initializeJoinedStrandStore();
|
|
813
|
-
//
|
|
814
|
-
|
|
900
|
+
// Each strand node's saved network state, which those joins are dialed from after
|
|
901
|
+
// a restart. Same placement, same reason.
|
|
902
|
+
this.initializeStrandNetworkStateStore();
|
|
815
903
|
// Read the party's enrolled-machine count out of its node-local record and
|
|
816
904
|
// capture it for buildControlNodeOptions below. This MUST precede
|
|
817
905
|
// createControlNode: Optimystic freezes the cluster policy when the node is
|
|
@@ -907,7 +995,7 @@ export class CadreNode {
|
|
|
907
995
|
getStrand: (strandId) => this.strandManager.getInstance(strandId),
|
|
908
996
|
wake: (strandId) => this.wakeStrand(strandId),
|
|
909
997
|
});
|
|
910
|
-
this.strandWakeService.initialize(this.controlNode);
|
|
998
|
+
await this.strandWakeService.initialize(this.controlNode);
|
|
911
999
|
// Register the control-network strand-address responder: a same-cadre peer
|
|
912
1000
|
// resolving a strand's bootstrap seed asks us for our live strand-network
|
|
913
1001
|
// multiaddrs (its CadrePeer row only knows our *control* address). Gated on
|
|
@@ -921,7 +1009,7 @@ export class CadreNode {
|
|
|
921
1009
|
getStrandMultiaddrs: (strandId) => this.getStrandMultiaddrs(strandId),
|
|
922
1010
|
onDelegateAnnounce: (announcer, strandId, delegate) => this.grantDelegateAdmission(announcer, strandId, delegate),
|
|
923
1011
|
});
|
|
924
|
-
this.strandAddrService.initialize(this.controlNode);
|
|
1012
|
+
await this.strandAddrService.initialize(this.controlNode);
|
|
925
1013
|
// Server-side push-wake fan-out: only when push is configured.
|
|
926
1014
|
if (this.config.push) {
|
|
927
1015
|
this.pushFanoutService = this.buildPushFanout(this.config.push);
|
|
@@ -1161,7 +1249,7 @@ export class CadreNode {
|
|
|
1161
1249
|
}
|
|
1162
1250
|
/**
|
|
1163
1251
|
* Construct (or adopt) the node-local joined-strand store (see
|
|
1164
|
-
* {@link joinedStrandStore}) and this session's
|
|
1252
|
+
* {@link joinedStrandStore}) and this session's {@link joinedStrands} over it.
|
|
1165
1253
|
*
|
|
1166
1254
|
* The store instance is kept across stop()→start(); the view is rebuilt. A store
|
|
1167
1255
|
* scoped to a different party is a configuration error, fail closed: its joins
|
|
@@ -1178,28 +1266,28 @@ export class CadreNode {
|
|
|
1178
1266
|
`but this node serves party ${partyId} — refusing to offer another party's joined strands`);
|
|
1179
1267
|
}
|
|
1180
1268
|
this.joinedStrandStore = store;
|
|
1181
|
-
this.
|
|
1269
|
+
this.joinedStrands = new JoinedStrandSession(store, this.createPartyJoinedStrandLedger());
|
|
1182
1270
|
}
|
|
1183
1271
|
/**
|
|
1184
|
-
* Construct (or adopt) the node-local strand
|
|
1185
|
-
* {@link
|
|
1272
|
+
* Construct (or adopt) the node-local strand network state (see
|
|
1273
|
+
* {@link strandNetworkStateStore}). Synchronous for the reason
|
|
1186
1274
|
* {@link initializeBootstrapPeerStore} is: an injected store has already loaded its
|
|
1187
1275
|
* persisted entries by the time it is handed in, and the in-memory fallback has
|
|
1188
1276
|
* nothing to load. The instance is kept across stop()→start(). A store scoped to a
|
|
1189
|
-
* different party is a configuration error, fail closed: another party's
|
|
1190
|
-
*
|
|
1277
|
+
* different party is a configuration error, fail closed: another party's routing
|
|
1278
|
+
* tables must never be imported into this node's strands.
|
|
1191
1279
|
*/
|
|
1192
|
-
|
|
1280
|
+
initializeStrandNetworkStateStore() {
|
|
1193
1281
|
const partyId = this.config.controlNetwork.partyId;
|
|
1194
|
-
if (this.
|
|
1282
|
+
if (this.strandNetworkStateStore) {
|
|
1195
1283
|
return;
|
|
1196
1284
|
}
|
|
1197
|
-
const store = this.config.
|
|
1285
|
+
const store = this.config.strandNetworkState?.store ?? new MemoryStrandNetworkStateStore(partyId);
|
|
1198
1286
|
if (store.partyId !== partyId) {
|
|
1199
|
-
throw new Error(`CadreNodeConfig:
|
|
1200
|
-
`but this node serves party ${partyId} — refusing to mix strand
|
|
1287
|
+
throw new Error(`CadreNodeConfig: strandNetworkState.store is scoped to party ${store.partyId}, ` +
|
|
1288
|
+
`but this node serves party ${partyId} — refusing to mix strand network state`);
|
|
1201
1289
|
}
|
|
1202
|
-
this.
|
|
1290
|
+
this.strandNetworkStateStore = store;
|
|
1203
1291
|
}
|
|
1204
1292
|
/**
|
|
1205
1293
|
* The node-local enrolled-machine store (null before {@link start}) — what this
|
|
@@ -1219,13 +1307,13 @@ export class CadreNode {
|
|
|
1219
1307
|
return this.bootstrapPeerStore;
|
|
1220
1308
|
}
|
|
1221
1309
|
/**
|
|
1222
|
-
* The node-local strand
|
|
1223
|
-
*
|
|
1224
|
-
* diagnostics and for a host that wants to show "
|
|
1225
|
-
*
|
|
1310
|
+
* The node-local strand network state (null before {@link start}) — per strand, the
|
|
1311
|
+
* state its strand node last saved and will re-import when next built. Exposed for
|
|
1312
|
+
* diagnostics and for a host that wants to show "what routing table would this
|
|
1313
|
+
* strand restart with?".
|
|
1226
1314
|
*/
|
|
1227
|
-
|
|
1228
|
-
return this.
|
|
1315
|
+
getStrandNetworkStateStore() {
|
|
1316
|
+
return this.strandNetworkStateStore;
|
|
1229
1317
|
}
|
|
1230
1318
|
/**
|
|
1231
1319
|
* The node-local trusted-owner anchor (null before {@link start}). This is
|
|
@@ -1391,7 +1479,7 @@ export class CadreNode {
|
|
|
1391
1479
|
const scope = controlStorageScope(this.config.controlNetwork.partyId);
|
|
1392
1480
|
// The second seam that hands a scope key to an embedder's provider; the first
|
|
1393
1481
|
// (`StrandInstanceManager.startStrand`) asserts the stricter strand rule. Holds by
|
|
1394
|
-
// construction today —
|
|
1482
|
+
// construction today — lowercase hex is inside the charset — so this guards a future
|
|
1395
1483
|
// edit to `controlStorageScope`, not a reachable input.
|
|
1396
1484
|
assertScopeKeyCharset(scope);
|
|
1397
1485
|
const resolved = typeof provider === 'function' ? provider(scope) : provider;
|
|
@@ -1458,7 +1546,6 @@ export class CadreNode {
|
|
|
1458
1546
|
// nothing); anything outside {tcp, ws/wss, p2p-circuit} throws here rather than
|
|
1459
1547
|
// being silently dropped at bring-up. See `relay-addrs.ts`.
|
|
1460
1548
|
const transportOptions = resolveTransportOptions(network, listenAddrs);
|
|
1461
|
-
const enableRelay = this.relayServerEnabled();
|
|
1462
1549
|
// The control node's own storage, resolved once per runtime (see
|
|
1463
1550
|
// {@link resolveControlStorage}) rather than per call.
|
|
1464
1551
|
const controlStorageProvider = this.resolveControlStorage();
|
|
@@ -1468,7 +1555,10 @@ export class CadreNode {
|
|
|
1468
1555
|
networkName: this.controlNetworkName(),
|
|
1469
1556
|
storage: controlStorageProvider,
|
|
1470
1557
|
fretProfile: profile === 'storage' ? 'core' : 'edge',
|
|
1471
|
-
relay:
|
|
1558
|
+
relay: this.relayServer.enabled,
|
|
1559
|
+
// Merged party-run defaults: no per-connection data/duration cap, a store sized for
|
|
1560
|
+
// one party's NAT'd machines. Strand nodes get the same init (`relay-server.ts`).
|
|
1561
|
+
...(this.relayServer.enabled && { relayServerInit: this.relayServer.init }),
|
|
1472
1562
|
// Fixed, and deliberately above any party's node count: every member reads the
|
|
1473
1563
|
// whole control database, so a cohort that excludes a member leaves it dependent
|
|
1474
1564
|
// on read repair — which cannot converge at a two-member cohort. Not a knob;
|
|
@@ -1484,12 +1574,14 @@ export class CadreNode {
|
|
|
1484
1574
|
// itself, so the unknown case is provably byte-for-byte the old behaviour.
|
|
1485
1575
|
// The yardstick moves ALONE: `assumedClusterSize` stays pinned at 2, because a
|
|
1486
1576
|
// party of phones cannot promise three quarters of its machines are awake.
|
|
1487
|
-
//
|
|
1488
|
-
//
|
|
1489
|
-
//
|
|
1577
|
+
// The read deadline is the host's own, else two link round trips at the declared
|
|
1578
|
+
// `linkRoundTripMs`, else nothing — the frozen constant already carries the same
|
|
1579
|
+
// derivation at the default declaration (`link-budget.ts`). With neither declared this
|
|
1580
|
+
// is still the frozen constant by identity, which is the production path. A degenerate
|
|
1581
|
+
// deadline is Optimystic's to refuse, in createControlNode below.
|
|
1490
1582
|
clusterPolicy: controlClusterPolicy({
|
|
1491
1583
|
enrolledMachines: this.declaredEnrolledMachines,
|
|
1492
|
-
cohortQueryTimeoutMs: network
|
|
1584
|
+
cohortQueryTimeoutMs: declaredCohortReadDeadlineMs(network)
|
|
1493
1585
|
}),
|
|
1494
1586
|
arachnode: { enableRingZulu: profile === 'storage' },
|
|
1495
1587
|
...(identityKey && { privateKey: identityKey }),
|
|
@@ -1500,10 +1592,12 @@ export class CadreNode {
|
|
|
1500
1592
|
// whose event loop is saturated by pure-JS Noise crypto. A configured value
|
|
1501
1593
|
// replaces the default whole (see DEFAULT_CONNECTION_MONITOR).
|
|
1502
1594
|
connectionMonitor: network?.connectionMonitor ?? DEFAULT_CONNECTION_MONITOR,
|
|
1503
|
-
//
|
|
1504
|
-
//
|
|
1505
|
-
//
|
|
1506
|
-
|
|
1595
|
+
// The declared link every cadre budget is derived from, stated to Optimystic too so it
|
|
1596
|
+
// derives its own deadlines from the same number: its request dials and responses, its
|
|
1597
|
+
// block pushes, and libp2p's `dialTimeout` and `inboundUpgradeTimeout`. Always stated,
|
|
1598
|
+
// default included: undeclared, Optimystic keeps LAN deadlines that cannot open a
|
|
1599
|
+
// relayed connection at the link sereus supports (`link-budget.ts`).
|
|
1600
|
+
linkRoundTripMs: resolveLinkRoundTripMs(network?.linkRoundTripMs),
|
|
1507
1601
|
// `{ wsPort }` when a listen entry names WebSocket, otherwise `{}` — and always
|
|
1508
1602
|
// `{}` when `network.transports` is set, since the embedder owns transport policy
|
|
1509
1603
|
// then. Spread NEXT to `transports` because the two answer the same question.
|
|
@@ -1526,7 +1620,7 @@ export class CadreNode {
|
|
|
1526
1620
|
admitInbound: (remotePeerId) => this.admitInboundControlConnection(remotePeerId),
|
|
1527
1621
|
admitRelayReservation: (remotePeerId) => this.admitControlRelayReservation(remotePeerId),
|
|
1528
1622
|
bringUpInFlight: () => this.controlBringUpInFlight
|
|
1529
|
-
}, network?.connectionGater),
|
|
1623
|
+
}, network?.connectionGater, ADMISSION_DECISION_TIMEOUT_MS, relayAdmissionReserveDeadlineMs(network?.linkRoundTripMs)),
|
|
1530
1624
|
// Fail-closed per-stream authorization for the four Optimystic control-DB
|
|
1531
1625
|
// protocols — the members-only layer the connection gater's stranger
|
|
1532
1626
|
// carve-outs cannot express (see authorizeInboundControlStream). Control
|
|
@@ -1536,15 +1630,17 @@ export class CadreNode {
|
|
|
1536
1630
|
return nodeOptions;
|
|
1537
1631
|
}
|
|
1538
1632
|
/**
|
|
1539
|
-
*
|
|
1540
|
-
*
|
|
1541
|
-
* (
|
|
1542
|
-
*
|
|
1543
|
-
* {@link admitInboundControlConnection} (whose deny/admit-for-relay branch
|
|
1544
|
-
* must agree with whether a reservation is even servable here).
|
|
1633
|
+
* One debug line naming what this machine's relay server forwards and holds —
|
|
1634
|
+
* the same fields the dedicated relay container prints at boot
|
|
1635
|
+
* (`ops/docker/libp2p-infra/src/main.ts`). Once per start, not per strand:
|
|
1636
|
+
* strand nodes resolve the same settings from the same config.
|
|
1545
1637
|
*/
|
|
1546
|
-
|
|
1547
|
-
|
|
1638
|
+
logRelayServerSettings() {
|
|
1639
|
+
if (!this.relayServer.enabled) {
|
|
1640
|
+
return;
|
|
1641
|
+
}
|
|
1642
|
+
const { applyDefaultLimit, maxReservations, reservationTtl } = this.relayServer.init.reservations;
|
|
1643
|
+
log('Relay server on: applyDefaultLimit=%s maxReservations=%d reservationTtl=%dms unauthorizedCap=%d', applyDefaultLimit, maxReservations, reservationTtl, this.unauthorizedRelayReservations.cap);
|
|
1548
1644
|
}
|
|
1549
1645
|
/**
|
|
1550
1646
|
* Decide whether an inbound CONTROL-network connection from `remotePeerId`
|
|
@@ -1593,7 +1689,7 @@ export class CadreNode {
|
|
|
1593
1689
|
* accepted it, exactly like the unreplicated-membership-row case below.
|
|
1594
1690
|
*
|
|
1595
1691
|
* When every check falls through, the verdict depends on whether this node
|
|
1596
|
-
* runs the circuit-relay server ({@link
|
|
1692
|
+
* runs the circuit-relay server ({@link relayServer}): without one,
|
|
1597
1693
|
* `'deny'`; with one, `'admit-for-relay'` — a circuit-relay reservation is
|
|
1598
1694
|
* established by the reserving peer DIALING the relay, so a connection deny
|
|
1599
1695
|
* here kills the reservation, and that deny is NOT self-healing: an outbound
|
|
@@ -1643,7 +1739,7 @@ export class CadreNode {
|
|
|
1643
1739
|
log('admitInboundControlConnection: outstanding-invitation check threw for %s — admitting (fail-open): %o', remotePeerId, error);
|
|
1644
1740
|
return 'admit';
|
|
1645
1741
|
}
|
|
1646
|
-
if (this.
|
|
1742
|
+
if (this.relayServer.enabled) {
|
|
1647
1743
|
log('admitInboundControlConnection: admitting %s FOR RELAY ONLY — not an authorized member and no enrollment path open; the gater drops the connection unless a reservation is admitted', remotePeerId);
|
|
1648
1744
|
return 'admit-for-relay';
|
|
1649
1745
|
}
|
|
@@ -1847,7 +1943,9 @@ export class CadreNode {
|
|
|
1847
1943
|
// the count of machines serving the network — the quantity
|
|
1848
1944
|
// `resolveRepairYardstick` asks for. It is NOT that quantity for a strand, which
|
|
1849
1945
|
// launches only on machines whose embedder registered its sApp config (see
|
|
1850
|
-
// {@link addStrand})
|
|
1946
|
+
// {@link addStrand}) plus storage-profile machines hosting it as a storage replica
|
|
1947
|
+
// ({@link CadreNodeConfig.hostUnclaimedStrands}) whose filter admits it — phones
|
|
1948
|
+
// serve only what their app claims — so this number must never be routed to a strand node:
|
|
1851
1949
|
// over-declaring pins Optimystic's repair corroboration floor at two peers and a
|
|
1852
1950
|
// strand that can field only one can then never repair. `launchStrand` therefore
|
|
1853
1951
|
// declares nothing; see the NOTE there.
|
|
@@ -1935,9 +2033,9 @@ export class CadreNode {
|
|
|
1935
2033
|
return this.bootstrapPeerIds;
|
|
1936
2034
|
}
|
|
1937
2035
|
/**
|
|
1938
|
-
* What the strand watcher polls: this party's control rows plus a row per
|
|
1939
|
-
*
|
|
1940
|
-
*
|
|
2036
|
+
* What the strand watcher polls: this party's control rows plus a row per cross-party
|
|
2037
|
+
* join (see {@link joinedStrands}), so a joined strand is offered and relaunched on
|
|
2038
|
+
* the same path as the party's own.
|
|
1941
2039
|
*/
|
|
1942
2040
|
createStrandQueryable() {
|
|
1943
2041
|
return {
|
|
@@ -1948,10 +2046,73 @@ export class CadreNode {
|
|
|
1948
2046
|
}
|
|
1949
2047
|
log('Querying strands from control database');
|
|
1950
2048
|
const control = await this.controlDatabase.queryStrands();
|
|
1951
|
-
return this.
|
|
2049
|
+
return this.joinedStrands ? await this.joinedStrands.withControlRows(control) : control;
|
|
1952
2050
|
}
|
|
1953
2051
|
};
|
|
1954
2052
|
}
|
|
2053
|
+
/**
|
|
2054
|
+
* The party-wide `JoinedStrand` table as {@link joinedStrands} reads and writes it.
|
|
2055
|
+
* Reads {@link controlDatabase} at call time, so start-up order does not matter.
|
|
2056
|
+
*/
|
|
2057
|
+
createPartyJoinedStrandLedger() {
|
|
2058
|
+
const database = () => {
|
|
2059
|
+
if (!this.controlDatabase) {
|
|
2060
|
+
throw new Error('CadreNode must be started before using the party-wide joined-strand table');
|
|
2061
|
+
}
|
|
2062
|
+
return this.controlDatabase;
|
|
2063
|
+
};
|
|
2064
|
+
return {
|
|
2065
|
+
// NOTE: one more control read per watcher poll (5 s), and for a party that never
|
|
2066
|
+
// joined anything it reads a never-written block, which consults the cohort. If poll
|
|
2067
|
+
// cost shows up in a device profile, cache the list and re-read it on the reconcile
|
|
2068
|
+
// cadence.
|
|
2069
|
+
list: () => database().queryJoinedStrands(),
|
|
2070
|
+
names: async (strandId) => (await database().queryStrand(strandId)) !== null
|
|
2071
|
+
|| (await database().queryJoinedStrand(strandId)) !== null,
|
|
2072
|
+
canSign: async () => (await this.enrolledOwnerSigningKey()) !== null,
|
|
2073
|
+
publish: async (record) => {
|
|
2074
|
+
const signingKey = await this.enrolledOwnerSigningKey();
|
|
2075
|
+
if (!signingKey) {
|
|
2076
|
+
throw new Error(`Cannot publish joined strand ${record.Id}: this machine is not an enrolled owner`);
|
|
2077
|
+
}
|
|
2078
|
+
try {
|
|
2079
|
+
await database().insertJoinedStrand(record, signingKey.publicKeyB64, signMessageWith(signingKey.privateKeyB64));
|
|
2080
|
+
}
|
|
2081
|
+
catch (error) {
|
|
2082
|
+
// Another machine of the party published this join first; its row wins.
|
|
2083
|
+
if (!isStrandIdConflict(error, 'JoinedStrand')) {
|
|
2084
|
+
throw error;
|
|
2085
|
+
}
|
|
2086
|
+
}
|
|
2087
|
+
},
|
|
2088
|
+
remove: async (strandId) => {
|
|
2089
|
+
const signingKey = await this.enrolledOwnerSigningKey();
|
|
2090
|
+
if (signingKey) {
|
|
2091
|
+
return database().deleteJoinedStrand(strandId, signingKey.publicKeyB64, signMessageWith(signingKey.privateKeyB64));
|
|
2092
|
+
}
|
|
2093
|
+
if (await database().queryJoinedStrand(strandId)) {
|
|
2094
|
+
throw new Error(`Cannot leave strand ${strandId} for the whole party: the join is recorded party-wide, and ` +
|
|
2095
|
+
`removing it takes an owner machine, which this one is not. stopStrand('${strandId}') stops ` +
|
|
2096
|
+
'it on this machine only.');
|
|
2097
|
+
}
|
|
2098
|
+
return false;
|
|
2099
|
+
}
|
|
2100
|
+
};
|
|
2101
|
+
}
|
|
2102
|
+
/**
|
|
2103
|
+
* This machine's owner signing key when it can make owner-gated control writes: its seed
|
|
2104
|
+
* bootstrap holds an owner key (`SeedBootstrapService.canAuthorize`) and the party's
|
|
2105
|
+
* `OwnerKey` table enrolls it. A phone that ran self-genesis after another machine
|
|
2106
|
+
* founded the party passes the first test and not the second (`runOwnerGenesis` in the
|
|
2107
|
+
* reference apps), and a write it signed would only be refused.
|
|
2108
|
+
*/
|
|
2109
|
+
async enrolledOwnerSigningKey() {
|
|
2110
|
+
const signingKey = this.seedBootstrapService?.canAuthorize() ? this.getSelfSigningKey() : null;
|
|
2111
|
+
if (!signingKey || !this.controlDatabase) {
|
|
2112
|
+
return null;
|
|
2113
|
+
}
|
|
2114
|
+
return (await this.controlDatabase.getOwnerKeys()).has(signingKey.publicKeyB64) ? signingKey : null;
|
|
2115
|
+
}
|
|
1955
2116
|
/**
|
|
1956
2117
|
* Schedule the node's initial self-record publish + ongoing refresh shortly
|
|
1957
2118
|
* after start (non-blocking). {@link registerSelf} is idempotent and safely
|
|
@@ -2569,14 +2730,18 @@ export class CadreNode {
|
|
|
2569
2730
|
// Ride this pass's cadence to refresh the per-stream gate's materialized
|
|
2570
2731
|
// authorized set — membership changes that ARRIVED BY REPLICATION (rather
|
|
2571
2732
|
// than a local write) are picked up here, bounding the snapshot's staleness
|
|
2572
|
-
// to the reconcile interval
|
|
2733
|
+
// to the time between pass starts: the reconcile interval, or the previous
|
|
2734
|
+
// pass's length when that is longer (passes are single-flight, and one
|
|
2735
|
+
// unreachable sibling's dial alone can take `controlDialBudget().totalMs`).
|
|
2573
2736
|
// NOTE: this refresh and the sibling enumeration below each run their own
|
|
2574
2737
|
// CadrePeer query (two reads per pass), plus a third from
|
|
2575
|
-
// `refreshStrandPeerAddrs` on
|
|
2576
|
-
// holds a control connection (one read for the whole pass, not one per
|
|
2577
|
-
// strand
|
|
2578
|
-
//
|
|
2579
|
-
//
|
|
2738
|
+
// `refreshStrandPeerAddrs` on every pass where a strand is running AND this
|
|
2739
|
+
// node holds a control connection (one read for the whole pass, not one per
|
|
2740
|
+
// strand; it runs even when no sibling is due, because pruning departed
|
|
2741
|
+
// siblings needs the current target set). Each also reads Revocation first.
|
|
2742
|
+
// Before the Revocation ledger marker exists (filed below, once connected)
|
|
2743
|
+
// that block is missing, so every one of those reads consults the cohort
|
|
2744
|
+
// about it; once the marker exists every
|
|
2580
2745
|
// block they touch is held and none of them does (both states pinned in
|
|
2581
2746
|
// control-founding-consult-budget.spec.ts). If those reads ever get costly,
|
|
2582
2747
|
// share one row-set across all three.
|
|
@@ -2592,10 +2757,10 @@ export class CadreNode {
|
|
|
2592
2757
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
2593
2758
|
return { dialed: [] };
|
|
2594
2759
|
}
|
|
2595
|
-
// Then re-warm each running strand's own address book from its
|
|
2596
|
-
//
|
|
2597
|
-
// below, one layer down: replication runs on the STRAND
|
|
2598
|
-
// layer under cadre-core dials strand peers by bare peer id.
|
|
2760
|
+
// Then re-warm each running strand's own address book from its FRET address
|
|
2761
|
+
// records and from whichever siblings are due an ask. Same reasoning as
|
|
2762
|
+
// warmSiblingAddrBook below, one layer down: replication runs on the STRAND
|
|
2763
|
+
// network, and every layer under cadre-core dials strand peers by bare peer id.
|
|
2599
2764
|
await this.refreshStrandPeerAddrs();
|
|
2600
2765
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
2601
2766
|
return { dialed: [] };
|
|
@@ -2662,6 +2827,19 @@ export class CadreNode {
|
|
|
2662
2827
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
2663
2828
|
return { dialed: [] };
|
|
2664
2829
|
}
|
|
2830
|
+
// Publish this machine's unpublished cross-party joins party-wide and remove the rows a
|
|
2831
|
+
// self-revocation queued (JoinedStrandSession.syncWithParty). Connected-only for the
|
|
2832
|
+
// reap's reason above; owner-only by the session's own signer check. Nothing is urgent:
|
|
2833
|
+
// a join reaches the party within one pass of this machine being connected.
|
|
2834
|
+
try {
|
|
2835
|
+
await this.joinedStrands?.syncWithParty();
|
|
2836
|
+
}
|
|
2837
|
+
catch (error) {
|
|
2838
|
+
log('reconcileControlCohort: joined-strand sync failed (retrying next pass): %o', error);
|
|
2839
|
+
}
|
|
2840
|
+
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
2841
|
+
return { dialed: [] };
|
|
2842
|
+
}
|
|
2665
2843
|
}
|
|
2666
2844
|
// 1. Enumerate known siblings. This membership read is itself a pull-on-read
|
|
2667
2845
|
// that helps the CadrePeer table converge — a reader-only node converges
|
|
@@ -2875,6 +3053,27 @@ export class CadreNode {
|
|
|
2875
3053
|
totalMs: cohort?.dialTimeoutMs ?? CONTROL_COHORT_DIAL_ADDRESS_ATTEMPTS * perAddressMs,
|
|
2876
3054
|
};
|
|
2877
3055
|
}
|
|
3056
|
+
/**
|
|
3057
|
+
* The link-derived limits shared by every {@link SeedBootstrapService} this node builds: its
|
|
3058
|
+
* owner and invite dials ({@link controlDialBudget}) and its seed delivery deadline, derived
|
|
3059
|
+
* from `network.linkRoundTripMs`. One helper so the four construction sites cannot drift.
|
|
3060
|
+
*/
|
|
3061
|
+
seedServiceBudgets() {
|
|
3062
|
+
return {
|
|
3063
|
+
dialBudget: this.controlDialBudget(),
|
|
3064
|
+
linkRoundTripMs: this.config.network?.linkRoundTripMs,
|
|
3065
|
+
};
|
|
3066
|
+
}
|
|
3067
|
+
/**
|
|
3068
|
+
* {@link collectStrandAddrs} at this node's declared `network.linkRoundTripMs`, so each ask's
|
|
3069
|
+
* deadline derives from it. One helper so the four call sites cannot drift.
|
|
3070
|
+
*/
|
|
3071
|
+
async collectSiblingStrandAddrs(controlNode, peers, strandId, delegatePeerId) {
|
|
3072
|
+
return await collectStrandAddrs(controlNode, peers, strandId, {
|
|
3073
|
+
delegatePeerId,
|
|
3074
|
+
linkRoundTripMs: this.config.network?.linkRoundTripMs,
|
|
3075
|
+
});
|
|
3076
|
+
}
|
|
2878
3077
|
/**
|
|
2879
3078
|
* Dial one sibling from its already-resolved addresses, best-effort. Returns
|
|
2880
3079
|
* whether the dial resolved (false when no address resolves or the dial fails).
|
|
@@ -2883,7 +3082,9 @@ export class CadreNode {
|
|
|
2883
3082
|
* the {@link controlDialBudget} expiring) is logged and swallowed so one
|
|
2884
3083
|
* unreachable sibling never aborts the pass — exactly like
|
|
2885
3084
|
* {@link SeedBootstrapService.applySeed}'s owner-dial loop. A failed dial is
|
|
2886
|
-
* simply retried on the next pass.
|
|
3085
|
+
* simply retried on the next pass. A sibling whose every address relays
|
|
3086
|
+
* through this node ({@link SelfRelayOnlyError}) is not dialed at all and gets
|
|
3087
|
+
* a one-line log instead: only the sibling can reconnect.
|
|
2887
3088
|
*/
|
|
2888
3089
|
async dialControlSibling(sibling, resolved) {
|
|
2889
3090
|
const controlNode = this.controlNode;
|
|
@@ -2901,7 +3102,12 @@ export class CadreNode {
|
|
|
2901
3102
|
return true;
|
|
2902
3103
|
}
|
|
2903
3104
|
catch (error) {
|
|
2904
|
-
|
|
3105
|
+
if (error instanceof SelfRelayOnlyError) {
|
|
3106
|
+
log('reconcileControlCohort: sibling %s is reachable only by relaying through this node; waiting for it to reconnect', sibling.peerId);
|
|
3107
|
+
}
|
|
3108
|
+
else {
|
|
3109
|
+
log('reconcileControlCohort: dial of sibling %s failed (continuing): %o', sibling.peerId, error);
|
|
3110
|
+
}
|
|
2905
3111
|
return false;
|
|
2906
3112
|
}
|
|
2907
3113
|
}
|
|
@@ -3145,7 +3351,12 @@ export class CadreNode {
|
|
|
3145
3351
|
return true;
|
|
3146
3352
|
}
|
|
3147
3353
|
catch (error) {
|
|
3148
|
-
|
|
3354
|
+
if (error instanceof SelfRelayOnlyError) {
|
|
3355
|
+
log('reconcileControlCohort(cold-start): bootstrap peer %s is reachable only by relaying through this node; waiting for it to reconnect', peerId);
|
|
3356
|
+
}
|
|
3357
|
+
else {
|
|
3358
|
+
log('reconcileControlCohort(cold-start): dial of bootstrap peer %s failed (continuing): %o', peerId, error);
|
|
3359
|
+
}
|
|
3149
3360
|
return false;
|
|
3150
3361
|
}
|
|
3151
3362
|
}
|
|
@@ -3162,7 +3373,7 @@ export class CadreNode {
|
|
|
3162
3373
|
// local-only). The remedy: detect "wrote while alone" (0 connections is a sound
|
|
3163
3374
|
// lower bound), queue the affected row, and RE-ISSUE the write once the cohort
|
|
3164
3375
|
// grows (0→≥1 connection), as an idempotent monotonic update that now broadcasts.
|
|
3165
|
-
// See docs/architecture.md (Control Network →
|
|
3376
|
+
// See docs/architecture.md (Control Network Convergence → "Writes made while alone").
|
|
3166
3377
|
// ============================================================================
|
|
3167
3378
|
/**
|
|
3168
3379
|
* Whether a control write happening now would commit local-only. A sound lower
|
|
@@ -3819,26 +4030,20 @@ export class CadreNode {
|
|
|
3819
4030
|
// Check if we have sApp config for this strand
|
|
3820
4031
|
const sAppConfig = this.sAppConfigs.get(strand.Id);
|
|
3821
4032
|
if (!sAppConfig) {
|
|
3822
|
-
|
|
3823
|
-
//
|
|
3824
|
-
//
|
|
3825
|
-
//
|
|
3826
|
-
//
|
|
3827
|
-
|
|
3828
|
-
|
|
3829
|
-
|
|
3830
|
-
// `getDiscoveredStrands()` sees this strand too. The watcher does not offer the
|
|
3831
|
-
// same strand twice — its `knownStrands` retains the id, and the one thing that
|
|
3832
|
-
// un-retains it (a failed `addStrand`, via `StrandWatcher.forgetStrand`) leaves the
|
|
3833
|
-
// sApp config registered, so the retry takes the auto-launch branch below rather
|
|
3834
|
-
// than this one. So this map — not the event — is what a late subscriber reads.
|
|
3835
|
-
// See the `strand:discovered` doc in types.ts.
|
|
3836
|
-
this.discoveredStrands.set(strand.Id, strand);
|
|
3837
|
-
this.emit('strand:discovered', { strandId: strand.Id, strand });
|
|
3838
|
-
return;
|
|
4033
|
+
this.announceDiscoveredStrand(strand);
|
|
4034
|
+
// Announce-only unless this node hosts storage replicas. Also skipped when a
|
|
4035
|
+
// listener claimed the strand synchronously (an `addStrand` call registers its
|
|
4036
|
+
// config before its first await): that claim is launching it with the app's schema,
|
|
4037
|
+
// and a replica launch here would only race it.
|
|
4038
|
+
if (!this.hostUnclaimedStrands || this.sAppConfigs.has(strand.Id)) {
|
|
4039
|
+
return;
|
|
4040
|
+
}
|
|
3839
4041
|
}
|
|
3840
4042
|
try {
|
|
3841
|
-
await this.launchStrand(strand, sAppConfig);
|
|
4043
|
+
const instance = await this.launchStrand(strand, sAppConfig);
|
|
4044
|
+
if (!sAppConfig) {
|
|
4045
|
+
void this.recordReplicaSAppId(instance.strandId);
|
|
4046
|
+
}
|
|
3842
4047
|
}
|
|
3843
4048
|
catch (error) {
|
|
3844
4049
|
log('Error starting strand %s: %o', strand.Id, error);
|
|
@@ -3855,6 +4060,59 @@ export class CadreNode {
|
|
|
3855
4060
|
throw error;
|
|
3856
4061
|
}
|
|
3857
4062
|
}
|
|
4063
|
+
/**
|
|
4064
|
+
* Offer a strand no local config claims to the hosting app as `strand:discovered`, so
|
|
4065
|
+
* it can decide whether to join (register a config + addStrand); the strand-agnostic
|
|
4066
|
+
* seam keeps this class free of any app's join policy.
|
|
4067
|
+
*
|
|
4068
|
+
* Once per strand: a storage-replica launch that fails is retried by the watcher
|
|
4069
|
+
* (`StrandWatcher.forgetStrand` + backoff), and each retry comes back through
|
|
4070
|
+
* {@link handleStrandAdded} with no config — the app must still see one announcement.
|
|
4071
|
+
* The other thing that un-retains a strand in the watcher (a failed `addStrand`) leaves
|
|
4072
|
+
* the sApp config registered, so that retry takes the claimed branch instead.
|
|
4073
|
+
*
|
|
4074
|
+
* Recorded BEFORE the emit so a handler that synchronously drains
|
|
4075
|
+
* `getDiscoveredStrands()` sees this strand too. So this map — not the event — is what
|
|
4076
|
+
* a late subscriber reads. See the `strand:discovered` doc in types.ts.
|
|
4077
|
+
*
|
|
4078
|
+
* NOTE: a replica launch that keeps failing leaves the watcher not tracking the strand,
|
|
4079
|
+
* so if its control row is removed during the backoff no `handleStrandRemoved` arrives
|
|
4080
|
+
* and this entry outlives the row. Needs a persistently failing launch AND a removal
|
|
4081
|
+
* inside the backoff; if it is ever seen, have the watcher report removal of rows it
|
|
4082
|
+
* forgot, since `detachStrand` is already a no-op for an untracked instance.
|
|
4083
|
+
*/
|
|
4084
|
+
announceDiscoveredStrand(strand) {
|
|
4085
|
+
if (this.discoveredStrands.has(strand.Id)) {
|
|
4086
|
+
return;
|
|
4087
|
+
}
|
|
4088
|
+
log('No sAppConfig registered for strand %s - emitting strand:discovered', strand.Id);
|
|
4089
|
+
this.discoveredStrands.set(strand.Id, strand);
|
|
4090
|
+
this.emit('strand:discovered', { strandId: strand.Id, strand });
|
|
4091
|
+
}
|
|
4092
|
+
/**
|
|
4093
|
+
* Record the sApp id a storage replica's `Strand.Header` names, for {@link getSAppId}.
|
|
4094
|
+
* Runs whenever a replica's database may have just been published (launch, the
|
|
4095
|
+
* first-sync gate opening, a wake); a no-op for a claimed strand, an unpublished
|
|
4096
|
+
* database, or an id already recorded. Never throws: a failed read logs and leaves the
|
|
4097
|
+
* id unknown, so an `sAppId` filter keeps the strand provisionally admitted — a
|
|
4098
|
+
* syncing replica, which is harmless.
|
|
4099
|
+
*/
|
|
4100
|
+
async recordReplicaSAppId(strandId) {
|
|
4101
|
+
const database = this.strandManager.getInstance(strandId)?.database;
|
|
4102
|
+
if (!database || this.sAppConfigs.has(strandId) || this.replicaSAppIds.has(strandId)) {
|
|
4103
|
+
return;
|
|
4104
|
+
}
|
|
4105
|
+
try {
|
|
4106
|
+
const sAppId = await readStrandHeaderSAppId(database.getDatabase());
|
|
4107
|
+
if (sAppId !== undefined) {
|
|
4108
|
+
this.replicaSAppIds.set(strandId, sAppId);
|
|
4109
|
+
log('Storage replica %s serves sApp %s', strandId, sAppId);
|
|
4110
|
+
}
|
|
4111
|
+
}
|
|
4112
|
+
catch (error) {
|
|
4113
|
+
log('Could not read the sApp id of storage replica %s (left unknown): %o', strandId, error);
|
|
4114
|
+
}
|
|
4115
|
+
}
|
|
3858
4116
|
async handleStrandRemoved(strandId) {
|
|
3859
4117
|
log('Handling strand removed: %s', strandId);
|
|
3860
4118
|
try {
|
|
@@ -3929,7 +4187,7 @@ export class CadreNode {
|
|
|
3929
4187
|
}
|
|
3930
4188
|
// Unregister strand solicitation service
|
|
3931
4189
|
if (this.strandSolicitationService && this.controlNode) {
|
|
3932
|
-
this.strandSolicitationService.unregisterResponder(this.controlNode);
|
|
4190
|
+
await this.strandSolicitationService.unregisterResponder(this.controlNode);
|
|
3933
4191
|
this.strandSolicitationService = null;
|
|
3934
4192
|
}
|
|
3935
4193
|
// Stop strand watcher
|
|
@@ -3943,6 +4201,7 @@ export class CadreNode {
|
|
|
3943
4201
|
this.sAppConfigs.clear();
|
|
3944
4202
|
this.strandLaunchRefusals.clear();
|
|
3945
4203
|
this.discoveredStrands.clear();
|
|
4204
|
+
this.replicaSAppIds.clear();
|
|
3946
4205
|
// Drop delegate-admission state: the grants are scoped to the session that
|
|
3947
4206
|
// recorded them, so a stop()/start() cycle on this object must not keep
|
|
3948
4207
|
// admitting delegates announced under the previous one, nor carry stale
|
|
@@ -4003,6 +4262,7 @@ export class CadreNode {
|
|
|
4003
4262
|
if (instance) {
|
|
4004
4263
|
this.hibernationManager.recordActivity(instance);
|
|
4005
4264
|
}
|
|
4265
|
+
void this.recordReplicaSAppId(strandId);
|
|
4006
4266
|
this.emit('strand:writable', { strandId });
|
|
4007
4267
|
}
|
|
4008
4268
|
// Hibernation callbacks
|
|
@@ -4016,27 +4276,18 @@ export class CadreNode {
|
|
|
4016
4276
|
}
|
|
4017
4277
|
/**
|
|
4018
4278
|
* Hibernate a strand: release its strand-network resources via the strand
|
|
4019
|
-
* manager (stop the libp2p node, close the StrandDatabase)
|
|
4279
|
+
* manager (stop the libp2p node, close the StrandDatabase), which marks it
|
|
4020
4280
|
* `hibernating`. A quiesced strand holds no open strand-network connections,
|
|
4021
|
-
* transports, or DB handles. No-ops if the strand is missing;
|
|
4022
|
-
*
|
|
4281
|
+
* transports, or DB handles. No-ops if the strand is missing; one already quiesced
|
|
4282
|
+
* is only marked, and still emits.
|
|
4023
4283
|
*/
|
|
4024
4284
|
async handleStrandHibernate(strandId) {
|
|
4025
|
-
|
|
4026
|
-
if (!instance) {
|
|
4285
|
+
if (!this.strandManager.getInstance(strandId)) {
|
|
4027
4286
|
log('handleStrandHibernate: strand %s not found', strandId);
|
|
4028
4287
|
return;
|
|
4029
4288
|
}
|
|
4030
|
-
if (!instance.libp2pNode && !instance.database) {
|
|
4031
|
-
// Already quiesced — only the status flag needs updating.
|
|
4032
|
-
instance.status = 'hibernating';
|
|
4033
|
-
log('Strand %s already quiesced; marked hibernating', strandId);
|
|
4034
|
-
this.emit('strand:hibernating', { strandId });
|
|
4035
|
-
return;
|
|
4036
|
-
}
|
|
4037
4289
|
log('Hibernating strand %s — releasing strand-network resources', strandId);
|
|
4038
4290
|
await this.strandManager.quiesceStrand(strandId);
|
|
4039
|
-
instance.status = 'hibernating';
|
|
4040
4291
|
this.emit('strand:hibernating', { strandId });
|
|
4041
4292
|
log('Strand %s hibernating (resources released)', strandId);
|
|
4042
4293
|
}
|
|
@@ -4046,7 +4297,13 @@ export class CadreNode {
|
|
|
4046
4297
|
* its runtime via the strand manager. If it is still live (e.g. waking an idle
|
|
4047
4298
|
* strand, which retains its resources), just flip the status. Overlapping wake
|
|
4048
4299
|
* triggers are coalesced upstream by `HibernationManager`, so this runs once
|
|
4049
|
-
* per wake;
|
|
4300
|
+
* per wake; a wake racing a check-in joins the check-in's rebuild in `resumeStrand`.
|
|
4301
|
+
*
|
|
4302
|
+
* A failed rebuild re-hibernates the strand (see {@link rehibernateIfResumeFailed});
|
|
4303
|
+
* every failure rethrows, so the waker still sees the error.
|
|
4304
|
+
*
|
|
4305
|
+
* Records no activity itself: whoever asked for the wake did ({@link wakeStrand},
|
|
4306
|
+
* `HibernationManager.recordActivity`), and {@link serviceWake}'s own probe must not.
|
|
4050
4307
|
*/
|
|
4051
4308
|
async handleStrandWake(strandId) {
|
|
4052
4309
|
const instance = this.strandManager.getInstance(strandId);
|
|
@@ -4055,10 +4312,14 @@ export class CadreNode {
|
|
|
4055
4312
|
return;
|
|
4056
4313
|
}
|
|
4057
4314
|
// Still live (idle wake, or defensive double-wake): no rebuild needed. A joiner
|
|
4058
|
-
// still behind its first-sync gate wakes back to `'syncing'`, not `'active'`.
|
|
4059
|
-
|
|
4315
|
+
// still behind its first-sync gate wakes back to `'syncing'`, not `'active'`. A
|
|
4316
|
+
// `'starting'` strand is mid-build (a check-in's resume, or its launch) with the node
|
|
4317
|
+
// attached before the database: not live yet, so it falls through and joins that build.
|
|
4318
|
+
// A strand being quiesced still holds its handles but is on its way down: it falls
|
|
4319
|
+
// through too, and its resume runs after the quiesce.
|
|
4320
|
+
if (instance.status !== 'starting' && (instance.libp2pNode || instance.database)
|
|
4321
|
+
&& !this.strandManager.isQuiescing(strandId)) {
|
|
4060
4322
|
instance.status = liveStrandStatus(instance);
|
|
4061
|
-
instance.lastActivity = new Date();
|
|
4062
4323
|
log('Strand %s woke (already live)', strandId);
|
|
4063
4324
|
this.emit('strand:waking', { strandId });
|
|
4064
4325
|
return;
|
|
@@ -4066,17 +4327,58 @@ export class CadreNode {
|
|
|
4066
4327
|
// Quiesced: re-resolve the volatile cohort input (the seed may have grown)
|
|
4067
4328
|
// and rebuild the runtime.
|
|
4068
4329
|
log('Waking strand %s — rebuilding strand-network resources', strandId);
|
|
4069
|
-
|
|
4070
|
-
|
|
4330
|
+
try {
|
|
4331
|
+
await this.resumeStrandRuntime(strandId);
|
|
4332
|
+
}
|
|
4333
|
+
catch (error) {
|
|
4334
|
+
await this.rehibernateIfResumeFailed(instance, 'Wake', error);
|
|
4335
|
+
throw error;
|
|
4336
|
+
}
|
|
4071
4337
|
this.emit('strand:waking', { strandId });
|
|
4072
4338
|
log('Strand %s awake (resources rebuilt)', strandId);
|
|
4073
4339
|
}
|
|
4340
|
+
/**
|
|
4341
|
+
* Undo a failed {@link resumeStrandRuntime}, but only when the rebuild itself failed — this
|
|
4342
|
+
* caller's, or the one it joined in `resumeStrand` — which leaves the instance `'error'`. A
|
|
4343
|
+
* failure before the rebuild (the cohort seed read) built nothing, and may find the strand
|
|
4344
|
+
* mid-build or just brought up by a concurrent wake or check-in; that runtime belongs to
|
|
4345
|
+
* whoever is building or holding it, so quiescing it here would stop a node still in use.
|
|
4346
|
+
*/
|
|
4347
|
+
async rehibernateIfResumeFailed(instance, context, error) {
|
|
4348
|
+
if (instance.status !== 'error') {
|
|
4349
|
+
log('%s of strand %s failed before its rebuild (status=%s); leaving the strand as it is: %o', context, instance.strandId, instance.status, error);
|
|
4350
|
+
return;
|
|
4351
|
+
}
|
|
4352
|
+
log('%s of strand %s failed; re-hibernating so a later wake or check-in retries: %o', context, instance.strandId, error);
|
|
4353
|
+
await this.rehibernateAfterFailedResume(instance, context);
|
|
4354
|
+
}
|
|
4355
|
+
/**
|
|
4356
|
+
* Put a strand whose rebuild or wake window failed back to `'hibernating'`, through a
|
|
4357
|
+
* best-effort quiesce that releases any runtime left up. `resumeStrand` leaves a failed
|
|
4358
|
+
* strand `'error'`, which nothing retries: `HibernationManager` wakes only
|
|
4359
|
+
* `idle`/`hibernating` strands, and reads any other status after a check-in as "woke",
|
|
4360
|
+
* ending its chain. Safe to run twice for one failure (a wake that joined a failed
|
|
4361
|
+
* check-in's rebuild): quiescing a quiesced strand only marks it.
|
|
4362
|
+
*
|
|
4363
|
+
* The quiesce writes the status itself, ordered before any resume requested meanwhile;
|
|
4364
|
+
* this writes it only when the quiesce failed, since a failed release leaves the status
|
|
4365
|
+
* alone.
|
|
4366
|
+
*/
|
|
4367
|
+
async rehibernateAfterFailedResume(instance, context) {
|
|
4368
|
+
try {
|
|
4369
|
+
await this.strandManager.quiesceStrand(instance.strandId);
|
|
4370
|
+
}
|
|
4371
|
+
catch (cleanupErr) {
|
|
4372
|
+
log('%s cleanup quiesce for strand %s failed: %o', context, instance.strandId, cleanupErr);
|
|
4373
|
+
instance.status = 'hibernating';
|
|
4374
|
+
}
|
|
4375
|
+
}
|
|
4074
4376
|
/**
|
|
4075
4377
|
* Rebuild a quiesced strand's runtime, re-resolving the volatile cohort input
|
|
4076
4378
|
* first: the discovery seed may have grown since the strand last ran. Shared by
|
|
4077
4379
|
* the wake (`handleStrandWake`) and check-in (`handleStrandCheckIn`) paths so both
|
|
4078
|
-
* apply the same fresh resolution. `resumeStrand`
|
|
4079
|
-
*
|
|
4380
|
+
* apply the same fresh resolution. `resumeStrand` returns a live instance unchanged
|
|
4381
|
+
* and joins a rebuild already in flight, so the two paths never build twice.
|
|
4080
4382
|
*/
|
|
4081
4383
|
async resumeStrandRuntime(strandId) {
|
|
4082
4384
|
// Re-derive the transport peerId (deterministic and cheap — a quiesced
|
|
@@ -4095,6 +4397,7 @@ export class CadreNode {
|
|
|
4095
4397
|
if (instance.libp2pNode) {
|
|
4096
4398
|
await this.mergeStrandPeerAddrs(instance.libp2pNode, bootstrapNodes, strandId);
|
|
4097
4399
|
}
|
|
4400
|
+
void this.recordReplicaSAppId(strandId);
|
|
4098
4401
|
}
|
|
4099
4402
|
/**
|
|
4100
4403
|
* Real cohort check-in for a hibernating strand (the `onCheckIn` callback).
|
|
@@ -4127,62 +4430,77 @@ export class CadreNode {
|
|
|
4127
4430
|
log('handleStrandCheckIn: strand %s not hibernating (status=%s); skipping', strandId, instance.status);
|
|
4128
4431
|
return;
|
|
4129
4432
|
}
|
|
4433
|
+
// Before the resume, so a wake that joins this rebuild, or activity recorded while it
|
|
4434
|
+
// runs, counts: the window then leaves the strand up for it.
|
|
4435
|
+
const activityMark = instance.lastActivity;
|
|
4436
|
+
// 1. Resume exactly as a wake does: re-resolve the (possibly grown) cohort
|
|
4437
|
+
// seed, then rebuild the runtime.
|
|
4438
|
+
log('Check-in: resuming strand %s to probe the cohort for pending activity', strandId);
|
|
4130
4439
|
try {
|
|
4131
|
-
// 1. Resume exactly as a wake does: re-resolve the (possibly grown) cohort
|
|
4132
|
-
// seed, then rebuild the runtime.
|
|
4133
|
-
log('Check-in: resuming strand %s to probe the cohort for pending activity', strandId);
|
|
4134
4440
|
await this.resumeStrandRuntime(strandId);
|
|
4135
|
-
// 2-3. Bounded window for the strand network to connect + the app to act,
|
|
4136
|
-
// then re-hibernate-if-idle. Shared with the on-demand serviceWake.
|
|
4137
|
-
const windowMs = this.config.hibernation?.checkInWindowMs ?? DEFAULT_CHECKIN_WINDOW_MS;
|
|
4138
|
-
await this.runWakeWindow(instance, windowMs);
|
|
4139
4441
|
}
|
|
4140
4442
|
catch (err) {
|
|
4141
|
-
//
|
|
4142
|
-
//
|
|
4143
|
-
//
|
|
4144
|
-
//
|
|
4145
|
-
//
|
|
4146
|
-
//
|
|
4147
|
-
//
|
|
4148
|
-
//
|
|
4149
|
-
|
|
4150
|
-
|
|
4151
|
-
|
|
4152
|
-
|
|
4153
|
-
|
|
4154
|
-
|
|
4155
|
-
|
|
4443
|
+
// Every failure resolves: `HibernationManager.runCheckIn` decides wake-vs-escalate
|
|
4444
|
+
// from `instance.status` alone. A failed rebuild — on a flaky network, the very
|
|
4445
|
+
// scenario hibernation targets — leaves `resumeStrand`'s `error`, which reads as
|
|
4446
|
+
// "woke" and would STOP the chain with no runtime and no future check-in, so it goes
|
|
4447
|
+
// back to `hibernating` and the manager escalates the backoff and retries. A failure
|
|
4448
|
+
// before the rebuild (the seed read) built nothing: the strand is still `hibernating`
|
|
4449
|
+
// (the manager escalates), or a wake that began meanwhile is building or holding it,
|
|
4450
|
+
// and the manager hands the chain to that wake, whose outcome restores or ends it.
|
|
4451
|
+
await this.rehibernateIfResumeFailed(instance, 'Check-in', err);
|
|
4452
|
+
return;
|
|
4453
|
+
}
|
|
4454
|
+
// 2-3. Bounded window for the strand network to connect + the app to act,
|
|
4455
|
+
// then re-hibernate-if-idle. Shared with the on-demand serviceWake.
|
|
4456
|
+
const windowMs = this.config.hibernation?.checkInWindowMs ?? DEFAULT_CHECKIN_WINDOW_MS;
|
|
4457
|
+
try {
|
|
4458
|
+
await this.runWakeWindow(instance, activityMark, windowMs);
|
|
4459
|
+
}
|
|
4460
|
+
catch (err) {
|
|
4461
|
+
// The window's quiesce threw with the rebuilt runtime up: release it best-effort and
|
|
4462
|
+
// leave the strand `hibernating`, for the same reason as a failed rebuild above.
|
|
4463
|
+
log('Check-in window failed for strand %s; re-hibernating to retry on backoff: %o', strandId, err);
|
|
4464
|
+
await this.rehibernateAfterFailedResume(instance, 'Check-in');
|
|
4156
4465
|
}
|
|
4157
4466
|
}
|
|
4158
4467
|
/**
|
|
4159
4468
|
* Window-then-decide for a just-resumed strand, shared by the check-in timer
|
|
4160
4469
|
* path ({@link handleStrandCheckIn}) and the on-demand {@link serviceWake}:
|
|
4161
4470
|
*
|
|
4162
|
-
* 1.
|
|
4163
|
-
* FRESH `Date`, so a changed reference after the window means real
|
|
4164
|
-
* activity landed during it — not millisecond-resolution noise.
|
|
4165
|
-
* 2. Hold the strand live for `windowMs` so its strand network reaches the
|
|
4471
|
+
* 1. Hold the strand live for `windowMs` so its strand network reaches the
|
|
4166
4472
|
* cohort and the app can drive pull-on-read activity.
|
|
4167
|
-
*
|
|
4168
|
-
* quiesce
|
|
4473
|
+
* 2. If activity landed since `activityMark`, leave the strand `active` (return
|
|
4474
|
+
* `true`); otherwise quiesce it, which marks it `hibernating` again (return `false`).
|
|
4475
|
+
* A wake or activity that lands during that quiesce rebuilds the strand after it.
|
|
4169
4476
|
*
|
|
4170
|
-
* @
|
|
4477
|
+
* @param activityMark - `instance.lastActivity` as the caller read it BEFORE bringing the
|
|
4478
|
+
* strand up. The bring-up records none, and every writer assigns a FRESH `Date`, so a
|
|
4479
|
+
* changed reference means a wake or activity landed during the resume or the window —
|
|
4480
|
+
* not millisecond-resolution noise.
|
|
4481
|
+
* @returns whether activity was observed (strand left active).
|
|
4171
4482
|
*/
|
|
4172
|
-
async runWakeWindow(instance, windowMs) {
|
|
4483
|
+
async runWakeWindow(instance, activityMark, windowMs) {
|
|
4173
4484
|
const strandId = instance.strandId;
|
|
4174
|
-
|
|
4485
|
+
// NOTE: activity during the window arms the idle countdown, so a window longer than the
|
|
4486
|
+
// hint's idle + hibernate timeouts (only reachable with a custom `windowMs`; the shortest
|
|
4487
|
+
// default pair, archive's, is 40 s vs a 15 s window) can let the timer path quiesce the
|
|
4488
|
+
// strand mid-window, which this then marks live. If such windows become real, suspend the
|
|
4489
|
+
// idle countdown while a window holds the strand.
|
|
4175
4490
|
await this.holdWakeWindow(instance, windowMs);
|
|
4176
4491
|
const sawActivity = instance.lastActivity !== activityMark;
|
|
4177
4492
|
if (sawActivity) {
|
|
4178
|
-
|
|
4179
|
-
|
|
4180
|
-
|
|
4493
|
+
// A force-hibernate during the window has released the runtime and marked the strand
|
|
4494
|
+
// `hibernating`, which stands; a wake queued behind it writes its own status.
|
|
4495
|
+
if (instance.libp2pNode || instance.database) {
|
|
4496
|
+
instance.status = liveStrandStatus(instance);
|
|
4497
|
+
this.emit('strand:waking', { strandId });
|
|
4498
|
+
}
|
|
4499
|
+
log('Wake window: strand %s saw activity during the window; not re-hibernating', strandId);
|
|
4181
4500
|
return true;
|
|
4182
4501
|
}
|
|
4183
4502
|
log('Wake window: no activity for strand %s; re-hibernating', strandId);
|
|
4184
4503
|
await this.strandManager.quiesceStrand(strandId);
|
|
4185
|
-
instance.status = 'hibernating';
|
|
4186
4504
|
return false;
|
|
4187
4505
|
}
|
|
4188
4506
|
/**
|
|
@@ -4230,9 +4548,10 @@ export class CadreNode {
|
|
|
4230
4548
|
* published (e.g. re-attaching its own orphan after a restart) founds it, and attaching
|
|
4231
4549
|
* anyone else's row joins, without the caller needing to know which it is.
|
|
4232
4550
|
*
|
|
4233
|
-
* A joined row (no `founder: true`) this party's control database does not
|
|
4234
|
-
* strand joined from another party — is remembered in the node's joined-strand store, so it
|
|
4235
|
-
* `strand:discovered` after a restart
|
|
4551
|
+
* A joined row (no `founder: true`) this party's control database does not name — a
|
|
4552
|
+
* strand joined from another party — is remembered in the node's joined-strand store, so it
|
|
4553
|
+
* is re-offered as `strand:discovered` after a restart, and published party-wide by an
|
|
4554
|
+
* owner machine's reconcile pass (see {@link forgetJoinedStrand}).
|
|
4236
4555
|
*
|
|
4237
4556
|
* A rejected call leaves nothing running but DOES leave the sApp config
|
|
4238
4557
|
* registered, deliberately: both an explicit retry and the {@link StrandWatcher}'s
|
|
@@ -4458,7 +4777,7 @@ export class CadreNode {
|
|
|
4458
4777
|
catch (error) {
|
|
4459
4778
|
// Only the uniqueness collision is a candidate for the idempotent reading; every
|
|
4460
4779
|
// other rejection (unauthorized signer, retired stamp) must keep surfacing.
|
|
4461
|
-
if (!isStrandIdConflict(error)) {
|
|
4780
|
+
if (!isStrandIdConflict(error, 'Strand')) {
|
|
4462
4781
|
throw error;
|
|
4463
4782
|
}
|
|
4464
4783
|
const landed = await this.controlDatabase.queryStrand(trimmed);
|
|
@@ -4606,7 +4925,7 @@ export class CadreNode {
|
|
|
4606
4925
|
* — and with it the consent re-seat foreclosure — does propagate; but the physical row
|
|
4607
4926
|
* deletion cannot be replayed, and `queryStrands` reads raw (no retired-stamp filter),
|
|
4608
4927
|
* so siblings that already hold the row keep running the strand until the collection
|
|
4609
|
-
* itself converges (logged loudly; see
|
|
4928
|
+
* itself converges (logged loudly; see "Deletes made while alone" in
|
|
4610
4929
|
* docs/architecture.md).
|
|
4611
4930
|
*
|
|
4612
4931
|
* A no-op (no throw, no tombstone) when the row is already absent — but a
|
|
@@ -4752,10 +5071,10 @@ export class CadreNode {
|
|
|
4752
5071
|
* NOTE: "founding machine" here is the ROW's provenance, not the launch's resolved
|
|
4753
5072
|
* `founder` flag — so an explicit `founder: false` over a row this machine published
|
|
4754
5073
|
* still mints, an owner-signed write a caller that said "I am not founding" did not ask
|
|
4755
|
-
* for.
|
|
4756
|
-
*
|
|
4757
|
-
*
|
|
4758
|
-
*
|
|
5074
|
+
* for. Reached by a storage replica of a row this machine published (always a joiner,
|
|
5075
|
+
* see {@link launchStrand}) and kept on purpose: the party needs that identity for the
|
|
5076
|
+
* strand either way, the mint is insert-if-absent, and a closed-strand replica runs its
|
|
5077
|
+
* membership reconciler — which seats its own `MemberPeer` binding — only with a party key.
|
|
4759
5078
|
*/
|
|
4760
5079
|
async resolveStrandPartyKey(strand, explicitKey) {
|
|
4761
5080
|
if (explicitKey !== undefined) {
|
|
@@ -4921,9 +5240,18 @@ export class CadreNode {
|
|
|
4921
5240
|
* A founder launch refused as pre-split (`PreSplitStrandIdentityError`) is recorded in
|
|
4922
5241
|
* {@link strandLaunchRefusals} for the formation arm, then rethrown; a founder launch
|
|
4923
5242
|
* that succeeds clears the record (it ran the bootstrap's pre-split check and passed).
|
|
5243
|
+
*
|
|
5244
|
+
* An absent `sAppConfig` launches a storage replica ({@link CadreNodeConfig.hostUnclaimedStrands}),
|
|
5245
|
+
* which is ALWAYS a joiner whatever the row says: the founder bootstrap writes the sApp
|
|
5246
|
+
* into `Strand.Header`, and a replica has none. So a self-founded row whose watcher poll
|
|
5247
|
+
* wins the race against its app's `addStrand` after a restart comes up as a replica — and
|
|
5248
|
+
* so does a `foundStrand` whose publish a watcher poll saw before its attach. A claim
|
|
5249
|
+
* that finds a tracked replica upgrades it in place, over the same node and store
|
|
5250
|
+
* ({@link StrandInstanceManager.attachSApp}), BEFORE any founding: the founder bootstrap
|
|
5251
|
+
* needs the sApp the attach supplies.
|
|
4924
5252
|
*/
|
|
4925
5253
|
async launchStrand(strand, sAppConfig, founder, explicitPartyKey) {
|
|
4926
|
-
const resolvedFounder = founder ?? this.isSelfFoundedRow(strand);
|
|
5254
|
+
const resolvedFounder = sAppConfig ? (founder ?? this.isSelfFoundedRow(strand)) : false;
|
|
4927
5255
|
try {
|
|
4928
5256
|
const instance = await this.startOrFoundStrand(strand, sAppConfig, resolvedFounder, explicitPartyKey);
|
|
4929
5257
|
if (resolvedFounder) {
|
|
@@ -4944,41 +5272,8 @@ export class CadreNode {
|
|
|
4944
5272
|
*/
|
|
4945
5273
|
async startOrFoundStrand(strand, sAppConfig, resolvedFounder, explicitPartyKey) {
|
|
4946
5274
|
const timed = (step, op) => timedStep('startOrFoundStrand', strand.Id, step, op);
|
|
4947
|
-
|
|
4948
|
-
|
|
4949
|
-
if (resolvedFounder) {
|
|
4950
|
-
// The resolver runs only when the retained config lacks a party key for a
|
|
4951
|
-
// closed strand (see foundExistingStrand), so the common watcher re-entry
|
|
4952
|
-
// ('already-founder') still costs no control read.
|
|
4953
|
-
const outcome = await timed('foundExistingStrand', () => this.strandManager.foundExistingStrand(strand.Id, () => this.resolveStrandPartyKey(strand, explicitPartyKey)));
|
|
4954
|
-
if (outcome === 'needs-resume') {
|
|
4955
|
-
try {
|
|
4956
|
-
// Quiesced instance: the retained config now founds, but founding promises
|
|
4957
|
-
// the bootstrap has RUN by the time the caller resolves — wake through the
|
|
4958
|
-
// hibernation manager (coalesced with any in-flight wake, timer-aware) so
|
|
4959
|
-
// the rebuild executes it now rather than at some eventual wake.
|
|
4960
|
-
await timed('wakeStrand', () => this.wakeStrand(strand.Id));
|
|
4961
|
-
// The wake's rebuild founds — UNLESS a wake was already in flight when the
|
|
4962
|
-
// config flipped, in which case it had already read the pre-flip config and
|
|
4963
|
-
// rebuilt as a joiner, and `wakeStrand` merely coalesced onto it. Re-run the
|
|
4964
|
-
// (insert-if-absent) bootstrap so founding never resolves headerless.
|
|
4965
|
-
await timed('ensureFounderBootstrap', () => this.strandManager.ensureFounderBootstrap(strand.Id));
|
|
4966
|
-
}
|
|
4967
|
-
catch (error) {
|
|
4968
|
-
// The founding did not happen (e.g. the rebuild refused a pre-split strand and
|
|
4969
|
-
// rolled back, leaving the instance tracked with no runtime): withdraw the flip
|
|
4970
|
-
// so the next attempt re-runs the founding instead of resolving
|
|
4971
|
-
// 'already-founder' over an instance that never founded.
|
|
4972
|
-
this.strandManager.withdrawFounderRequest(strand.Id);
|
|
4973
|
-
throw error;
|
|
4974
|
-
}
|
|
4975
|
-
}
|
|
4976
|
-
log('launchStrand: strand %s already tracked — founder request honored (%s)', strand.Id, outcome);
|
|
4977
|
-
}
|
|
4978
|
-
else {
|
|
4979
|
-
log('launchStrand: strand %s already tracked locally — skipping re-launch', strand.Id);
|
|
4980
|
-
}
|
|
4981
|
-
return existing;
|
|
5275
|
+
if (this.strandManager.getInstance(strand.Id)) {
|
|
5276
|
+
return this.claimTrackedStrand(strand, sAppConfig, resolvedFounder, explicitPartyKey);
|
|
4982
5277
|
}
|
|
4983
5278
|
// A closed strand's launch carries the party's OWN membership identity key: the
|
|
4984
5279
|
// explicit attach-time key, else the control-layer StrandPartyKey row — minted here
|
|
@@ -5000,9 +5295,9 @@ export class CadreNode {
|
|
|
5000
5295
|
// `privateKey` — tests, and an embedder that opted out of a stable identity) the
|
|
5001
5296
|
// strand node still runs under a key CADRE-CORE HOLDS: a fresh random Ed25519 key,
|
|
5002
5297
|
// exactly what libp2p would generate internally if handed none, except that the
|
|
5003
|
-
//
|
|
5004
|
-
// the retained launch config keeps
|
|
5005
|
-
//
|
|
5298
|
+
// delegate announcement below can name the strand peer id before the node exists and
|
|
5299
|
+
// the retained launch config keeps that id across a hibernation resume. Stability
|
|
5300
|
+
// across RESTARTS still needs an identity key.
|
|
5006
5301
|
//
|
|
5007
5302
|
// NOTE: derivation requires an Ed25519 identity key, so a node configured
|
|
5008
5303
|
// with some other key type now fails strand launch outright (surfaced as
|
|
@@ -5021,6 +5316,14 @@ export class CadreNode {
|
|
|
5021
5316
|
// client awaits the replies).
|
|
5022
5317
|
const delegatePeerId = transportKey ? peerIdFromPrivateKey(transportKey).toString() : undefined;
|
|
5023
5318
|
const bootstrapNodes = await timed('resolveCohortSeed', () => this.resolveCohortSeed(strand.Id, delegatePeerId));
|
|
5319
|
+
// Checked again after the awaits above: another launch of this strand may have started
|
|
5320
|
+
// meanwhile (a replica's watcher launch racing an app's claim, in either order), and
|
|
5321
|
+
// `startStrand` would hand its instance back unchanged — a claim holding a replica with
|
|
5322
|
+
// no `App` tables, a founder request dropped. Nothing awaits between this check and
|
|
5323
|
+
// `startStrand`'s own, so no third launch can slip in.
|
|
5324
|
+
if (this.strandManager.getInstance(strand.Id)) {
|
|
5325
|
+
return this.claimTrackedStrand(strand, sAppConfig, resolvedFounder, explicitPartyKey);
|
|
5326
|
+
}
|
|
5024
5327
|
const instance = await timed('strandManager.startStrand', () => this.strandManager.startStrand({
|
|
5025
5328
|
strandRow: strand,
|
|
5026
5329
|
sAppConfig,
|
|
@@ -5046,15 +5349,12 @@ export class CadreNode {
|
|
|
5046
5349
|
this.forgetRevokedJoin(revokedStrandId);
|
|
5047
5350
|
this.emit('strand:revoked', { strandId: revokedStrandId });
|
|
5048
5351
|
},
|
|
5049
|
-
//
|
|
5050
|
-
//
|
|
5051
|
-
//
|
|
5052
|
-
|
|
5053
|
-
//
|
|
5054
|
-
|
|
5055
|
-
// the launch config too. `?? undefined`: the field is `null` before start(), and a
|
|
5056
|
-
// strand cannot launch before start(), so this is belt and braces.
|
|
5057
|
-
strandPeerBook: this.strandPeerBookStore ?? undefined,
|
|
5352
|
+
// Where the strand node saves its network state and what it re-imports when it is
|
|
5353
|
+
// built. Retained with the launch config, so a hibernation wake rebuilds the node
|
|
5354
|
+
// over the table the quiesced node last saved. `?? undefined`: the field is `null`
|
|
5355
|
+
// before start(), and a strand cannot launch before start(), so this is belt and
|
|
5356
|
+
// braces.
|
|
5357
|
+
networkState: this.strandNetworkStateStore ?? undefined,
|
|
5058
5358
|
onRejoinBlocked: (blockedStrandId) => this.emit('strand:rejoin-blocked', { strandId: blockedStrandId }),
|
|
5059
5359
|
// The joiner's first-sync write gate (strand-first-sync-gate.ts): a launch that
|
|
5060
5360
|
// comes up `'syncing'` announces the moment its database is published.
|
|
@@ -5089,6 +5389,63 @@ export class CadreNode {
|
|
|
5089
5389
|
this.emit('strand:started', { strandId: strand.Id });
|
|
5090
5390
|
return instance;
|
|
5091
5391
|
}
|
|
5392
|
+
/**
|
|
5393
|
+
* {@link startOrFoundStrand} for a strand the manager already tracks: a claim of a
|
|
5394
|
+
* storage replica gives it the app's schema in place, then a founder request founds it,
|
|
5395
|
+
* waking a quiesced instance so the bootstrap has run before this resolves. Emits
|
|
5396
|
+
* nothing — `strand:started` fired when the instance launched.
|
|
5397
|
+
*
|
|
5398
|
+
* A launch of the strand still in flight (the watcher's launch racing an app's
|
|
5399
|
+
* `addStrand`, in either order) is waited out first: until its build settles the tracked
|
|
5400
|
+
* instance is `'starting'` with no database, and returning it would resolve `addStrand`
|
|
5401
|
+
* with a strand the app cannot use — and skip that call's first-sync wait, which only a
|
|
5402
|
+
* `'syncing'` instance gets. If that launch fails, this rejects too: its record is gone.
|
|
5403
|
+
*/
|
|
5404
|
+
async claimTrackedStrand(strand, sAppConfig, resolvedFounder, explicitPartyKey) {
|
|
5405
|
+
const timed = (step, op) => timedStep('startOrFoundStrand', strand.Id, step, op);
|
|
5406
|
+
const existing = await this.strandManager.whenRuntimeBuilt(strand.Id);
|
|
5407
|
+
if (!existing) {
|
|
5408
|
+
throw new Error(`Strand ${strand.Id}: the launch this one waited on failed (reported by the call that started it)`);
|
|
5409
|
+
}
|
|
5410
|
+
// Before the founder branch: the founder bootstrap writes the sApp into Strand.Header,
|
|
5411
|
+
// so it must run against the attached config.
|
|
5412
|
+
if (sAppConfig && !existing.sAppInfo) {
|
|
5413
|
+
const attached = await timed('attachSApp', () => this.strandManager.attachSApp(strand.Id, sAppConfig, { requireSignedSchemas: this.config.requireSignedSchemas }));
|
|
5414
|
+
log('launchStrand: strand %s was running as a storage replica — sApp %s claimed it (%s)', strand.Id, sAppConfig.id, attached);
|
|
5415
|
+
}
|
|
5416
|
+
if (!resolvedFounder) {
|
|
5417
|
+
log('launchStrand: strand %s already tracked locally — skipping re-launch', strand.Id);
|
|
5418
|
+
return existing;
|
|
5419
|
+
}
|
|
5420
|
+
// The resolver runs only when the retained config lacks a party key for a
|
|
5421
|
+
// closed strand (see foundExistingStrand), so the common watcher re-entry
|
|
5422
|
+
// ('already-founder') still costs no control read.
|
|
5423
|
+
const outcome = await timed('foundExistingStrand', () => this.strandManager.foundExistingStrand(strand.Id, () => this.resolveStrandPartyKey(strand, explicitPartyKey)));
|
|
5424
|
+
if (outcome === 'needs-resume') {
|
|
5425
|
+
try {
|
|
5426
|
+
// Quiesced instance: the retained config now founds, but founding promises
|
|
5427
|
+
// the bootstrap has RUN by the time the caller resolves — wake through the
|
|
5428
|
+
// hibernation manager (coalesced with any in-flight wake, timer-aware) so
|
|
5429
|
+
// the rebuild executes it now rather than at some eventual wake.
|
|
5430
|
+
await timed('wakeStrand', () => this.wakeStrand(strand.Id));
|
|
5431
|
+
// The wake's rebuild founds — UNLESS a wake was already in flight when the
|
|
5432
|
+
// config flipped, in which case it had already read the pre-flip config and
|
|
5433
|
+
// rebuilt as a joiner, and `wakeStrand` merely coalesced onto it. Re-run the
|
|
5434
|
+
// (insert-if-absent) bootstrap so founding never resolves headerless.
|
|
5435
|
+
await timed('ensureFounderBootstrap', () => this.strandManager.ensureFounderBootstrap(strand.Id));
|
|
5436
|
+
}
|
|
5437
|
+
catch (error) {
|
|
5438
|
+
// The founding did not happen (e.g. the rebuild refused a pre-split strand and
|
|
5439
|
+
// rolled back, leaving the instance tracked with no runtime): withdraw the flip
|
|
5440
|
+
// so the next attempt re-runs the founding instead of resolving
|
|
5441
|
+
// 'already-founder' over an instance that never founded.
|
|
5442
|
+
this.strandManager.withdrawFounderRequest(strand.Id);
|
|
5443
|
+
throw error;
|
|
5444
|
+
}
|
|
5445
|
+
}
|
|
5446
|
+
log('launchStrand: strand %s already tracked — founder request honored (%s)', strand.Id, outcome);
|
|
5447
|
+
return existing;
|
|
5448
|
+
}
|
|
5092
5449
|
/**
|
|
5093
5450
|
* Resolve a strand's discovery seed — the dialable strand-network multiaddr
|
|
5094
5451
|
* strings for cohort siblings. Membership comes from the control network's
|
|
@@ -5111,37 +5468,17 @@ export class CadreNode {
|
|
|
5111
5468
|
* relay reservation — the gate denial there is fatal, not degraded. A
|
|
5112
5469
|
* party-member relay answers the RPC (its control node admits us as a
|
|
5113
5470
|
* member); a dedicated ops/ relay does not speak the protocol and the
|
|
5114
|
-
* per-peer failure
|
|
5471
|
+
* per-peer failure comes back `unreachable` — harmless, it has no membership gate and
|
|
5115
5472
|
* needs no grant. The relay's direct addr rides along as the dial fallback
|
|
5116
5473
|
* for a relay we are not yet connected to.
|
|
5117
5474
|
*/
|
|
5118
5475
|
async resolveCohortSeed(strandId, delegatePeerId) {
|
|
5119
5476
|
const siblings = await this.resolveSiblingSeed(strandId, delegatePeerId);
|
|
5120
|
-
// Sibling answers FIRST: they were resolved just now, while
|
|
5121
|
-
//
|
|
5477
|
+
// Sibling answers FIRST: they were resolved just now, while the formation-carried
|
|
5478
|
+
// addresses are as old as the formation. `unionAddrs` appends only what the siblings
|
|
5122
5479
|
// did not already name, so a fresher answer is never displaced and each source keeps
|
|
5123
|
-
// its own
|
|
5124
|
-
return unionAddrs(siblings, this.
|
|
5125
|
-
}
|
|
5126
|
-
/**
|
|
5127
|
-
* The strand peer book's contribution to a strand's seed: every live entry's
|
|
5128
|
-
* addresses, freshest peer first, minus this node's own entry when `selfPeerId` is
|
|
5129
|
-
* known (the book swap files one under the strand transport id; a node must never
|
|
5130
|
-
* dial itself). Empty before {@link start} builds the store.
|
|
5131
|
-
*/
|
|
5132
|
-
strandPeerBookAddrs(strandId, selfPeerId) {
|
|
5133
|
-
const store = this.strandPeerBookStore;
|
|
5134
|
-
if (!store) {
|
|
5135
|
-
return [];
|
|
5136
|
-
}
|
|
5137
|
-
const addrs = [];
|
|
5138
|
-
for (const entry of store.entries(strandId)) {
|
|
5139
|
-
if (entry.peerId === selfPeerId) {
|
|
5140
|
-
continue;
|
|
5141
|
-
}
|
|
5142
|
-
addrs.push(...entry.addrs);
|
|
5143
|
-
}
|
|
5144
|
-
return addrs;
|
|
5480
|
+
// its own ordering.
|
|
5481
|
+
return unionAddrs(siblings, this.formationStrandAddrs.get(strandId) ?? []);
|
|
5145
5482
|
}
|
|
5146
5483
|
/**
|
|
5147
5484
|
* The own-party half of {@link resolveCohortSeed}: strand-network addresses resolved
|
|
@@ -5161,10 +5498,13 @@ export class CadreNode {
|
|
|
5161
5498
|
}
|
|
5162
5499
|
}
|
|
5163
5500
|
const bootstrapNodes = targets.length
|
|
5164
|
-
? await
|
|
5501
|
+
? (await this.collectSiblingStrandAddrs(this.controlNode, targets, strandId, delegatePeerId)).addrs
|
|
5165
5502
|
: [];
|
|
5166
5503
|
// Throttle state for the RELAY targets only — refreshDelegateGrants never
|
|
5167
|
-
// looks up a sibling key, so recording one would only be dead weight.
|
|
5504
|
+
// looks up a sibling key, so recording one would only be dead weight. The
|
|
5505
|
+
// siblings' refresh due times are deliberately left alone too: stamping here
|
|
5506
|
+
// would race the refresh pass's pruning while the strand node is still coming
|
|
5507
|
+
// up, and all it would save is one extra RPC per sibling on the first tick.
|
|
5168
5508
|
this.recordDelegateAnnounces(relays.map((r) => r.relayPeerId), strandId);
|
|
5169
5509
|
return bootstrapNodes;
|
|
5170
5510
|
}
|
|
@@ -5246,9 +5586,10 @@ export class CadreNode {
|
|
|
5246
5586
|
* Record the announce timestamps {@link refreshDelegateGrants} throttles on,
|
|
5247
5587
|
* for every relay a delegate-carrying announce pass dialed.
|
|
5248
5588
|
*
|
|
5249
|
-
* Recorded OPTIMISTICALLY at announce time
|
|
5250
|
-
* per
|
|
5251
|
-
*
|
|
5589
|
+
* Recorded OPTIMISTICALLY at announce time, whatever `collectStrandAddrs`
|
|
5590
|
+
* reports per peer: a dedicated `ops/` relay never speaks the strand-addr
|
|
5591
|
+
* protocol, so recording only on success would re-announce to it on every
|
|
5592
|
+
* reconcile tick, one wasted protocol negotiation each. A failed INITIAL announce
|
|
5252
5593
|
* costs the strand supervisor its first attempt only (the relay denies the
|
|
5253
5594
|
* reservation; every re-drive re-announces first through
|
|
5254
5595
|
* {@link announceDelegateToRelay}); a failed REFRESH retries within
|
|
@@ -5284,7 +5625,7 @@ export class CadreNode {
|
|
|
5284
5625
|
running.set(strandId, instance.libp2pNode.peerId.toString());
|
|
5285
5626
|
}
|
|
5286
5627
|
}
|
|
5287
|
-
|
|
5628
|
+
prunePeerStrandKeys(this.delegateAnnounceAt, new Set(running.keys()));
|
|
5288
5629
|
if (running.size === 0) {
|
|
5289
5630
|
return;
|
|
5290
5631
|
}
|
|
@@ -5311,16 +5652,16 @@ export class CadreNode {
|
|
|
5311
5652
|
* again right away.
|
|
5312
5653
|
*
|
|
5313
5654
|
* Against a dedicated ops relay (no strand-addr RPC) the request fails per-peer
|
|
5314
|
-
* and `collectStrandAddrs`
|
|
5655
|
+
* and `collectStrandAddrs` reports it `unreachable`: one wasted protocol negotiation per
|
|
5315
5656
|
* re-drive attempt, bounded by the supervisor's backoff. Never throws on that
|
|
5316
5657
|
* path; a relay addr that names no peer id is logged and skipped (the hook's
|
|
5317
5658
|
* caller runs the drive regardless).
|
|
5318
5659
|
*
|
|
5319
5660
|
* NOTE: against a relay that is DOWN this hook costs up to two strand-addr
|
|
5320
|
-
* timeouts (
|
|
5321
|
-
* starts, so one failed re-drive holds the supervisor
|
|
5322
|
-
* drive's own deadline —
|
|
5323
|
-
* that declared a slower one (`link-budget.ts`).
|
|
5661
|
+
* timeouts (dial by peer id, then by addr; 23 s each at the default declared link round trip)
|
|
5662
|
+
* before the reservation drive even starts, so one failed re-drive holds the supervisor
|
|
5663
|
+
* `driving` for those 46 s plus the drive's own deadline — 64 s at the default, and longer on
|
|
5664
|
+
* a host that declared a slower one (`link-budget.ts`).
|
|
5324
5665
|
* Bounded and harmless while the relay is unreachable anyway; if recovery
|
|
5325
5666
|
* latency after a relay comes back ever matters, skip the announce when the
|
|
5326
5667
|
* control node holds no connection to the relay (the drive's own dial fails
|
|
@@ -5336,7 +5677,7 @@ export class CadreNode {
|
|
|
5336
5677
|
log('announceDelegateToRelay: relay addr %s names no peer id; strand %s not announced', relayAddr, strandId);
|
|
5337
5678
|
return;
|
|
5338
5679
|
}
|
|
5339
|
-
await
|
|
5680
|
+
await this.collectSiblingStrandAddrs(controlNode, [{ peerId: relayPeerId, addrs: [multiaddr(relayAddr)] }], strandId, delegatePeerId);
|
|
5340
5681
|
this.recordDelegateAnnounces([relayPeerId], strandId);
|
|
5341
5682
|
}
|
|
5342
5683
|
/** One strand's share of {@link refreshDelegateGrants}: announce to the relays whose grant is due. */
|
|
@@ -5349,21 +5690,29 @@ export class CadreNode {
|
|
|
5349
5690
|
if (due.length === 0) {
|
|
5350
5691
|
return;
|
|
5351
5692
|
}
|
|
5352
|
-
await
|
|
5693
|
+
await this.collectSiblingStrandAddrs(controlNode, due.map(relayStrandAddrPeer), strandId, delegatePeerId);
|
|
5353
5694
|
this.recordDelegateAnnounces(due.map((relay) => relay.relayPeerId), strandId, now);
|
|
5354
5695
|
}
|
|
5355
5696
|
/**
|
|
5356
|
-
*
|
|
5357
|
-
*
|
|
5358
|
-
* {@link
|
|
5697
|
+
* Keep each running strand's own libp2p address book warm: re-merge the address
|
|
5698
|
+
* records its FRET routing table holds on every pass
|
|
5699
|
+
* ({@link remergeStrandFretRecords}), and re-ask each connected SIBLING for its
|
|
5700
|
+
* strand addresses over the control mesh when that (sibling, strand) is due
|
|
5701
|
+
* ({@link strandAddrAskDueAt}).
|
|
5359
5702
|
*
|
|
5360
|
-
* Without this, a strand's address book
|
|
5703
|
+
* Without this, cadre-core writes a strand's address book exactly once — the
|
|
5361
5704
|
* launch/resume seed — and everything below cadre-core that dials a strand peer
|
|
5362
5705
|
* by bare peer id (Optimystic's cluster and repo clients, FRET ping/announce)
|
|
5363
|
-
* loses its address for any
|
|
5706
|
+
* loses its address for any peer it is not currently connected to: a sibling
|
|
5364
5707
|
* that restarted its strand node or rotated its relay reservation is never
|
|
5365
|
-
* re-resolved, and
|
|
5366
|
-
* one-hour expiry (see `peer-addr-book.ts`).
|
|
5708
|
+
* re-resolved, and every address, another party's included, falls off at the
|
|
5709
|
+
* peerStore's one-hour expiry (see `peer-addr-book.ts`).
|
|
5710
|
+
*
|
|
5711
|
+
* Due times are per (sibling, strand) and set from each sibling's own outcome, so
|
|
5712
|
+
* a sibling that connects late — a phone joining after the party's always-on
|
|
5713
|
+
* machines — is asked on the next tick, and one that could not answer or refused
|
|
5714
|
+
* (its view of the membership may not include us yet) is retried within
|
|
5715
|
+
* {@link STRAND_PEER_ADDR_RETRY_MS} rather than {@link STRAND_PEER_ADDR_REFRESH_MS}.
|
|
5367
5716
|
*
|
|
5368
5717
|
* Distinct from {@link refreshDelegateGrants}, deliberately: that pass covers
|
|
5369
5718
|
* RELAYS on a `DELEGATE_GRANT_TTL_MS / 2` throttle to keep circuit-relay
|
|
@@ -5388,54 +5737,65 @@ export class CadreNode {
|
|
|
5388
5737
|
running.add(strandId);
|
|
5389
5738
|
}
|
|
5390
5739
|
}
|
|
5391
|
-
|
|
5392
|
-
|
|
5393
|
-
// than inherit the stamp its previous incarnation left.
|
|
5394
|
-
for (const strandId of this.strandPeerAddrRefreshAt.keys()) {
|
|
5395
|
-
if (!running.has(strandId)) {
|
|
5396
|
-
this.strandPeerAddrRefreshAt.delete(strandId);
|
|
5397
|
-
}
|
|
5398
|
-
}
|
|
5399
|
-
const refreshMs = this.config.network?.controlCohort?.strandAddrRefreshMs ?? STRAND_PEER_ADDR_REFRESH_MS;
|
|
5400
|
-
const due = [...running].filter((strandId) => now - (this.strandPeerAddrRefreshAt.get(strandId) ?? 0) >= refreshMs);
|
|
5401
|
-
if (due.length === 0) {
|
|
5740
|
+
if (running.size === 0) {
|
|
5741
|
+
this.strandAddrAskDueAt.clear();
|
|
5402
5742
|
return;
|
|
5403
5743
|
}
|
|
5404
|
-
|
|
5405
|
-
// connections its answer is empty whatever the table holds, so decide it from the
|
|
5406
|
-
// connection list instead of paying for the read once a tick. Note this only skips
|
|
5407
|
-
// the RPC: a strand with peer-book entries still refreshes below, because those
|
|
5408
|
-
// never came from a sibling in the first place.
|
|
5409
|
-
//
|
|
5410
|
-
// NOTE: one strand-addr RPC per (due strand × connected sibling) per refresh
|
|
5411
|
-
// interval — each a tiny request/response on an already-open control
|
|
5412
|
-
// connection. If a node ever runs MANY strands at once, batch the RPC to
|
|
5413
|
-
// carry several strand ids per request rather than one fan-out per strand.
|
|
5414
|
-
const targets = this.controlNode.getConnections().length === 0
|
|
5415
|
-
? []
|
|
5416
|
-
: await this.connectedSiblingTargets().catch((error) => {
|
|
5417
|
-
log('refreshStrandPeerAddrs: sibling enumeration failed (skipping pass): %o', error);
|
|
5418
|
-
return [];
|
|
5419
|
-
});
|
|
5744
|
+
const targets = await this.strandAddrRefreshTargets();
|
|
5420
5745
|
// A shutdown landed mid-enumeration.
|
|
5421
5746
|
if (!this._running || !this.controlNode) {
|
|
5422
5747
|
return;
|
|
5423
5748
|
}
|
|
5424
|
-
|
|
5749
|
+
// Drop due times for strands no longer running and siblings no longer connected:
|
|
5750
|
+
// the map must not grow for the node's lifetime, a resumed strand must not
|
|
5751
|
+
// inherit its previous incarnation's due times, and a sibling that reconnects (a
|
|
5752
|
+
// phone restart, a new relay reservation) must be asked on its next connected
|
|
5753
|
+
// tick. An enumeration failure prunes everything, which costs one extra round of
|
|
5754
|
+
// asks, never a missed one.
|
|
5755
|
+
prunePeerStrandKeys(this.strandAddrAskDueAt, running, new Set(targets.map((t) => t.peerId)));
|
|
5756
|
+
await Promise.all([...running].map((strandId) => this.refreshOneStrandPeerAddrs(strandId, targets, now)));
|
|
5757
|
+
}
|
|
5758
|
+
/**
|
|
5759
|
+
* The siblings {@link refreshStrandPeerAddrs} may ask this pass: the connected
|
|
5760
|
+
* cohort, or none when enumeration fails (logged, never thrown).
|
|
5761
|
+
*
|
|
5762
|
+
* `connectedSiblingTargets`' membership read is unbounded, and with zero
|
|
5763
|
+
* connections its answer is empty whatever the table holds, so decide it from the
|
|
5764
|
+
* connection list instead of paying for the read once a tick.
|
|
5765
|
+
*
|
|
5766
|
+
* NOTE: otherwise the read runs on EVERY tick with a running strand, not only when
|
|
5767
|
+
* some (sibling, strand) is due: pruning a departed sibling needs the current
|
|
5768
|
+
* target set, and whether anyone is due cannot be told without it (a connected
|
|
5769
|
+
* non-member never gets a due time to compare). One more `CadrePeer` read per 15 s
|
|
5770
|
+
* tick, on top of the two `runReconcileControlCohort` already makes; if those reads
|
|
5771
|
+
* get costly, share one row-set across the pass (see the NOTE there) rather than
|
|
5772
|
+
* skipping ticks here.
|
|
5773
|
+
*
|
|
5774
|
+
* NOTE: one strand-addr RPC per (running strand × due sibling) — each a tiny
|
|
5775
|
+
* request/response on an already-open control connection. If a node ever runs
|
|
5776
|
+
* MANY strands at once, batch the RPC to carry several strand ids per request
|
|
5777
|
+
* rather than one fan-out per strand.
|
|
5778
|
+
*/
|
|
5779
|
+
async strandAddrRefreshTargets() {
|
|
5780
|
+
if (this.controlNode?.getConnections().length === 0) {
|
|
5781
|
+
return [];
|
|
5782
|
+
}
|
|
5783
|
+
return this.connectedSiblingTargets().catch((error) => {
|
|
5784
|
+
log('refreshStrandPeerAddrs: sibling enumeration failed (asking nobody this pass): %o', error);
|
|
5785
|
+
return [];
|
|
5786
|
+
});
|
|
5425
5787
|
}
|
|
5426
5788
|
/**
|
|
5427
|
-
* One strand's share of {@link refreshStrandPeerAddrs}: RPC the siblings
|
|
5428
|
-
* answers
|
|
5429
|
-
* address
|
|
5430
|
-
* the others their refresh.
|
|
5789
|
+
* One strand's share of {@link refreshStrandPeerAddrs}: RPC the siblings that are
|
|
5790
|
+
* due and merge their answers into the strand's address book, then re-merge the
|
|
5791
|
+
* address records the strand node's FRET table holds. Errors are logged and
|
|
5792
|
+
* swallowed so one strand's failure never costs the others their refresh.
|
|
5431
5793
|
*
|
|
5432
|
-
*
|
|
5433
|
-
*
|
|
5434
|
-
*
|
|
5435
|
-
*
|
|
5436
|
-
*
|
|
5437
|
-
* reconcile tick retries rather than sitting out the whole refresh interval having
|
|
5438
|
-
* done nothing.
|
|
5794
|
+
* The sibling half runs first, so a failure reading the FRET table cannot cost a
|
|
5795
|
+
* sibling answer its merge. The FRET half, and the formation-carried addresses of
|
|
5796
|
+
* the peers it holds no record for, run on EVERY pass, whether or not any sibling is
|
|
5797
|
+
* due or connected — see {@link remergeStrandFretRecords} and
|
|
5798
|
+
* {@link remergeUnrecordedFormationAddrs} for why.
|
|
5439
5799
|
*/
|
|
5440
5800
|
async refreshOneStrandPeerAddrs(strandId, targets, now) {
|
|
5441
5801
|
const controlNode = this.controlNode;
|
|
@@ -5446,42 +5806,124 @@ export class CadreNode {
|
|
|
5446
5806
|
try {
|
|
5447
5807
|
// Inside the try: a strand node torn down mid-pass may throw on any read, and one
|
|
5448
5808
|
// strand's failure must never cost the others their refresh.
|
|
5449
|
-
const
|
|
5450
|
-
|
|
5451
|
-
return;
|
|
5452
|
-
}
|
|
5809
|
+
const strandPeerId = strandNode.peerId.toString();
|
|
5810
|
+
const due = targets.filter((target) => now >= (this.strandAddrAskDueAt.get(peerStrandKey(target.peerId, strandId)) ?? 0));
|
|
5453
5811
|
// The running strand node's own peerId is the delegate to announce — see
|
|
5454
5812
|
// the relationship note on refreshStrandPeerAddrs.
|
|
5455
|
-
const siblingAddrs =
|
|
5813
|
+
const siblingAddrs = due.length === 0
|
|
5456
5814
|
? []
|
|
5457
|
-
: await
|
|
5458
|
-
|
|
5459
|
-
});
|
|
5460
|
-
// Stamped on the pass having happened, not on its answer: the fan-out is
|
|
5461
|
-
// what the throttle exists to bound, and an all-empty round is a normal
|
|
5462
|
-
// steady state for a strand no connected sibling currently runs.
|
|
5463
|
-
this.strandPeerAddrRefreshAt.set(strandId, now);
|
|
5464
|
-
// Re-read the instance after the await — a strand stopped mid-pass (or one
|
|
5465
|
-
// already restarted onto a new node) must never have its store written to.
|
|
5466
|
-
if (!this._running || this.strandManager.getInstance(strandId)?.libp2pNode !== strandNode) {
|
|
5815
|
+
: await this.askSiblingsForStrandAddrs(controlNode, due, strandId, strandPeerId, now);
|
|
5816
|
+
if (!this.isRunningStrandNode(strandId, strandNode)) {
|
|
5467
5817
|
return;
|
|
5468
5818
|
}
|
|
5469
|
-
|
|
5819
|
+
if (siblingAddrs.length > 0) {
|
|
5820
|
+
await this.mergeStrandPeerAddrs(strandNode, siblingAddrs, strandId);
|
|
5821
|
+
}
|
|
5822
|
+
const recorded = await this.remergeStrandFretRecords(strandNode, strandId);
|
|
5823
|
+
await this.remergeUnrecordedFormationAddrs(strandNode, strandId, recorded);
|
|
5470
5824
|
}
|
|
5471
5825
|
catch (error) {
|
|
5472
5826
|
log('refreshStrandPeerAddrs: strand %s refresh failed (continuing): %o', strandId, error);
|
|
5473
5827
|
}
|
|
5474
5828
|
}
|
|
5829
|
+
/**
|
|
5830
|
+
* Is `strandNode` still the node `strandId` runs on? Asked again after every await
|
|
5831
|
+
* in a refresh pass: a strand stopped mid-pass, or one already restarted onto a new
|
|
5832
|
+
* node, must never have its store written to.
|
|
5833
|
+
*/
|
|
5834
|
+
isRunningStrandNode(strandId, strandNode) {
|
|
5835
|
+
return this._running && this.strandManager.getInstance(strandId)?.libp2pNode === strandNode;
|
|
5836
|
+
}
|
|
5837
|
+
/**
|
|
5838
|
+
* Re-merge, into a running strand node's address book, the addresses of every peer
|
|
5839
|
+
* its FRET routing table holds a signed address record for (`strand-fret-addrs.ts`).
|
|
5840
|
+
*
|
|
5841
|
+
* This is what keeps ANOTHER PARTY's strand nodes dialable from a node that stays
|
|
5842
|
+
* up. Nothing re-resolves their addresses — the strand-addr RPC answers own-party
|
|
5843
|
+
* siblings only — and the peerStore hides an address one hour after it was first
|
|
5844
|
+
* observed. FRET keeps each peer's record but hands it to the peerStore only when it
|
|
5845
|
+
* arrives or is imported, so without this pass a connection to another party that
|
|
5846
|
+
* drops after the first hour could not be redialed by either side.
|
|
5847
|
+
*
|
|
5848
|
+
* A record is held while FRET holds the peer's table entry, and at most 14 days after
|
|
5849
|
+
* it was last confirmed, so that is also how long a wrong address can keep being
|
|
5850
|
+
* re-merged. Each record is signed by the peer it names and verified before use, so
|
|
5851
|
+
* the exposure is failed dials to an address that peer itself published.
|
|
5852
|
+
*
|
|
5853
|
+
* NOTE: every pass serialises the whole FRET table (`exportTable()`), verifies one
|
|
5854
|
+
* signature per record and makes one `peerStore.merge` per peer, per running strand
|
|
5855
|
+
* per 15 s tick. Fine at strand table sizes, and an unchanged address set writes
|
|
5856
|
+
* nothing in libp2p's persistent peer store. If many strands or large tables make it
|
|
5857
|
+
* show up, read only the entries whose address stamp is near the one-hour mark.
|
|
5858
|
+
*
|
|
5859
|
+
* @returns the ids of the peers the table holds a usable record for.
|
|
5860
|
+
*/
|
|
5861
|
+
async remergeStrandFretRecords(strandNode, strandId) {
|
|
5862
|
+
const { peers, rejected } = await strandFretPeerAddrs(strandNode);
|
|
5863
|
+
const recorded = new Set(peers.keys());
|
|
5864
|
+
if ((peers.size === 0 && rejected === 0) || !this.isRunningStrandNode(strandId, strandNode)) {
|
|
5865
|
+
return recorded;
|
|
5866
|
+
}
|
|
5867
|
+
const counts = { merged: 0, restamped: 0, skipped: 0, failed: 0 };
|
|
5868
|
+
for (const [peerId, addrs] of peers) {
|
|
5869
|
+
counts[await mergePeerAddrs(strandNode, peerId, addrs)]++;
|
|
5870
|
+
}
|
|
5871
|
+
log('strand %s FRET address records re-merged (peers=%d, merged=%d, restamped=%d, failed=%d, rejected=%d)', strandId, peers.size, counts.merged, counts.restamped, counts.failed, rejected);
|
|
5872
|
+
return recorded;
|
|
5873
|
+
}
|
|
5874
|
+
/**
|
|
5875
|
+
* Re-merge the addresses this strand's formation carried ({@link formationStrandAddrs})
|
|
5876
|
+
* for every peer the strand node's FRET table holds no record for (`recorded`).
|
|
5877
|
+
*
|
|
5878
|
+
* Until the first connection to the responder, the carried addresses are the only
|
|
5879
|
+
* thing naming it, and the launch seed writes them once. FRET's `bootstraps` keep only
|
|
5880
|
+
* the peer id, so if the responder stays unreachable past the peerStore's one-hour
|
|
5881
|
+
* expiry, nothing could dial it when it returns. Once FRET holds the peer's own signed
|
|
5882
|
+
* record, that record supersedes what the formation carried and this stops.
|
|
5883
|
+
*/
|
|
5884
|
+
async remergeUnrecordedFormationAddrs(strandNode, strandId, recorded) {
|
|
5885
|
+
const carried = this.formationStrandAddrs.get(strandId);
|
|
5886
|
+
if (!carried) {
|
|
5887
|
+
return;
|
|
5888
|
+
}
|
|
5889
|
+
const unrecorded = [...groupAddrsByPeerId(carried)]
|
|
5890
|
+
.filter(([peerId]) => !recorded.has(peerId))
|
|
5891
|
+
.flatMap(([, addrs]) => addrs.map((addr) => addr.toString()));
|
|
5892
|
+
if (unrecorded.length === 0 || !this.isRunningStrandNode(strandId, strandNode)) {
|
|
5893
|
+
return;
|
|
5894
|
+
}
|
|
5895
|
+
await this.mergeStrandPeerAddrs(strandNode, unrecorded, strandId);
|
|
5896
|
+
}
|
|
5897
|
+
/**
|
|
5898
|
+
* RPC `due` siblings for `strandId` and set each one's next due time from its own
|
|
5899
|
+
* outcome: an answer, even an empty one, waits the full refresh interval; anything
|
|
5900
|
+
* else retries in {@link STRAND_PEER_ADDR_RETRY_MS} (or the refresh interval, if
|
|
5901
|
+
* configured shorter). Stamped before the caller
|
|
5902
|
+
* re-checks that the strand is still running, so a strand torn down mid-pass still
|
|
5903
|
+
* records who was asked.
|
|
5904
|
+
*/
|
|
5905
|
+
async askSiblingsForStrandAddrs(controlNode, due, strandId, delegatePeerId, now) {
|
|
5906
|
+
const { addrs, outcomes } = await this.collectSiblingStrandAddrs(controlNode, [...due], strandId, delegatePeerId);
|
|
5907
|
+
const refreshMs = this.config.network?.controlCohort?.strandAddrRefreshMs ?? STRAND_PEER_ADDR_REFRESH_MS;
|
|
5908
|
+
// A configured refresh shorter than the retry must not leave a failing sibling
|
|
5909
|
+
// waiting longer than a healthy one.
|
|
5910
|
+
const retryMs = Math.min(refreshMs, STRAND_PEER_ADDR_RETRY_MS);
|
|
5911
|
+
for (const [peerId, outcome] of outcomes) {
|
|
5912
|
+
const waitMs = siblingAnswered(outcome) ? refreshMs : retryMs;
|
|
5913
|
+
this.strandAddrAskDueAt.set(peerStrandKey(peerId, strandId), now + waitMs);
|
|
5914
|
+
}
|
|
5915
|
+
return addrs;
|
|
5916
|
+
}
|
|
5475
5917
|
/**
|
|
5476
5918
|
* Merge strand-network addresses into ONE strand node's libp2p address book,
|
|
5477
5919
|
* attributed per peer. Best-effort throughout: an address-book write must never
|
|
5478
5920
|
* fail a launch, a resume, or a reconcile pass.
|
|
5479
5921
|
*
|
|
5480
|
-
* `addrs` is
|
|
5481
|
-
* {@link groupAddrsByPeerId} attributes each
|
|
5482
|
-
* peerId in its final `/p2p/` component — never
|
|
5483
|
-
* which names a different libp2p node entirely.
|
|
5484
|
-
* dropped there and counted here, once per pass.
|
|
5922
|
+
* `addrs` is a peer-agnostic list — the union the strand-addr RPC returns, or the
|
|
5923
|
+
* addresses a formation carried — so {@link groupAddrsByPeerId} attributes each
|
|
5924
|
+
* entry to the **strand transport** peerId in its final `/p2p/` component — never
|
|
5925
|
+
* the sibling's control peerId, which names a different libp2p node entirely.
|
|
5926
|
+
* Entries that name no peer are dropped there and counted here, once per pass.
|
|
5485
5927
|
*
|
|
5486
5928
|
* NOTE: a member could answer with arbitrary multiaddrs bound to arbitrary peer
|
|
5487
5929
|
* ids and poison this address book. That is the same exposure the launch-time
|
|
@@ -5491,14 +5933,13 @@ export class CadreNode {
|
|
|
5491
5933
|
* peerStore's one-hour expiry. No new gating here — cross-party strand trust is
|
|
5492
5934
|
* `backlog/strand-network-nat-relay-reachability`.
|
|
5493
5935
|
*
|
|
5494
|
-
* NOTE: one input
|
|
5495
|
-
* ({@link
|
|
5496
|
-
*
|
|
5497
|
-
*
|
|
5498
|
-
*
|
|
5499
|
-
*
|
|
5500
|
-
*
|
|
5501
|
-
* exposure is a handful of failed dials, not a trust hole.
|
|
5936
|
+
* NOTE: one input outlives that hour — a formation's carried addresses
|
|
5937
|
+
* ({@link formationStrandAddrs}) are in every launch and resume seed for as long as
|
|
5938
|
+
* this process remembers the formation, and re-merged on every refresh pass until the
|
|
5939
|
+
* strand node's FRET table holds the peer's own record, so a junk address from a
|
|
5940
|
+
* responder that is never met keeps being re-merged. Bounded by `sanitizeStrandAddrs`,
|
|
5941
|
+
* replaced by a re-formation, dropped on `unpublishStrand` / `forgetJoinedStrand`, and
|
|
5942
|
+
* still authority-free — so the exposure is a handful of failed dials, not a trust hole.
|
|
5502
5943
|
*/
|
|
5503
5944
|
async mergeStrandPeerAddrs(strandNode, addrs, strandId) {
|
|
5504
5945
|
const counts = { merged: 0, restamped: 0, skipped: 0, failed: 0 };
|
|
@@ -5547,7 +5988,7 @@ export class CadreNode {
|
|
|
5547
5988
|
// is normally a no-op; it is the guard for a bare listen addr and for a relay hop
|
|
5548
5989
|
// whose trailing `/p2p/` names the RELAY. An addr terminating in a DIFFERENT id is
|
|
5549
5990
|
// dropped rather than announced — it does not reach this node, and announcing it
|
|
5550
|
-
// would file our address under someone else's id in the receiver's book.
|
|
5991
|
+
// would file our address under someone else's id in the receiver's address book.
|
|
5551
5992
|
//
|
|
5552
5993
|
// Re-parsed through this package's own `multiaddr` rather than passed straight in:
|
|
5553
5994
|
// libp2p hands back its nested `@multiformats/multiaddr` copy, a structurally
|
|
@@ -5565,27 +6006,36 @@ export class CadreNode {
|
|
|
5565
6006
|
return orderSignalingFirst(addrs);
|
|
5566
6007
|
}
|
|
5567
6008
|
/**
|
|
5568
|
-
* Leave a strand joined from another party
|
|
5569
|
-
*
|
|
5570
|
-
*
|
|
5571
|
-
*
|
|
6009
|
+
* Leave a strand joined from another party, for the whole party: remove its party-wide
|
|
6010
|
+
* `JoinedStrand` row (owner-signed, with a `Revocation` tombstone), then this machine's
|
|
6011
|
+
* unpublished record, its formation-carried addresses, its saved network state and any
|
|
6012
|
+
* session-kept entry, then
|
|
6013
|
+
* {@link stopStrand} it here. Every other machine's watcher then sees the row gone and
|
|
6014
|
+
* detaches the strand, a storage replica included; a machine offline at the time reaps the
|
|
6015
|
+
* row once the tombstone reaches it. The joiner's counterpart of {@link unpublishStrand},
|
|
6016
|
+
* which only works on a row the party's own `Strand` table holds. For one of those, or for
|
|
6017
|
+
* a join still local-only, there is no party-wide row and this is a local forget plus
|
|
6018
|
+
* {@link stopStrand}.
|
|
5572
6019
|
*
|
|
5573
|
-
*
|
|
5574
|
-
*
|
|
6020
|
+
* The other members of the strand are not told, and this party's membership row in the
|
|
6021
|
+
* strand stays, so a later re-formation reuses the same identity.
|
|
6022
|
+
*
|
|
6023
|
+
* @throws when a party-wide row exists and this machine cannot sign its removal (not an
|
|
6024
|
+
* enrolled owner) — {@link stopStrand} stops the strand on this machine only — or when
|
|
6025
|
+
* the party-wide table cannot be read.
|
|
5575
6026
|
*/
|
|
5576
6027
|
async forgetJoinedStrand(strandId) {
|
|
5577
6028
|
if (!this._running) {
|
|
5578
6029
|
throw new Error('CadreNode not running');
|
|
5579
6030
|
}
|
|
5580
|
-
await this.
|
|
6031
|
+
await this.joinedStrands.leave(strandId);
|
|
5581
6032
|
this.forgetStrandPeers(strandId, 'forgetJoinedStrand');
|
|
5582
6033
|
await this.stopStrand(strandId);
|
|
5583
6034
|
}
|
|
5584
6035
|
/**
|
|
5585
|
-
* Record `row` as a join from another party when this party's control database
|
|
5586
|
-
*
|
|
5587
|
-
*
|
|
5588
|
-
* every start), and keeps the first `joinedAt`.
|
|
6036
|
+
* Record `row` as a join from another party when this party's control database names it
|
|
6037
|
+
* neither as its own strand nor as a party-wide join — the one case nothing else would
|
|
6038
|
+
* re-offer after a restart (`JoinedStrandSession.rememberForeign`).
|
|
5589
6039
|
*
|
|
5590
6040
|
* Best-effort, for {@link addStrand}: a failure costs the strand its re-offer after the
|
|
5591
6041
|
* next restart, not this attach, so it is reported and the attach goes on.
|
|
@@ -5599,24 +6049,11 @@ export class CadreNode {
|
|
|
5599
6049
|
* tombstone here before recording.
|
|
5600
6050
|
*/
|
|
5601
6051
|
async rememberForeignStrand(row) {
|
|
5602
|
-
|
|
5603
|
-
if (!store || !this.controlDatabase) {
|
|
6052
|
+
if (!this.joinedStrands || !this.controlDatabase) {
|
|
5604
6053
|
return;
|
|
5605
6054
|
}
|
|
5606
6055
|
try {
|
|
5607
|
-
|
|
5608
|
-
return;
|
|
5609
|
-
}
|
|
5610
|
-
const existing = (await store.list()).find((record) => record.Id === row.Id);
|
|
5611
|
-
if (existing?.Type === row.Type && existing.MemberPrivateKey === row.MemberPrivateKey) {
|
|
5612
|
-
return;
|
|
5613
|
-
}
|
|
5614
|
-
await store.record({
|
|
5615
|
-
Id: row.Id,
|
|
5616
|
-
Type: row.Type,
|
|
5617
|
-
MemberPrivateKey: row.MemberPrivateKey,
|
|
5618
|
-
joinedAt: existing?.joinedAt ?? Date.now()
|
|
5619
|
-
});
|
|
6056
|
+
await this.joinedStrands.rememberForeign(row);
|
|
5620
6057
|
}
|
|
5621
6058
|
catch (error) {
|
|
5622
6059
|
console.warn(`addStrand(${row.Id}): could not remember this strand as joined from another party, ` +
|
|
@@ -5624,66 +6061,47 @@ export class CadreNode {
|
|
|
5624
6061
|
}
|
|
5625
6062
|
}
|
|
5626
6063
|
/**
|
|
5627
|
-
* Self-revocation arm of the joined-strand
|
|
5628
|
-
* not re-attach it on every launch. The strand keeps running this
|
|
5629
|
-
* `strand:revoked` contract is that nothing is torn down for the app —
|
|
5630
|
-
*
|
|
5631
|
-
*
|
|
5632
|
-
*
|
|
6064
|
+
* Self-revocation arm of the joined-strand records: a party removed from a strand must
|
|
6065
|
+
* not re-attach it on every launch of every machine. The strand keeps running this
|
|
6066
|
+
* session here — the `strand:revoked` contract is that nothing is torn down for the app —
|
|
6067
|
+
* and its party-wide row is queued for removal by the next connected owner reconcile pass
|
|
6068
|
+
* (`JoinedStrandSession.forgetAfterThisSession`). A no-op for this party's own strands,
|
|
6069
|
+
* which have neither record; the formation-carried addresses and the saved network
|
|
6070
|
+
* state are forgotten either way, since a removed party must not keep dialing the
|
|
6071
|
+
* strand's peers.
|
|
6072
|
+
*
|
|
6073
|
+
* NOTE: accepted tradeoff — a sibling machine that has not raised `strand:revoked` itself
|
|
6074
|
+
* detaches the strand (`strand:stopped`) when the party-wide removal reaches it, instead
|
|
6075
|
+
* of keeping it for the session, which weakens "nothing is stopped on the removed
|
|
6076
|
+
* machine's behalf" for siblings. Weighed against a party-wide row that brings a revoked
|
|
6077
|
+
* strand back on every start of every machine. Revisit if apps need the "you were
|
|
6078
|
+
* removed" screen on every device: the sibling could re-check its own revocation state
|
|
6079
|
+
* (`refreshRevocationEnforcement`) before detaching a vanished joined row.
|
|
5633
6080
|
*
|
|
5634
6081
|
* NOTE: a manager can re-admit a removed party directly (`addMemberByManager`), and the
|
|
5635
|
-
* membership loop then finishes the join on its own — but the
|
|
5636
|
-
* the next start does not re-attach the strand. Re-forming records it again. If direct
|
|
6082
|
+
* membership loop then finishes the join on its own — but the records are gone by then,
|
|
6083
|
+
* so the next start does not re-attach the strand. Re-forming records it again. If direct
|
|
5637
6084
|
* re-admission becomes a routine flow, re-record the join when the revoked-peer gate
|
|
5638
6085
|
* clears for this node.
|
|
5639
6086
|
*/
|
|
5640
6087
|
forgetRevokedJoin(strandId) {
|
|
5641
|
-
void this.
|
|
6088
|
+
void this.joinedStrands?.forgetAfterThisSession(strandId).catch((error) => {
|
|
5642
6089
|
log('forgetting revoked joined strand %s failed; it will be re-offered on the next start: %o', strandId, error);
|
|
5643
6090
|
});
|
|
5644
6091
|
// The strand's peers go with the join: a removed party must not keep dialing them.
|
|
5645
6092
|
this.forgetStrandPeers(strandId, 'self-revocation');
|
|
5646
6093
|
}
|
|
5647
6094
|
/**
|
|
5648
|
-
* Forget a strand's
|
|
5649
|
-
*
|
|
5650
|
-
*
|
|
6095
|
+
* Forget what this node holds about a strand's peers: the addresses its formation
|
|
6096
|
+
* carried (see {@link formationStrandAddrs}) and its saved network state (see
|
|
6097
|
+
* {@link strandNetworkStateStore}). The store write is fire-and-log — the removal is
|
|
6098
|
+
* visible synchronously by the store's contract, and a failed persist only leaves the
|
|
6099
|
+
* entry on disk for the next start.
|
|
5651
6100
|
*/
|
|
5652
6101
|
forgetStrandPeers(strandId, caller) {
|
|
5653
|
-
|
|
5654
|
-
|
|
5655
|
-
|
|
5656
|
-
}
|
|
5657
|
-
/**
|
|
5658
|
-
* The strand peer book's observation writer (`StartStrandConfig.onStrandPeerIdentified`):
|
|
5659
|
-
* a strand node identified a strand peer at `observation.addrs`, so remember it as an
|
|
5660
|
-
* unsigned entry seen just now. The observer never reports self, and the store binds
|
|
5661
|
-
* every address to the peer id before filing it.
|
|
5662
|
-
*/
|
|
5663
|
-
observeStrandPeer(strandId, observation) {
|
|
5664
|
-
this.rememberStrandPeer(strandId, {
|
|
5665
|
-
peerId: observation.peerId,
|
|
5666
|
-
addrs: observation.addrs,
|
|
5667
|
-
issuedAt: 0,
|
|
5668
|
-
lastSeenAt: Date.now()
|
|
5669
|
-
}, 'observeStrandPeer');
|
|
5670
|
-
}
|
|
5671
|
-
/**
|
|
5672
|
-
* Merge one entry into the peer book. Fire-and-log: the entry is visible
|
|
5673
|
-
* synchronously by the store's contract, and the promise tracks durability only. A
|
|
5674
|
-
* persist failure costs restart survival, never this session's seed — the same trade
|
|
5675
|
-
* {@link retainDialTarget} makes.
|
|
5676
|
-
*/
|
|
5677
|
-
rememberStrandPeer(strandId, entry, caller) {
|
|
5678
|
-
const store = this.strandPeerBookStore;
|
|
5679
|
-
if (!store) {
|
|
5680
|
-
// Unreachable in production: start() builds the store before any strand can
|
|
5681
|
-
// launch or form. Logged rather than thrown, for the same reason as retainDialTarget.
|
|
5682
|
-
log('%s: no strand peer book yet — not remembering %s for strand %s', caller, entry.peerId, strandId);
|
|
5683
|
-
return;
|
|
5684
|
-
}
|
|
5685
|
-
void store.merge(strandId, entry).catch((error) => {
|
|
5686
|
-
log('%s: persisting strand %s peer %s failed (continuing): %o', caller, strandId, entry.peerId, error);
|
|
6102
|
+
this.formationStrandAddrs.delete(strandId);
|
|
6103
|
+
void this.strandNetworkStateStore?.forget(strandId).catch((error) => {
|
|
6104
|
+
log('%s: forgetting strand %s network state failed (continuing): %o', caller, strandId, error);
|
|
5687
6105
|
});
|
|
5688
6106
|
}
|
|
5689
6107
|
/**
|
|
@@ -5774,10 +6192,19 @@ export class CadreNode {
|
|
|
5774
6192
|
void this.pushFanoutService?.notify(strandId, reason);
|
|
5775
6193
|
}
|
|
5776
6194
|
/**
|
|
5777
|
-
* Force wake a hibernating strand
|
|
6195
|
+
* Force wake a hibernating strand. A requested wake is activity, recorded before the
|
|
6196
|
+
* wake starts: a check-in holding the strand — or rebuilding it, a rebuild this wake then
|
|
6197
|
+
* joins — leaves it up instead of re-quiescing it at the end of its window. Once awake the
|
|
6198
|
+
* strand's idle countdown runs, so it hibernates again if nothing uses it.
|
|
5778
6199
|
*/
|
|
5779
6200
|
async wakeStrand(strandId) {
|
|
5780
|
-
|
|
6201
|
+
const instance = this.strandManager.getInstance(strandId);
|
|
6202
|
+
if (!instance) {
|
|
6203
|
+
log('wakeStrand: strand %s not found; nothing to wake', strandId);
|
|
6204
|
+
return;
|
|
6205
|
+
}
|
|
6206
|
+
instance.lastActivity = new Date();
|
|
6207
|
+
await this.hibernationManager.wakeStrand(instance);
|
|
5781
6208
|
}
|
|
5782
6209
|
// ============================================================================
|
|
5783
6210
|
// Mobile background lifecycle primitives
|
|
@@ -5793,7 +6220,7 @@ export class CadreNode {
|
|
|
5793
6220
|
* Routes through {@link HibernationManager.forceHibernate}, which cancels the
|
|
5794
6221
|
* strand's pending idle/hibernate (and check-in) timers — so a stale timer
|
|
5795
6222
|
* can't re-fire on or resurrect the strand — then runs the same `onHibernate`
|
|
5796
|
-
* path as the timer (`quiesceStrand
|
|
6223
|
+
* path as the timer (`quiesceStrand`, which marks it `hibernating`, then
|
|
5797
6224
|
* `strand:hibernating`). Unlike the timer path it does NOT re-arm check-ins:
|
|
5798
6225
|
* the strand stays down until the caller drives a wake (e.g. {@link serviceWake}).
|
|
5799
6226
|
*/
|
|
@@ -5854,9 +6281,14 @@ export class CadreNode {
|
|
|
5854
6281
|
* strand share one in-flight operation ({@link serviceWakePromises}), and the
|
|
5855
6282
|
* underlying resume coalesces with a racing push-wake via
|
|
5856
6283
|
* {@link HibernationManager}'s wake coalescing — one runtime build, one window,
|
|
5857
|
-
* one re-hibernate decision.
|
|
5858
|
-
* the
|
|
5859
|
-
*
|
|
6284
|
+
* one re-hibernate decision. A wake or activity from elsewhere that lands during
|
|
6285
|
+
* the resume or the window leaves the strand up; this call's own wake does not.
|
|
6286
|
+
* Afterwards a strand left up has its idle countdown running, and a re-hibernated one
|
|
6287
|
+
* gets back the check-in chain this call interrupted — it never gains one it lacked,
|
|
6288
|
+
* so a strand the mobile runner force-hibernated stays down until the next push.
|
|
6289
|
+
* Returns `{ serviced: false }` (never throws) when
|
|
6290
|
+
* the node is not running or the strand is unknown, and surfaces a wake or
|
|
6291
|
+
* window failure as `{ serviced: true, hadActivity: false }`.
|
|
5860
6292
|
*
|
|
5861
6293
|
* @param strandId - the strand a push said has pending activity.
|
|
5862
6294
|
* @param opts.windowMs - override the live-window duration (defaults to the
|
|
@@ -5890,30 +6322,58 @@ export class CadreNode {
|
|
|
5890
6322
|
log('serviceWake: strand %s unknown to this node; not serviced', strandId);
|
|
5891
6323
|
return { strandId, serviced: false, hadActivity: false };
|
|
5892
6324
|
}
|
|
5893
|
-
// Already live (active or idle — both retain their runtime)
|
|
5894
|
-
//
|
|
5895
|
-
// may be actively using, and do NOT
|
|
5896
|
-
|
|
6325
|
+
// Already live (active or idle — both retain their runtime) or coming up (a launch,
|
|
6326
|
+
// or a check-in with its own window): servicing is a no-op success. Do NOT run a
|
|
6327
|
+
// window that would re-hibernate a strand the app may be actively using, and do NOT
|
|
6328
|
+
// rebuild a second runtime. `'starting'` counts before any handle is attached. A strand
|
|
6329
|
+
// being quiesced does not count: the wake below rebuilds it after the quiesce.
|
|
6330
|
+
if (!this.strandManager.isQuiescing(strandId)
|
|
6331
|
+
&& (instance.status === 'starting' || instance.libp2pNode || instance.database)) {
|
|
5897
6332
|
log('serviceWake: strand %s already live; no-op success', strandId);
|
|
5898
6333
|
return { strandId, serviced: true, hadActivity: true };
|
|
5899
6334
|
}
|
|
6335
|
+
// Before the resume, as the check-in takes it (see handleStrandCheckIn).
|
|
6336
|
+
// NOTE: a `wakeStrand` already resuming when this runs stamped its activity before this
|
|
6337
|
+
// mark, so the probe joins it without counting it and the window can re-hibernate a strand
|
|
6338
|
+
// that wake asked to keep up; if foreground wakes and serviceWake overlap in practice, mark
|
|
6339
|
+
// before the joined wake's stamp instead.
|
|
6340
|
+
const activityMark = instance.lastActivity;
|
|
6341
|
+
const windowMs = opts?.windowMs ?? this.config.hibernation?.checkInWindowMs ?? DEFAULT_CHECKIN_WINDOW_MS;
|
|
6342
|
+
try {
|
|
6343
|
+
return await this.wakeAndHoldForService(instance, activityMark, windowMs);
|
|
6344
|
+
}
|
|
6345
|
+
finally {
|
|
6346
|
+
// After the window (or a failure) decided the state: the idle countdown if the strand
|
|
6347
|
+
// stayed up, else the check-in chain this probe interrupted, if any.
|
|
6348
|
+
this.hibernationManager.endProbe(instance);
|
|
6349
|
+
}
|
|
6350
|
+
}
|
|
6351
|
+
/**
|
|
6352
|
+
* The wake and window of {@link runServiceWake}. A failure in either — the network
|
|
6353
|
+
* unreachable inside a Doze grant, say — must not throw out of a background task, so it
|
|
6354
|
+
* reports no activity. Each stage cleans up only what it owns.
|
|
6355
|
+
*/
|
|
6356
|
+
async wakeAndHoldForService(instance, activityMark, windowMs) {
|
|
6357
|
+
const { strandId } = instance;
|
|
6358
|
+
try {
|
|
6359
|
+
// Coalesced resume: HibernationManager.beginWake, so a racing push-wake shares this
|
|
6360
|
+
// single runtime build. Not through `wakeStrand`, which records the wake as activity:
|
|
6361
|
+
// this probe's own wake must not count as a reason to stay up.
|
|
6362
|
+
await this.hibernationManager.probeWake(instance);
|
|
6363
|
+
}
|
|
6364
|
+
catch (error) {
|
|
6365
|
+
// handleStrandWake has already re-hibernated a failed rebuild, and left alone a strand
|
|
6366
|
+
// it failed before building, which may be a check-in's.
|
|
6367
|
+
log('serviceWake: wake of strand %s failed: %o', strandId, error);
|
|
6368
|
+
return { strandId, serviced: true, hadActivity: false };
|
|
6369
|
+
}
|
|
5900
6370
|
try {
|
|
5901
|
-
|
|
5902
|
-
// so a racing push-wake shares this single runtime build.
|
|
5903
|
-
await this.wakeStrand(strandId);
|
|
5904
|
-
const windowMs = opts?.windowMs ?? this.config.hibernation?.checkInWindowMs ?? DEFAULT_CHECKIN_WINDOW_MS;
|
|
5905
|
-
const hadActivity = await this.runWakeWindow(instance, windowMs);
|
|
6371
|
+
const hadActivity = await this.runWakeWindow(instance, activityMark, windowMs);
|
|
5906
6372
|
return { strandId, serviced: true, hadActivity };
|
|
5907
6373
|
}
|
|
5908
6374
|
catch (error) {
|
|
5909
|
-
// Resume failing mid-window (network unreachable inside a Doze grant, etc.)
|
|
5910
|
-
// must not throw out of a background task: re-hibernate and report no
|
|
5911
|
-
// activity, mirroring handleStrandCheckIn's re-hibernate-on-error.
|
|
5912
6375
|
log('serviceWake: strand %s failed during wake window; re-hibernating: %o', strandId, error);
|
|
5913
|
-
await this.
|
|
5914
|
-
log('serviceWake cleanup quiesce for strand %s failed: %o', strandId, cleanupErr);
|
|
5915
|
-
});
|
|
5916
|
-
instance.status = 'hibernating';
|
|
6376
|
+
await this.rehibernateAfterFailedResume(instance, 'serviceWake');
|
|
5917
6377
|
return { strandId, serviced: true, hadActivity: false };
|
|
5918
6378
|
}
|
|
5919
6379
|
}
|
|
@@ -5946,8 +6406,8 @@ export class CadreNode {
|
|
|
5946
6406
|
*
|
|
5947
6407
|
* BUDGET: the supervisor's first attempt is what `start()` waits on — deliberately the drive's
|
|
5948
6408
|
* ordinary deadline rather than a boot-specific one. That deadline is now COUNTED, four link
|
|
5949
|
-
* round trips at the declared `network.linkRoundTripMs`
|
|
5950
|
-
* fixed 10 s): a healthy dial-plus-reserve is sub-second even over a WAN, so this is slack for
|
|
6409
|
+
* round trips at the declared `network.linkRoundTripMs` plus the relay's two admission
|
|
6410
|
+
* decisions (18 s at its default, where it was a fixed 10 s): a healthy dial-plus-reserve is sub-second even over a WAN, so this is slack for
|
|
5951
6411
|
* a slow link, while going much longer would make a dead relay indistinguishable from a hung
|
|
5952
6412
|
* start and much shorter would fail nodes on links that were merely slow. A host that declares
|
|
5953
6413
|
* a slower link lengthens it without touching this path — `link-budget.ts`. The retries carry
|
|
@@ -6086,7 +6546,7 @@ export class CadreNode {
|
|
|
6086
6546
|
*
|
|
6087
6547
|
* @param ownerPrivateKey - The owner's private key (base64url encoded)
|
|
6088
6548
|
*/
|
|
6089
|
-
initializeSeedBootstrap(ownerPrivateKey) {
|
|
6549
|
+
async initializeSeedBootstrap(ownerPrivateKey) {
|
|
6090
6550
|
if (!this.controlNode || !this.controlDatabase) {
|
|
6091
6551
|
throw new Error('CadreNode must be started before initializing seed bootstrap');
|
|
6092
6552
|
}
|
|
@@ -6110,18 +6570,16 @@ export class CadreNode {
|
|
|
6110
6570
|
.trust(ed25519PublicKeyFromPrivate(ownerPrivateKey), 'genesis')
|
|
6111
6571
|
.catch((error) => log('Trusted-owner genesis anchor persist failed: %o', error));
|
|
6112
6572
|
}
|
|
6113
|
-
this.
|
|
6573
|
+
await this.installSeedBootstrapService(new SeedBootstrapService({
|
|
6114
6574
|
partyId: this.config.controlNetwork.partyId,
|
|
6115
6575
|
ownerPrivateKey,
|
|
6116
6576
|
inviteAddressResolver: () => this.resolveInviteAddresses(),
|
|
6117
|
-
|
|
6577
|
+
...this.seedServiceBudgets(),
|
|
6118
6578
|
trustPolicy: this.config.seedTrustPolicy,
|
|
6119
6579
|
// Seed trust anchors on the node-local store (seeded just above with this
|
|
6120
6580
|
// node's own genesis key), never on the replicated OwnerKey table.
|
|
6121
6581
|
...(this.trustedOwnerStore ? { trustedOwners: this.trustedOwnerStore } : {}),
|
|
6122
|
-
});
|
|
6123
|
-
this.seedBootstrapService.setEventCallbacks(this.seedEventCallbacks());
|
|
6124
|
-
this.seedBootstrapService.initialize(this.controlNode, this.controlDatabase);
|
|
6582
|
+
}), this.controlNode, this.controlDatabase);
|
|
6125
6583
|
log('Seed bootstrap service initialized');
|
|
6126
6584
|
}
|
|
6127
6585
|
/**
|
|
@@ -6318,7 +6776,8 @@ export class CadreNode {
|
|
|
6318
6776
|
* is reachable through its circuit-relay address), dials `WAKE_PROTOCOL`, sends
|
|
6319
6777
|
* the {@link WakeRequest}, and returns the peer's {@link WakeAck}. The receiver
|
|
6320
6778
|
* gates the request on cadre membership and only resumes a strand it already
|
|
6321
|
-
* participates in.
|
|
6779
|
+
* participates in; it acks once it has decided, before the strand is up. The
|
|
6780
|
+
* dial deadlines derive from this node's `network.linkRoundTripMs`.
|
|
6322
6781
|
*
|
|
6323
6782
|
* @param targetPeerId - The hibernating cadre peer to wake.
|
|
6324
6783
|
* @param strandId - The strand the caller knows has pending activity.
|
|
@@ -6334,14 +6793,14 @@ export class CadreNode {
|
|
|
6334
6793
|
throw new Error(`No dialable control-network address for peer ${targetPeerId}`);
|
|
6335
6794
|
}
|
|
6336
6795
|
const request = { strandId, reason };
|
|
6337
|
-
return await dialWake(this.controlNode, addrs, request);
|
|
6796
|
+
return await dialWake(this.controlNode, addrs, request, { linkRoundTripMs: this.config.network?.linkRoundTripMs });
|
|
6338
6797
|
}
|
|
6339
6798
|
/**
|
|
6340
6799
|
* Enable the seed listener for receiving seeds via the /sereus/seed/1.0.0 protocol.
|
|
6341
6800
|
* This is for drone nodes that need to receive seeds without being an owner.
|
|
6342
6801
|
* Does not require an owner key.
|
|
6343
6802
|
*/
|
|
6344
|
-
enableSeedListener() {
|
|
6803
|
+
async enableSeedListener() {
|
|
6345
6804
|
if (!this.controlNode || !this.controlDatabase) {
|
|
6346
6805
|
throw new Error('CadreNode must be started before enabling seed listener');
|
|
6347
6806
|
}
|
|
@@ -6350,19 +6809,17 @@ export class CadreNode {
|
|
|
6350
6809
|
log('Seed bootstrap service already initialized');
|
|
6351
6810
|
return;
|
|
6352
6811
|
}
|
|
6353
|
-
this.
|
|
6812
|
+
await this.installSeedBootstrapService(new SeedBootstrapService({
|
|
6354
6813
|
partyId: this.config.controlNetwork.partyId,
|
|
6355
6814
|
// No owner key - this node only receives seeds
|
|
6356
6815
|
inviteAddressResolver: () => this.resolveInviteAddresses(),
|
|
6357
|
-
|
|
6816
|
+
...this.seedServiceBudgets(),
|
|
6358
6817
|
trustPolicy: this.config.seedTrustPolicy,
|
|
6359
6818
|
// A listener-only node accepts a wire-delivered seed solely against this
|
|
6360
6819
|
// anchor (there is no per-call override on the inbound handler): with no
|
|
6361
6820
|
// genesis/invite/operator pin it authorizes nobody, which is the point.
|
|
6362
6821
|
...(this.trustedOwnerStore ? { trustedOwners: this.trustedOwnerStore } : {}),
|
|
6363
|
-
});
|
|
6364
|
-
this.seedBootstrapService.setEventCallbacks(this.seedEventCallbacks());
|
|
6365
|
-
this.seedBootstrapService.initialize(this.controlNode, this.controlDatabase);
|
|
6822
|
+
}), this.controlNode, this.controlDatabase);
|
|
6366
6823
|
log('Seed listener enabled');
|
|
6367
6824
|
}
|
|
6368
6825
|
/**
|
|
@@ -6371,6 +6828,30 @@ export class CadreNode {
|
|
|
6371
6828
|
getSeedBootstrapService() {
|
|
6372
6829
|
return this.seedBootstrapService;
|
|
6373
6830
|
}
|
|
6831
|
+
/**
|
|
6832
|
+
* Make `service` this node's seed service and register its inbound seed handler.
|
|
6833
|
+
* The field is set before the registration is awaited, so a concurrent
|
|
6834
|
+
* {@link enableSeedListener} finds it and does not register a second handler; a
|
|
6835
|
+
* failed registration puts the previous service back and rethrows.
|
|
6836
|
+
*
|
|
6837
|
+
* NOTE: libp2p's registrar stores the handler before its peer-store merge, so a
|
|
6838
|
+
* failed merge leaves SEED_PROTOCOL registered with no service owning it and a retry
|
|
6839
|
+
* here rejects as a duplicate; if peer-store writes can fail in practice, unhandle on
|
|
6840
|
+
* a non-duplicate failure (never on a duplicate — that handler belongs to someone else).
|
|
6841
|
+
*/
|
|
6842
|
+
async installSeedBootstrapService(service, controlNode, controlDatabase) {
|
|
6843
|
+
service.setEventCallbacks(this.seedEventCallbacks());
|
|
6844
|
+
const previous = this.seedBootstrapService;
|
|
6845
|
+
this.seedBootstrapService = service;
|
|
6846
|
+
try {
|
|
6847
|
+
await service.initialize(controlNode, controlDatabase);
|
|
6848
|
+
}
|
|
6849
|
+
catch (error) {
|
|
6850
|
+
if (this.seedBootstrapService === service)
|
|
6851
|
+
this.seedBootstrapService = previous;
|
|
6852
|
+
throw error;
|
|
6853
|
+
}
|
|
6854
|
+
}
|
|
6374
6855
|
/**
|
|
6375
6856
|
* The event callbacks every {@link SeedBootstrapService} this node owns is
|
|
6376
6857
|
* wired with — shared by the owner-capable service ({@link initializeSeedBootstrap})
|
|
@@ -6585,7 +7066,7 @@ export class CadreNode {
|
|
|
6585
7066
|
const tempService = new SeedBootstrapService({
|
|
6586
7067
|
partyId: seed.partyId,
|
|
6587
7068
|
trustPolicy: this.config.seedTrustPolicy,
|
|
6588
|
-
|
|
7069
|
+
...this.seedServiceBudgets(),
|
|
6589
7070
|
// The anchor is node-scoped, not service-scoped: a throwaway service
|
|
6590
7071
|
// must consult (and persist an accepted signer into) the SAME store the
|
|
6591
7072
|
// persistent one would, or a cold-start enrollment via this path would
|
|
@@ -6593,7 +7074,7 @@ export class CadreNode {
|
|
|
6593
7074
|
...(this.trustedOwnerStore ? { trustedOwners: this.trustedOwnerStore } : {}),
|
|
6594
7075
|
});
|
|
6595
7076
|
if (this.controlNode && this.controlDatabase) {
|
|
6596
|
-
tempService.initialize(this.controlNode, this.controlDatabase, { registerHandler: false });
|
|
7077
|
+
await tempService.initialize(this.controlNode, this.controlDatabase, { registerHandler: false });
|
|
6597
7078
|
}
|
|
6598
7079
|
const tempResult = await tempService.applySeed(seed, options);
|
|
6599
7080
|
this.noteAppliedSeed(tempResult, seed);
|
|
@@ -6765,10 +7246,10 @@ export class CadreNode {
|
|
|
6765
7246
|
const tempService = new SeedBootstrapService({
|
|
6766
7247
|
partyId: invite.partyId,
|
|
6767
7248
|
trustPolicy: this.config.seedTrustPolicy,
|
|
6768
|
-
|
|
7249
|
+
...this.seedServiceBudgets(),
|
|
6769
7250
|
});
|
|
6770
7251
|
if (this.controlNode && this.controlDatabase) {
|
|
6771
|
-
tempService.initialize(this.controlNode, this.controlDatabase, { registerHandler: false });
|
|
7252
|
+
await tempService.initialize(this.controlNode, this.controlDatabase, { registerHandler: false });
|
|
6772
7253
|
}
|
|
6773
7254
|
await tempService.dialInvite(invite);
|
|
6774
7255
|
return;
|
|
@@ -6784,14 +7265,20 @@ export class CadreNode {
|
|
|
6784
7265
|
*
|
|
6785
7266
|
* @param options - Configuration for the solicitation service
|
|
6786
7267
|
*/
|
|
6787
|
-
initializeStrandSolicitation(options) {
|
|
7268
|
+
async initializeStrandSolicitation(options) {
|
|
6788
7269
|
if (!this.controlNode) {
|
|
6789
7270
|
throw new Error('CadreNode must be started before initializing strand solicitation');
|
|
6790
7271
|
}
|
|
6791
|
-
|
|
7272
|
+
const service = new StrandSolicitationService({
|
|
6792
7273
|
...options,
|
|
6793
7274
|
partyId: this.config.controlNetwork.partyId,
|
|
6794
7275
|
cadrePeerAddrs: this.getMultiaddrs(),
|
|
7276
|
+
// Every formation deadline derives from this node's declared link, like every other
|
|
7277
|
+
// dial budget here; a caller's own declaration on the formation config still wins.
|
|
7278
|
+
formationConfig: {
|
|
7279
|
+
linkRoundTripMs: this.config.network?.linkRoundTripMs,
|
|
7280
|
+
...options?.formationConfig
|
|
7281
|
+
},
|
|
6795
7282
|
// Overrides any caller-supplied hook, exactly like partyId/cadrePeerAddrs: only
|
|
6796
7283
|
// this node can say which strand-network addresses it is actually listening on,
|
|
6797
7284
|
// and a wrong answer here seeds a joiner's mesh with addresses that reach nobody.
|
|
@@ -6799,10 +7286,23 @@ export class CadreNode {
|
|
|
6799
7286
|
// Overridden on the same grounds: only this node holds the running strand
|
|
6800
7287
|
// instance's database and this party's `StrandPartyKey` identity, and a wrong
|
|
6801
7288
|
// issuer here would admit joiners under someone else's authority.
|
|
6802
|
-
issueMembershipInvite: (strandId) => this.issueStrandMembershipInvite(strandId)
|
|
7289
|
+
issueMembershipInvite: (strandId, signal) => this.issueStrandMembershipInvite(strandId, signal)
|
|
6803
7290
|
});
|
|
6804
|
-
//
|
|
6805
|
-
this
|
|
7291
|
+
// Set before the registration is awaited, so a concurrent createOpenInvitation /
|
|
7292
|
+
// formStrand uses this service instead of building a second one whose handler
|
|
7293
|
+
// would collide; a failed registration puts the previous service back.
|
|
7294
|
+
// NOTE: same stray-handler caveat as installSeedBootstrapService — a failed
|
|
7295
|
+
// peer-store merge leaves the formation handler registered, so a retry is a duplicate.
|
|
7296
|
+
const previous = this.strandSolicitationService;
|
|
7297
|
+
this.strandSolicitationService = service;
|
|
7298
|
+
try {
|
|
7299
|
+
await service.registerResponder(this.controlNode);
|
|
7300
|
+
}
|
|
7301
|
+
catch (error) {
|
|
7302
|
+
if (this.strandSolicitationService === service)
|
|
7303
|
+
this.strandSolicitationService = previous;
|
|
7304
|
+
throw error;
|
|
7305
|
+
}
|
|
6806
7306
|
log('Strand solicitation service initialized');
|
|
6807
7307
|
}
|
|
6808
7308
|
/**
|
|
@@ -6822,7 +7322,7 @@ export class CadreNode {
|
|
|
6822
7322
|
) {
|
|
6823
7323
|
if (!this.strandSolicitationService) {
|
|
6824
7324
|
// Create a temporary service for creating invitations
|
|
6825
|
-
this.initializeStrandSolicitation();
|
|
7325
|
+
await this.initializeStrandSolicitation();
|
|
6826
7326
|
}
|
|
6827
7327
|
const bootstrap = this.getMultiaddrs();
|
|
6828
7328
|
if (bootstrap.length === 0) {
|
|
@@ -6842,10 +7342,10 @@ export class CadreNode {
|
|
|
6842
7342
|
throw new Error('CadreNode must be started before forming strands');
|
|
6843
7343
|
}
|
|
6844
7344
|
if (!this.strandSolicitationService) {
|
|
6845
|
-
this.initializeStrandSolicitation();
|
|
7345
|
+
await this.initializeStrandSolicitation();
|
|
6846
7346
|
}
|
|
6847
7347
|
const result = await this.strandSolicitationService.formStrand(invitation, disclosure, this.controlNode);
|
|
6848
|
-
this.
|
|
7348
|
+
await this.recordFormationStrandAddrs(result.strandId, result.strandAddrs);
|
|
6849
7349
|
// A closed host strand's approval carries the joiner's own membership invitation —
|
|
6850
7350
|
// persist this party's identity key and stage the invitation for bring-up to redeem.
|
|
6851
7351
|
// Runs AFTER the addr recording so a persistence failure (which throws — see the
|
|
@@ -6858,7 +7358,9 @@ export class CadreNode {
|
|
|
6858
7358
|
}
|
|
6859
7359
|
/**
|
|
6860
7360
|
* Remember the strand a formation just joined (see {@link joinedStrandStore}), so it
|
|
6861
|
-
* is re-offered after a restart even when the app is killed before its `addStrand
|
|
7361
|
+
* is re-offered after a restart even when the app is killed before its `addStrand`, and
|
|
7362
|
+
* published party-wide by the next connected owner reconcile pass. A re-join also
|
|
7363
|
+
* cancels a party-wide removal a self-revocation queued for the strand.
|
|
6862
7364
|
* Last in {@link formStrand}, after the membership adoption, so a failure here leaves
|
|
6863
7365
|
* the party key and staged invitation in place for the re-formation it asks for.
|
|
6864
7366
|
*
|
|
@@ -6867,7 +7369,7 @@ export class CadreNode {
|
|
|
6867
7369
|
*/
|
|
6868
7370
|
async rememberFormedStrand(result) {
|
|
6869
7371
|
try {
|
|
6870
|
-
await this.
|
|
7372
|
+
await this.joinedStrands.remember({
|
|
6871
7373
|
Id: result.strandId,
|
|
6872
7374
|
Type: result.memberPrivateKey ? 'c' : 'o',
|
|
6873
7375
|
MemberPrivateKey: result.memberPrivateKey ?? null,
|
|
@@ -6958,9 +7460,15 @@ export class CadreNode {
|
|
|
6958
7460
|
* founder's publish/launch paths mint it, so this is a not-yet-converged sibling.)
|
|
6959
7461
|
* The manager maps the throw to a clean retryable rejection BEFORE the formation
|
|
6960
7462
|
* token is spent.
|
|
6961
|
-
* - Closed host strand
|
|
6962
|
-
*
|
|
6963
|
-
*
|
|
7463
|
+
* - Closed host strand whose runtime is HIBERNATING or being quiesced → woken first
|
|
7464
|
+
* ({@link wakeHostStrandForFormation}, bounded by `signal`), then issued as below. In
|
|
7465
|
+
* every state the redemption counts as activity, so the host stays up for the
|
|
7466
|
+
* joiner's first sync.
|
|
7467
|
+
* - Closed host strand with no running local instance/database (never launched, still
|
|
7468
|
+
* starting, or a hibernating one whose wake failed or outran `signal`) →
|
|
7469
|
+
* throw, same mapping: a joiner admitted without an invitation would look joined and
|
|
7470
|
+
* never become a member, and a responder not running the strand cannot serve its sync
|
|
7471
|
+
* anyway.
|
|
6964
7472
|
* - Closed host strand whose LIVE rows carry the pre-split fingerprint
|
|
6965
7473
|
* (`assertNotPreSplitStrand`) → throw `PreSplitStrandIdentityError`, same mapping as
|
|
6966
7474
|
* the recorded refusal. Covers the responders that never ran a refused founder launch:
|
|
@@ -6970,7 +7478,12 @@ export class CadreNode {
|
|
|
6970
7478
|
* permanent diagnosis. Identity is checked BEFORE the runtime: it is the cheaper read
|
|
6971
7479
|
* and the more actionable diagnosis when both are missing (a missing runtime is
|
|
6972
7480
|
* transient, a missing identity is not), and it keeps the branch reachable without
|
|
6973
|
-
* standing a strand runtime up.
|
|
7481
|
+
* standing a strand runtime up. For the same reason every control-database check runs
|
|
7482
|
+
* before a wake: a strand that cannot issue anyway is not woken.
|
|
7483
|
+
*
|
|
7484
|
+
* `signal` is the formation's provisioning budget. Once it has aborted nothing is
|
|
7485
|
+
* issued — the joiner has already been told to retry, and an invitation written now
|
|
7486
|
+
* would only sit in the strand until it expires.
|
|
6974
7487
|
*
|
|
6975
7488
|
* The invitation expires `MEMBERSHIP_INVITE_TTL_MS` from now — see that constant for
|
|
6976
7489
|
* the slow-joiner / lost-result tradeoff.
|
|
@@ -6985,7 +7498,7 @@ export class CadreNode {
|
|
|
6985
7498
|
* `MEMBERSHIP_INVITE_UNAVAILABLE_REASON` forever; that flow needs manager delegation,
|
|
6986
7499
|
* not a retry.
|
|
6987
7500
|
*/
|
|
6988
|
-
async issueStrandMembershipInvite(strandId) {
|
|
7501
|
+
async issueStrandMembershipInvite(strandId, signal) {
|
|
6989
7502
|
if (!this.controlDatabase) {
|
|
6990
7503
|
throw new Error(`Cannot issue a membership invitation for strand ${strandId}: control database unavailable`);
|
|
6991
7504
|
}
|
|
@@ -7007,56 +7520,99 @@ export class CadreNode {
|
|
|
7007
7520
|
throw new Error(`Cannot issue a membership invitation for closed strand ${strandId}: this party holds ` +
|
|
7008
7521
|
'no StrandPartyKey row for it (identity not yet converged from the machine that published it)');
|
|
7009
7522
|
}
|
|
7523
|
+
await this.wakeHostStrandForFormation(strandId, signal);
|
|
7010
7524
|
const db = this.strandManager.getInstance(strandId)?.database?.getDatabase();
|
|
7011
7525
|
if (!db) {
|
|
7012
7526
|
throw new Error(`Cannot issue a membership invitation for closed strand ${strandId}: its runtime is ` +
|
|
7013
|
-
'not live on this responder (not launched,
|
|
7527
|
+
'not live on this responder (not launched, or still starting)');
|
|
7014
7528
|
}
|
|
7015
7529
|
if (row.MemberPrivateKey) {
|
|
7016
7530
|
await assertNotPreSplitStrand(db, strandId, strandMemberKeyPair(row.MemberPrivateKey).publicKeyB64);
|
|
7017
7531
|
}
|
|
7532
|
+
if (signal?.aborted) {
|
|
7533
|
+
throw new Error(`Cannot issue a membership invitation for closed strand ${strandId}: the formation ` +
|
|
7534
|
+
'provisioning budget expired before issuance');
|
|
7535
|
+
}
|
|
7018
7536
|
return await issueInvite(db, {
|
|
7019
7537
|
managerKeyPair: strandMemberKeyPair(partyKey),
|
|
7020
7538
|
expiration: Date.now() + MEMBERSHIP_INVITE_TTL_MS
|
|
7021
7539
|
});
|
|
7022
7540
|
}
|
|
7023
7541
|
/**
|
|
7024
|
-
*
|
|
7025
|
-
*
|
|
7026
|
-
*
|
|
7027
|
-
*
|
|
7542
|
+
* Count a bound closed-strand redemption as activity on the host strand, and wake the
|
|
7543
|
+
* strand when it is HIBERNATING or being quiesced (its database is closing) so the
|
|
7544
|
+
* membership invitation can be issued. Only reached after the formation manager has
|
|
7545
|
+
* authorized the redemption (token, disclosure, outside approval, seat pre-check), so only
|
|
7546
|
+
* a caller already entitled to the strand's member key can cause a wake. No other state is
|
|
7547
|
+
* woken — never launched or still starting is not something this node recovers from on
|
|
7548
|
+
* demand; the caller's live-database check refuses those.
|
|
7549
|
+
*
|
|
7550
|
+
* The activity is recorded in every state, through the hibernation manager rather than
|
|
7551
|
+
* {@link recordStrandActivity}, whose push fan-out would wake this party's phones for
|
|
7552
|
+
* nothing. It keeps the host up for the joiner's first sync: a live strand's idle timer
|
|
7553
|
+
* restarts, a check-in window that happens to have the strand live sees activity and
|
|
7554
|
+
* leaves it up instead of re-quiescing it, and a hibernating strand has its idle →
|
|
7555
|
+
* hibernate timers re-armed once the wake leaves it `active`. The explicit
|
|
7556
|
+
* {@link wakeStrand} coalesces onto the wake `recordActivity` began, and still wakes when
|
|
7557
|
+
* `recordActivity` is a no-op (hibernation disabled but the strand force-hibernated, or
|
|
7558
|
+
* the manager stopped).
|
|
7559
|
+
*
|
|
7560
|
+
* Bounded by `signal` (the formation's provisioning budget): when it aborts first this
|
|
7561
|
+
* throws — a retryable rejection, token unspent — and leaves the wake running, so the
|
|
7562
|
+
* joiner's retry finds the strand live.
|
|
7563
|
+
*/
|
|
7564
|
+
async wakeHostStrandForFormation(strandId, signal) {
|
|
7565
|
+
const instance = this.strandManager.getInstance(strandId);
|
|
7566
|
+
if (!instance) {
|
|
7567
|
+
return;
|
|
7568
|
+
}
|
|
7569
|
+
this.hibernationManager.recordActivity(instance);
|
|
7570
|
+
if (!this.strandManager.isQuiescing(strandId) && (instance.status !== 'hibernating' || instance.database)) {
|
|
7571
|
+
return;
|
|
7572
|
+
}
|
|
7573
|
+
log('wakeHostStrandForFormation: waking host strand %s (%s) for an authorized formation', strandId, instance.status);
|
|
7574
|
+
const wake = this.wakeStrand(strandId);
|
|
7575
|
+
if (await resolvesBeforeAbort(wake, signal)) {
|
|
7576
|
+
return;
|
|
7577
|
+
}
|
|
7578
|
+
void wake.catch((error) => log('wakeHostStrandForFormation: background wake of host strand %s failed: %o', strandId, error));
|
|
7579
|
+
throw new Error(`Cannot issue a membership invitation for closed strand ${strandId}: its hibernating ` +
|
|
7580
|
+
'runtime did not wake within the formation provisioning budget (still waking)');
|
|
7581
|
+
}
|
|
7582
|
+
/**
|
|
7583
|
+
* Keep the responder's strand-network addresses for a strand this node just formed
|
|
7584
|
+
* (see {@link formationStrandAddrs}), so the strand's discovery seed has something to
|
|
7585
|
+
* dial when the app launches it.
|
|
7028
7586
|
*
|
|
7029
7587
|
* The carried list is peer-agnostic (`sanitizeStrandAddrs` bounds and parses it,
|
|
7030
7588
|
* nothing more), so it is attributed per peer here (`groupAddrsByPeerId`, the same
|
|
7031
|
-
* rule the address-book merge applies) and
|
|
7032
|
-
*
|
|
7033
|
-
*
|
|
7034
|
-
*
|
|
7035
|
-
*
|
|
7036
|
-
*
|
|
7037
|
-
|
|
7038
|
-
|
|
7039
|
-
|
|
7040
|
-
|
|
7589
|
+
* rule the address-book merge applies) and kept grouped by peer; an entry naming no
|
|
7590
|
+
* destination peer is dropped there. Scoped strictly to `strandId`: these addresses
|
|
7591
|
+
* reach ONE strand node of ONE other party and must never seed another strand's mesh
|
|
7592
|
+
* or the control peerStore.
|
|
7593
|
+
*
|
|
7594
|
+
* A list that names a peer REPLACES the strand's earlier one: the responder
|
|
7595
|
+
* disclosed its current addresses, so the older list is stale. An empty list, or one
|
|
7596
|
+
* naming no peer, records nothing, so it cannot wipe an earlier disclosure.
|
|
7597
|
+
*
|
|
7598
|
+
* A strand already running when this lands (a re-formation, which is the recovery
|
|
7599
|
+
* path for a dead address) gets the new addresses merged into its address book now.
|
|
7600
|
+
* Nothing else would deliver them: the seed is read only at launch and resume, and the
|
|
7601
|
+
* refresh pass skips a peer FRET already holds a record for — the stale one, here.
|
|
7602
|
+
*/
|
|
7603
|
+
async recordFormationStrandAddrs(strandId, strandAddrs) {
|
|
7604
|
+
const attributed = [...groupAddrsByPeerId([...strandAddrs]).values()]
|
|
7605
|
+
.flatMap((peerAddrs) => peerAddrs.map((addr) => addr.toString()));
|
|
7606
|
+
if (attributed.length === 0) {
|
|
7607
|
+
log('formStrand: responder disclosed no usable strand addrs for %s (%d carried) — cross-party seed unchanged', strandId, strandAddrs.length);
|
|
7041
7608
|
return;
|
|
7042
7609
|
}
|
|
7043
|
-
|
|
7044
|
-
|
|
7045
|
-
|
|
7046
|
-
|
|
7047
|
-
|
|
7048
|
-
|
|
7049
|
-
issuedAt: 0,
|
|
7050
|
-
lastSeenAt: now
|
|
7051
|
-
}, 'formStrand');
|
|
7052
|
-
peers++;
|
|
7053
|
-
}
|
|
7054
|
-
// A strand already running when this lands (a re-formation) would otherwise wait out
|
|
7055
|
-
// the rest of its ~10-minute refresh throttle before the new addresses reached its
|
|
7056
|
-
// address book. Clearing the stamp makes the next 15 s reconcile tick merge them,
|
|
7057
|
-
// which matters precisely because re-forming is the recovery path for a dead entry.
|
|
7058
|
-
this.strandPeerAddrRefreshAt.delete(strandId);
|
|
7059
|
-
log('formStrand: recorded %d cross-party strand addr(s) for %s under %d peer(s)', strandAddrs.length, strandId, peers);
|
|
7610
|
+
this.formationStrandAddrs.set(strandId, attributed);
|
|
7611
|
+
log('formStrand: kept %d of %d cross-party strand addr(s) for %s', attributed.length, strandAddrs.length, strandId);
|
|
7612
|
+
const runningNode = this.strandManager.getInstance(strandId)?.libp2pNode;
|
|
7613
|
+
if (runningNode) {
|
|
7614
|
+
await this.mergeStrandPeerAddrs(runningNode, attributed, strandId);
|
|
7615
|
+
}
|
|
7060
7616
|
}
|
|
7061
7617
|
/**
|
|
7062
7618
|
* Encode an open invitation for out-of-band delivery (QR, link, etc.).
|