@serfab/cadre-core 0.12.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -8
- package/dist/bootstrap-peer-store-file.js +2 -2
- package/dist/bootstrap-peer-store.d.ts +32 -12
- package/dist/bootstrap-peer-store.js +41 -15
- package/dist/bootstrap-peer-store.js.map +1 -1
- package/dist/cadre-node.d.ts +883 -102
- package/dist/cadre-node.js +1800 -209
- package/dist/cadre-node.js.map +1 -1
- package/dist/control-authorization.d.ts +2 -2
- package/dist/control-authorization.js +1 -0
- package/dist/control-authorization.js.map +1 -1
- package/dist/control-cohort.d.ts +13 -26
- package/dist/control-cohort.js +0 -26
- package/dist/control-cohort.js.map +1 -1
- package/dist/control-database.d.ts +318 -44
- package/dist/control-database.js +516 -74
- package/dist/control-database.js.map +1 -1
- package/dist/control-read-retry.d.ts +40 -11
- package/dist/control-read-retry.js +48 -15
- package/dist/control-read-retry.js.map +1 -1
- package/dist/control-retry.d.ts +58 -3
- package/dist/control-retry.js +58 -3
- package/dist/control-retry.js.map +1 -1
- package/dist/control-schema.d.ts +1 -1
- package/dist/control-schema.js +133 -21
- package/dist/control-schema.js.map +1 -1
- package/dist/control-write-retry.d.ts +80 -11
- package/dist/control-write-retry.js +168 -29
- package/dist/control-write-retry.js.map +1 -1
- package/dist/enrolled-machine-store-file.d.ts +13 -0
- package/dist/enrolled-machine-store-file.js +41 -0
- package/dist/enrolled-machine-store-file.js.map +1 -0
- package/dist/enrolled-machine-store.d.ts +81 -0
- package/dist/enrolled-machine-store.js +225 -0
- package/dist/enrolled-machine-store.js.map +1 -0
- package/dist/file-durable-slot.js +5 -3
- package/dist/file-durable-slot.js.map +1 -1
- package/dist/index.d.ts +17 -8
- package/dist/index.js +54 -9
- package/dist/index.js.map +1 -1
- package/dist/membership-connection-gater.d.ts +15 -1
- package/dist/membership-connection-gater.js +9 -3
- package/dist/membership-connection-gater.js.map +1 -1
- package/dist/node-local-snapshot.d.ts +21 -7
- package/dist/node-local-snapshot.js +30 -9
- package/dist/node-local-snapshot.js.map +1 -1
- package/dist/peer-dial.d.ts +143 -0
- package/dist/peer-dial.js +176 -0
- package/dist/peer-dial.js.map +1 -0
- package/dist/{strand-backfill.d.ts → peer-join-backfill.d.ts} +49 -20
- package/dist/{strand-backfill.js → peer-join-backfill.js} +133 -52
- package/dist/peer-join-backfill.js.map +1 -0
- package/dist/relay-addrs.d.ts +160 -49
- package/dist/relay-addrs.js +287 -60
- package/dist/relay-addrs.js.map +1 -1
- package/dist/relay-reservation.d.ts +71 -3
- package/dist/relay-reservation.js +116 -8
- package/dist/relay-reservation.js.map +1 -1
- package/dist/seed-bootstrap.d.ts +30 -1
- package/dist/seed-bootstrap.js +62 -17
- package/dist/seed-bootstrap.js.map +1 -1
- package/dist/storage-scope.d.ts +62 -0
- package/dist/storage-scope.js +75 -0
- package/dist/storage-scope.js.map +1 -0
- package/dist/strand-database.d.ts +41 -11
- package/dist/strand-database.js +48 -18
- package/dist/strand-database.js.map +1 -1
- package/dist/strand-first-sync-gate.d.ts +152 -0
- package/dist/strand-first-sync-gate.js +208 -0
- package/dist/strand-first-sync-gate.js.map +1 -0
- package/dist/strand-formation-manager.d.ts +99 -5
- package/dist/strand-formation-manager.js +119 -8
- package/dist/strand-formation-manager.js.map +1 -1
- package/dist/strand-formation-protocol.d.ts +97 -3
- package/dist/strand-formation-protocol.js +108 -2
- package/dist/strand-formation-protocol.js.map +1 -1
- package/dist/strand-instance-manager.d.ts +389 -19
- package/dist/strand-instance-manager.js +678 -65
- package/dist/strand-instance-manager.js.map +1 -1
- package/dist/strand-member-key.d.ts +20 -12
- package/dist/strand-member-key.js +20 -12
- package/dist/strand-member-key.js.map +1 -1
- package/dist/strand-member-registry.js +3 -0
- package/dist/strand-member-registry.js.map +1 -1
- package/dist/strand-membership-reconciler.d.ts +361 -0
- package/dist/strand-membership-reconciler.js +548 -0
- package/dist/strand-membership-reconciler.js.map +1 -0
- package/dist/strand-membership-writer.d.ts +143 -11
- package/dist/strand-membership-writer.js +419 -135
- package/dist/strand-membership-writer.js.map +1 -1
- package/dist/strand-network-config.d.ts +116 -0
- package/dist/strand-network-config.js +192 -0
- package/dist/strand-network-config.js.map +1 -0
- package/dist/strand-revocation-enforcer.d.ts +392 -0
- package/dist/strand-revocation-enforcer.js +533 -0
- package/dist/strand-revocation-enforcer.js.map +1 -0
- package/dist/strand-solicitation.d.ts +17 -1
- package/dist/strand-solicitation.js +13 -2
- package/dist/strand-solicitation.js.map +1 -1
- package/dist/strand-wake-protocol.d.ts +11 -7
- package/dist/strand-wake-protocol.js +20 -52
- package/dist/strand-wake-protocol.js.map +1 -1
- package/dist/strand-watcher.js +12 -1
- package/dist/strand-watcher.js.map +1 -1
- package/dist/timed-step.d.ts +12 -0
- package/dist/timed-step.js +28 -0
- package/dist/timed-step.js.map +1 -0
- package/dist/timeout-scheduler.d.ts +19 -0
- package/dist/timeout-scheduler.js +22 -0
- package/dist/timeout-scheduler.js.map +1 -0
- package/dist/types.d.ts +472 -46
- package/dist/types.js +7 -1
- package/dist/types.js.map +1 -1
- package/package.json +16 -8
- package/src/bootstrap-peer-store-file.ts +2 -2
- package/src/bootstrap-peer-store.ts +69 -27
- package/src/cadre-node.ts +1926 -228
- package/src/control-authorization.ts +2 -1
- package/src/control-cohort.ts +122 -130
- package/src/control-database.ts +586 -74
- package/src/control-read-retry.ts +202 -167
- package/src/control-retry.ts +297 -190
- package/src/control-schema.ts +133 -21
- package/src/control-write-retry.ts +525 -383
- package/src/enrolled-machine-store-file.ts +43 -0
- package/src/enrolled-machine-store.ts +280 -0
- package/src/file-durable-slot.ts +5 -3
- package/src/index.ts +134 -15
- package/src/membership-connection-gater.ts +9 -3
- package/src/node-local-snapshot.ts +38 -9
- package/src/peer-addr-book.ts +251 -250
- package/src/peer-dial.ts +215 -0
- package/src/{strand-backfill.ts → peer-join-backfill.ts} +478 -389
- package/src/relay-addrs.ts +458 -218
- package/src/relay-reservation.ts +951 -828
- package/src/seed-bootstrap.ts +76 -18
- package/src/storage-scope.ts +78 -0
- package/src/strand-database.ts +61 -21
- package/src/strand-first-sync-gate.ts +258 -0
- package/src/strand-formation-manager.ts +162 -8
- package/src/strand-formation-protocol.ts +850 -692
- package/src/strand-instance-manager.ts +877 -74
- package/src/strand-member-key.ts +20 -12
- package/src/strand-member-registry.ts +3 -0
- package/src/strand-membership-reconciler.ts +680 -0
- package/src/strand-membership-writer.ts +550 -190
- package/src/strand-network-config.ts +250 -0
- package/src/strand-revocation-enforcer.ts +653 -0
- package/src/strand-solicitation.ts +499 -471
- package/src/strand-wake-protocol.ts +21 -66
- package/src/strand-watcher.ts +325 -314
- package/src/timed-step.ts +28 -0
- package/src/timeout-scheduler.ts +28 -0
- package/src/types.ts +487 -52
- package/dist/strand-backfill.js.map +0 -1
package/dist/cadre-node.js
CHANGED
|
@@ -4,39 +4,47 @@ import { peerIdFromString, peerIdFromPrivateKey } from '@libp2p/peer-id';
|
|
|
4
4
|
import { createLibp2pNode } from '@optimystic/db-p2p';
|
|
5
5
|
import { wrapStorageWithCache, disposeStorageCache } from '@serfab/quereus-plugin-sereus';
|
|
6
6
|
import { multiaddr } from '@multiformats/multiaddr';
|
|
7
|
-
import {
|
|
7
|
+
import { controlClusterPolicy, CONTROL_REPLICATION_BREADTH, DEFAULT_CHECKIN_WINDOW_MS } from './types.js';
|
|
8
8
|
import { sign } from '@optimystic/quereus-plugin-crypto';
|
|
9
9
|
import { ed25519KeyPairFromLibp2p, ed25519PublicKeyFromPrivate, requireEd25519PublicKeyB64 } from './ed25519-key.js';
|
|
10
10
|
import { strandTransportKey } from './strand-transport-key.js';
|
|
11
|
+
import { controlStorageScope } from './storage-scope.js';
|
|
12
|
+
import { generateStrandMemberKey, strandMemberKeyPair } from './strand-member-key.js';
|
|
13
|
+
import { assertNotPreSplitStrand, issueInvite, PreSplitStrandIdentityError } from './strand-membership-writer.js';
|
|
14
|
+
import { MEMBERSHIP_INVITE_TTL_MS } from './strand-formation-manager.js';
|
|
11
15
|
import { DEFAULT_IDENTITY_KEY_ID } from './key-store.js';
|
|
12
16
|
import { loadOrCreateIdentityKey } from './identity-key.js';
|
|
13
17
|
import { MemoryTrustedOwnerStore } from './trusted-owner-store.js';
|
|
14
18
|
import { MemoryBootstrapPeerStore } from './bootstrap-peer-store.js';
|
|
19
|
+
import { MemoryEnrolledMachineStore } from './enrolled-machine-store.js';
|
|
15
20
|
import { mergePeerAddrs, groupAddrsByPeerId } from './peer-addr-book.js';
|
|
16
21
|
import { verifyCadrePeerVoucher } from './peer-authorization.js';
|
|
17
22
|
import { ed25519PublicKeyB64FromPeerId } from './seed-bootstrap.js';
|
|
18
|
-
import { signPeerRecord, verifyPeerRecordSignature, isPeerRecordFresh, orderSignalingFirst, isSignalingAddr, withTrailingPeerId, currentMemberTrustPolicy, DEFAULT_PEER_RECORD_MAX_AGE_MS, DEFAULT_PEER_RECORD_HEARTBEAT_MS } from './peer-record.js';
|
|
23
|
+
import { signPeerRecord, verifyPeerRecordSignature, isPeerRecordFresh, orderSignalingFirst, isSignalingAddr, trailingPeerId, withTrailingPeerId, currentMemberTrustPolicy, DEFAULT_PEER_RECORD_MAX_AGE_MS, DEFAULT_PEER_RECORD_HEARTBEAT_MS } from './peer-record.js';
|
|
19
24
|
import { signDeviceTokenRecord, verifyDeviceTokenSignature, isPushPlatform } from './device-token.js';
|
|
20
25
|
import { StrandWatcher } from './strand-watcher.js';
|
|
21
|
-
import { StrandInstanceManager } from './strand-instance-manager.js';
|
|
26
|
+
import { StrandInstanceManager, liveStrandStatus } from './strand-instance-manager.js';
|
|
27
|
+
import { PeerJoinBackfill } from './peer-join-backfill.js';
|
|
22
28
|
import { deriveCohortMembers } from './strand-cohort.js';
|
|
23
|
-
import { selectControlCohortDials,
|
|
24
|
-
import {
|
|
29
|
+
import { selectControlCohortDials, DEFAULT_CONTROL_COHORT_RECONCILE_MS, DEFAULT_CONTROL_COHORT_TARGET_DEGREE } from './control-cohort.js';
|
|
30
|
+
import { dialPeerAddrs, DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS, DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS } from './peer-dial.js';
|
|
25
31
|
import { EnrollmentService } from './enrollment.js';
|
|
26
32
|
import { HibernationManager } from './hibernation-manager.js';
|
|
27
|
-
import { ControlDatabase } from './control-database.js';
|
|
33
|
+
import { ControlDatabase, isStrandIdConflict } from './control-database.js';
|
|
28
34
|
import { SeedBootstrapService } from './seed-bootstrap.js';
|
|
29
35
|
import { StrandSolicitationService } from './strand-solicitation.js';
|
|
36
|
+
import { MAX_STRAND_ADDRS } from './strand-formation-protocol.js';
|
|
30
37
|
import { createMembershipConnectionGater, DEFAULT_ENROLLMENT_WINDOW_MS, UnauthorizedReservationBudget } from './membership-connection-gater.js';
|
|
31
38
|
import { StrandWakeService, dialWake } from './strand-wake-protocol.js';
|
|
32
39
|
import { StrandAddrService, collectStrandAddrs } from './strand-addr-protocol.js';
|
|
33
40
|
import { DelegateAdmissionStore, extractCircuitRelayTargets, dueRelayAnnounces, pruneStoppedStrandAnnounces, peerStrandKey } from './delegate-admission.js';
|
|
34
|
-
import { relayCircuitAddrs, resolveListenAddrs, RelayReservationFailedError } from './relay-addrs.js';
|
|
41
|
+
import { relayCircuitAddrs, resolveListenAddrs, resolveTransportOptions, RelayReservationFailedError } from './relay-addrs.js';
|
|
35
42
|
import { replacesAdvertisedAddrs, resolveAnnounceAddrs } from './announce-addrs.js';
|
|
36
43
|
import { superviseRelayReservation, resolveRelayReservationState } from './relay-reservation.js';
|
|
37
44
|
import { PushFanoutService } from './push-fanout.js';
|
|
38
45
|
import { summarizeConnectionPaths, classifyTransport } from './diagnostics/connection-path.js';
|
|
39
46
|
import { TurnRelayTracker } from './diagnostics/webrtc-turn-tracker.js';
|
|
47
|
+
import { timedStep } from './timed-step.js';
|
|
40
48
|
const log = debug('sereus:cadre:node');
|
|
41
49
|
const timing = debug('sereus:cadre:timing');
|
|
42
50
|
/**
|
|
@@ -64,6 +72,30 @@ export const STRAND_PEER_ADDR_REFRESH_MS = 10 * 60 * 1000;
|
|
|
64
72
|
function relayStrandAddrPeer(relay) {
|
|
65
73
|
return { peerId: relay.relayPeerId, addrs: [multiaddr(relay.relayAddr)] };
|
|
66
74
|
}
|
|
75
|
+
/**
|
|
76
|
+
* `primary` followed by the entries of `extra` it does not already contain, de-duplicated
|
|
77
|
+
* and order-preserving.
|
|
78
|
+
*
|
|
79
|
+
* Used wherever a strand's discovery seed unions two sources of address strings — the
|
|
80
|
+
* freshly-RPC'd sibling answers and the cross-party addresses learned at formation. The
|
|
81
|
+
* caller passes the FRESHER source as `primary`, so a stale entry can only ever be
|
|
82
|
+
* appended, never promoted ahead of a current one, and each source keeps whatever
|
|
83
|
+
* signaling-first ordering it arrived with.
|
|
84
|
+
*/
|
|
85
|
+
function unionAddrs(primary, extra) {
|
|
86
|
+
if (extra.length === 0) {
|
|
87
|
+
return [...primary];
|
|
88
|
+
}
|
|
89
|
+
const out = [...primary];
|
|
90
|
+
const seen = new Set(primary);
|
|
91
|
+
for (const addr of extra) {
|
|
92
|
+
if (seen.has(addr))
|
|
93
|
+
continue;
|
|
94
|
+
seen.add(addr);
|
|
95
|
+
out.push(addr);
|
|
96
|
+
}
|
|
97
|
+
return out;
|
|
98
|
+
}
|
|
67
99
|
/**
|
|
68
100
|
* Build the signing callback the `ControlDatabase` writers expect: they hand it the
|
|
69
101
|
* canonical row-bound message BYTES (see `buildAuthorizationMessage`), and it ed25519-signs
|
|
@@ -84,6 +116,56 @@ function requireNonBlank(value, label) {
|
|
|
84
116
|
}
|
|
85
117
|
return trimmed;
|
|
86
118
|
}
|
|
119
|
+
/**
|
|
120
|
+
* Is the live `Strand` row the one a publish of `desired` would have produced?
|
|
121
|
+
*
|
|
122
|
+
* "Identical content" means `(Type, MemberPrivateKey)` and deliberately NOT
|
|
123
|
+
* `FounderOwnerKey`: that column is provenance (WHICH machine published the row), not
|
|
124
|
+
* content. Two machines of one party racing to found the same id must keep resolving as
|
|
125
|
+
* "the winner's row stands" — the loser adopts the row and, since the winner's key is on
|
|
126
|
+
* it, correctly comes up as a joiner (see `launchStrand`'s founder derivation). Comparing
|
|
127
|
+
* it here would turn that benign race into a hard throw. `Id` is the lookup key and
|
|
128
|
+
* `StampId` is a single-use nonce, so the two compared columns are all the content there
|
|
129
|
+
* is. A live row matching on both is, by construction, the state a repeat publish would
|
|
130
|
+
* have reached, whichever branch of `Strand.AuthorizedInsert` seated it (owner-signed, or
|
|
131
|
+
* the unsigned consent branch an invite redemption uses).
|
|
132
|
+
*
|
|
133
|
+
* Returns the mismatching column names, empty when the row matches. The KEY's value is
|
|
134
|
+
* deliberately never returned or logged — it is the closed strand's read-gating secret.
|
|
135
|
+
*/
|
|
136
|
+
function strandRowMismatches(live, desired) {
|
|
137
|
+
const mismatches = [];
|
|
138
|
+
if (live.Type !== desired.Type) {
|
|
139
|
+
mismatches.push(`Type is '${live.Type}', not the requested '${desired.Type}'`);
|
|
140
|
+
}
|
|
141
|
+
if ((live.MemberPrivateKey ?? null) !== (desired.MemberPrivateKey ?? null)) {
|
|
142
|
+
mismatches.push('MemberPrivateKey differs from the one supplied');
|
|
143
|
+
}
|
|
144
|
+
return mismatches;
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* The live row when it matches `desired` ({@link strandRowMismatches}); otherwise throw
|
|
148
|
+
* naming the columns that differ.
|
|
149
|
+
*
|
|
150
|
+
* A mismatch must stay a hard error: accepting one would let a retry reopen a strand the
|
|
151
|
+
* party closed, or swap the key that gates its reads — so the two cases the raw
|
|
152
|
+
* `UNIQUE constraint failed: Strand.Id` could not distinguish ("already done, carry on"
|
|
153
|
+
* vs. "genuine conflict, stop") are separated here rather than left to the caller.
|
|
154
|
+
*
|
|
155
|
+
* @param situation - How the row came to be there, as a verb phrase completing
|
|
156
|
+
* `Strand <id> …`, e.g. `'is already published'`.
|
|
157
|
+
*/
|
|
158
|
+
function requireMatchingStrandRow(live, desired, situation) {
|
|
159
|
+
const mismatches = strandRowMismatches(live, desired);
|
|
160
|
+
if (mismatches.length > 0) {
|
|
161
|
+
throw new Error(`Strand ${desired.Id} ${situation} with DIFFERENT content, so this publish is a ` +
|
|
162
|
+
`conflict, not a repeat: ${mismatches.join('; ')}. Reconcile deliberately — attach ` +
|
|
163
|
+
'the existing strand with addStrand/foundStrand to keep it, or unpublishStrand first ' +
|
|
164
|
+
'to re-seat it (destructive for a closed strand: its MemberPrivateKey is stored ' +
|
|
165
|
+
'nowhere else).');
|
|
166
|
+
}
|
|
167
|
+
return live;
|
|
168
|
+
}
|
|
87
169
|
/**
|
|
88
170
|
* CadreNode is the main entry point for a cadre member.
|
|
89
171
|
* It manages:
|
|
@@ -106,6 +188,14 @@ export class CadreNode {
|
|
|
106
188
|
this.trustedOwnerStore = null;
|
|
107
189
|
this.controlNode = null;
|
|
108
190
|
this.controlDatabase = null;
|
|
191
|
+
/**
|
|
192
|
+
* The CONTROL network's peer-join block catch-up — same module as the
|
|
193
|
+
* per-strand ones, but membership-gated (see {@link startControlBackfill}).
|
|
194
|
+
* Created in {@link start} after the control node and database are up, stopped
|
|
195
|
+
* and dropped in {@link cleanup} before either is torn down, so a stop()→
|
|
196
|
+
* start() cycle rebuilds it with a fresh caught-up-peer memo.
|
|
197
|
+
*/
|
|
198
|
+
this.controlBackfill = null;
|
|
109
199
|
this.strandWatcher = null;
|
|
110
200
|
this.seedBootstrapService = null;
|
|
111
201
|
this.strandSolicitationService = null;
|
|
@@ -128,10 +218,11 @@ export class CadreNode {
|
|
|
128
218
|
this._running = false;
|
|
129
219
|
/**
|
|
130
220
|
* The control database's own raw storage (cache-wrapped), resolved once per
|
|
131
|
-
* `start()` and released in {@link cleanup}. Owning it is what keeps
|
|
132
|
-
*
|
|
133
|
-
* `
|
|
134
|
-
*
|
|
221
|
+
* `start()` and released in {@link cleanup}. Owning it is what keeps the
|
|
222
|
+
* provider call for the control scope — `controlStorageScope(partyId)`, see
|
|
223
|
+
* `storage-scope.ts` — a once-per-runtime call: a `stop()` then `start()` cycle
|
|
224
|
+
* on this object re-resolves against a live cache, and never orphans the
|
|
225
|
+
* previous wrapper's registration in the shared cache pool.
|
|
135
226
|
*/
|
|
136
227
|
this.controlStorage = null;
|
|
137
228
|
/**
|
|
@@ -187,6 +278,14 @@ export class CadreNode {
|
|
|
187
278
|
this.controlBringUpInFlight = false;
|
|
188
279
|
/** Map of strandId -> sAppConfig for sAppId filtering and management */
|
|
189
280
|
this.sAppConfigs = new Map();
|
|
281
|
+
/**
|
|
282
|
+
* Strands the control network advertises that no local sAppConfig claims — the
|
|
283
|
+
* backlog behind `strand:discovered`. Maintained in {@link handleStrandAdded}
|
|
284
|
+
* (added), {@link addStrand} (claimed), {@link detachStrand} (stopped or its
|
|
285
|
+
* control row vanished) and {@link cleanup}. Read through
|
|
286
|
+
* {@link getDiscoveredStrands}.
|
|
287
|
+
*/
|
|
288
|
+
this.discoveredStrands = new Map();
|
|
190
289
|
/**
|
|
191
290
|
* Most-recently pushed invite addresses (see {@link setInviteAddresses}).
|
|
192
291
|
* When non-null these take priority over `libp2pNode.getMultiaddrs()` when
|
|
@@ -273,39 +372,131 @@ export class CadreNode {
|
|
|
273
372
|
*/
|
|
274
373
|
this.strandPeerAddrRefreshAt = new Map();
|
|
275
374
|
/**
|
|
276
|
-
*
|
|
277
|
-
*
|
|
278
|
-
*
|
|
279
|
-
*
|
|
280
|
-
*
|
|
281
|
-
*
|
|
375
|
+
* CROSS-PARTY strand-network addresses learned at formation, keyed by strandId — the
|
|
376
|
+
* responder's live strand addrs, carried back on the formation result
|
|
377
|
+
* (`FormationResultMessage.strandAddrs`) and recorded by {@link formStrand}.
|
|
378
|
+
*
|
|
379
|
+
* This is the ONLY cross-party discovery input there is. The strand-addr RPC that
|
|
380
|
+
* resolves a strand's addresses is membership-gated and answers own-party siblings
|
|
381
|
+
* only, so without this map a joiner's cohort seed for a two-party strand is empty and
|
|
382
|
+
* the mesh never forms.
|
|
383
|
+
*
|
|
384
|
+
* Read by {@link resolveCohortSeed} (launch + hibernation resume) and re-merged by
|
|
385
|
+
* every {@link refreshStrandPeerAddrs} pass — the re-merge is what keeps the entries
|
|
386
|
+
* alive past the peerStore's one-hour address expiry, since nothing can re-resolve
|
|
387
|
+
* them.
|
|
388
|
+
*
|
|
389
|
+
* IN-MEMORY and ONE-SHOT, deliberately (see `docs/strands.md`): the addresses die with
|
|
390
|
+
* the process, and a restarted initiator with no sibling running the strand is back to
|
|
391
|
+
* an empty seed until the durability work lands
|
|
392
|
+
* (`backlog/feat-cross-party-strand-addr-durability`). They are also never refreshed,
|
|
393
|
+
* so a responder whose relay reservation rotates before the initiator dials leaves a
|
|
394
|
+
* dead entry behind; recovery today is a fresh invitation.
|
|
395
|
+
*
|
|
396
|
+
* NOTE: entries are kept for the node's lifetime, including across a
|
|
397
|
+
* {@link stopStrand} — a stopped strand may be rediscovered by the watcher and must
|
|
398
|
+
* still get its seed. Each strand's list is capped at `MAX_STRAND_ADDRS`, but the
|
|
399
|
+
* number of KEYS is bounded only by the strands this node has ever formed as an
|
|
400
|
+
* initiator (not by time); if a node ever forms strands at scale, evict on
|
|
401
|
+
* `unpublishStrand` or cap the map.
|
|
402
|
+
*/
|
|
403
|
+
this.crossPartyStrandAddrs = new Map();
|
|
404
|
+
/**
|
|
405
|
+
* PENDING strand membership invitations learned at formation, keyed by strandId — the
|
|
406
|
+
* single-use `Strand.Invite` credential a closed-strand formation result carried back
|
|
407
|
+
* ({@link FormStrandResult.membershipInvite}), waiting for this node's strand bring-up
|
|
408
|
+
* to redeem (`consumeInvite` seats the `Strand.Member` row under this party's own key
|
|
409
|
+
* — the `strand-node-binds-member-peer` half of the party-identity chain).
|
|
410
|
+
*
|
|
411
|
+
* IN-MEMORY, deliberately, like {@link crossPartyStrandAddrs}: a restarted joiner
|
|
412
|
+
* re-forms from scratch anyway (durability is
|
|
413
|
+
* `backlog/feat-cross-party-strand-addr-durability`'s problem), and re-forming issues
|
|
414
|
+
* a fresh invitation. A re-formation against the same strand REPLACES the entry — the
|
|
415
|
+
* fresh invitation supersedes one that may have expired. The party key the invitation
|
|
416
|
+
* admits, by contrast, IS persisted (`StrandPartyKey`, seated by
|
|
417
|
+
* {@link adoptFormationMembershipInvite} before the entry lands here), so a lost
|
|
418
|
+
* invitation never orphans an identity.
|
|
419
|
+
*
|
|
420
|
+
* INVALIDATION is owned by the strand's bring-up membership reconciler
|
|
421
|
+
* (`strand-membership-reconciler.ts`, wired via `StartStrandConfig.pendingMembershipInvite`
|
|
422
|
+
* in {@link launchStrand}): it deletes the entry once the invitation is redeemed, burned
|
|
423
|
+
* against an already-seated member, or found dead (expired / cancelled / consumed
|
|
424
|
+
* elsewhere / the strand sealed). Until the strand is actually launched here the entry
|
|
425
|
+
* just waits.
|
|
426
|
+
*
|
|
427
|
+
* NOTE: entries live for the node's lifetime (one small pair per formed closed
|
|
428
|
+
* strand), the same unbounded-keys tripwire {@link crossPartyStrandAddrs} documents —
|
|
429
|
+
* if a node ever forms strands at scale, evict alongside that map.
|
|
430
|
+
*/
|
|
431
|
+
this.pendingMembershipInvites = new Map();
|
|
432
|
+
/**
|
|
433
|
+
* Founder launches refused because the strand was founded before the per-party
|
|
434
|
+
* identity split (`PreSplitStrandIdentityError`), keyed by strandId. Read by
|
|
435
|
+
* {@link issueStrandMembershipInvite}, which rethrows the recorded error so a bound
|
|
436
|
+
* redemption against the strand is rejected as "must be recreated" — not told to retry
|
|
437
|
+
* because the runtime is not live (a refused fresh launch leaves none) or failed by the
|
|
438
|
+
* `Strand.Invite` gate (a refused in-place founding leaves a joiner instance up whose
|
|
439
|
+
* party key is no manager).
|
|
440
|
+
*
|
|
441
|
+
* Written and cleared by {@link launchStrand}; also cleared by {@link detachStrand},
|
|
442
|
+
* {@link unpublishStrand} and {@link cleanup}, so a recreated id starts clean.
|
|
443
|
+
* In-memory: after a restart the next founder launch of the strand records it again;
|
|
444
|
+
* until then issuance still reads the fingerprint off the live rows whenever a runtime
|
|
445
|
+
* is up — only a strand with no runtime at all answers retryably meanwhile.
|
|
446
|
+
*/
|
|
447
|
+
this.strandLaunchRefusals = new Map();
|
|
448
|
+
/**
|
|
449
|
+
* Dial targets learned out of band, keyed by peer id, valued by the multiaddr
|
|
450
|
+
* strings this node was handed (parsed lazily, at dial time). Two writers:
|
|
451
|
+
* {@link recordSeedBootstrapPeers} retains the owner-flagged peers of every seed
|
|
452
|
+
* this node applies, and {@link addDrone} retains the addresses of every node
|
|
453
|
+
* this node adds. Two readers: {@link dialColdStartBootstrap} dials every entry
|
|
454
|
+
* while the control database still has no siblings, and
|
|
455
|
+
* {@link resolveControlDialAddrs} falls back to a sibling's entry when neither
|
|
456
|
+
* its signed record nor the address book yields an address.
|
|
282
457
|
*
|
|
283
458
|
* A STORE rather than a plain map (see `bootstrap-peer-store.ts`), because the
|
|
284
459
|
* targets must outlive the process: a node seeded into a party it could not
|
|
285
460
|
* reach has nothing else on disk naming that party's addresses (`applySeed`
|
|
286
461
|
* writes no control row, and `CadrePeer` fills in only after a connection
|
|
287
462
|
* succeeds), so an in-memory-only set left it stranded permanently across a
|
|
288
|
-
* restart.
|
|
463
|
+
* restart. An owner that added a node it cannot be dialed by is in the same
|
|
464
|
+
* position: the added node's row stays unsigned, so unresolvable, until that
|
|
465
|
+
* node self-publishes over a connection only the owner can open. Constructed (or
|
|
466
|
+
* adopted from `config.bootstrapPeers.store`) by
|
|
289
467
|
* {@link initializeBootstrapPeerStore} during {@link start}; deliberately NOT
|
|
290
468
|
* cleared by {@link cleanup}, so it survives a stop()→start() cycle of the same
|
|
291
469
|
* node instance — same lifecycle as {@link trustedOwnerStore}. Durability
|
|
292
|
-
* depends on the injected backend
|
|
293
|
-
* RN/browser stay ephemeral for now.
|
|
470
|
+
* depends on the injected backend; the default is in-memory.
|
|
294
471
|
*
|
|
295
472
|
* Deliberately NOT the libp2p peer store, which {@link peerStoreAddrs} already
|
|
296
473
|
* reads for the steady-state path. The peer store is shared with everything
|
|
297
474
|
* libp2p discovers, so "dial every entry" would grow into dialing arbitrary
|
|
298
475
|
* discovered peers as the node lives longer; this store holds exactly the peers
|
|
299
|
-
* an owner-signed, trust-anchored seed nominated as owners
|
|
300
|
-
*
|
|
301
|
-
*
|
|
476
|
+
* an owner-signed, trust-anchored seed nominated as owners, and the nodes this
|
|
477
|
+
* node chose to add. `applySeed` also merges a seed's addresses into the peer
|
|
478
|
+
* store, where they age out; an added node's addresses are never merged there,
|
|
479
|
+
* because only verified addresses go into the address book (see
|
|
480
|
+
* {@link warmSiblingAddrBook}).
|
|
302
481
|
*
|
|
303
|
-
* A later
|
|
482
|
+
* A later record OVERWRITES an entry rather than merging, so a re-seed after an
|
|
304
483
|
* owner's address changes replaces the stale address instead of accumulating
|
|
305
|
-
* dead ones
|
|
306
|
-
* the
|
|
484
|
+
* dead ones, and {@link warmSiblingAddrBook} replaces an entry with the addresses
|
|
485
|
+
* the sibling's signed record resolves to when they differ. Only this node's own
|
|
486
|
+
* {@link removePeer} evicts an entry: otherwise they are the node's only way back
|
|
487
|
+
* to those peers if it is ever stranded again.
|
|
307
488
|
*/
|
|
308
489
|
this.bootstrapPeerStore = null;
|
|
490
|
+
/**
|
|
491
|
+
* Node-local record of the party's enrolled-machine count (see
|
|
492
|
+
* `enrolled-machine-store.ts`), kept so the CONTROL node can declare a
|
|
493
|
+
* block-repair yardstick at bring-up. Constructed (or adopted from
|
|
494
|
+
* `config.enrolledMachines.store`) by {@link initializeEnrolledMachineStore};
|
|
495
|
+
* written by {@link refreshAuthorizedControlPeers}. Like
|
|
496
|
+
* {@link bootstrapPeerStore} it is deliberately NOT cleared by {@link cleanup},
|
|
497
|
+
* so it survives a stop()→start() cycle of the same node instance.
|
|
498
|
+
*/
|
|
499
|
+
this.enrolledMachineStore = null;
|
|
309
500
|
/** Initial self-registration timer (see {@link scheduleSelfRegistration}). */
|
|
310
501
|
this.selfRegistrationTimer = null;
|
|
311
502
|
/** TTL heartbeat that re-publishes the self record before it goes stale. */
|
|
@@ -333,6 +524,19 @@ export class CadreNode {
|
|
|
333
524
|
* both read "no row yet" and race a duplicate INSERT (a `CadrePeer` PK conflict).
|
|
334
525
|
*/
|
|
335
526
|
this.registerSelfInFlight = null;
|
|
527
|
+
/**
|
|
528
|
+
* When this node last PUBLISHED its own `CadrePeer` address record (inserted or
|
|
529
|
+
* refreshed), or null if it never has in this session. Stamped by
|
|
530
|
+
* {@link noteSelfRecordPublished} on every successful publish, wherever it was driven
|
|
531
|
+
* from — the boot pass, the heartbeat, an address change, a drain or an explicit call.
|
|
532
|
+
*
|
|
533
|
+
* Read by {@link escalateIfSelfRecordStale} for the one consequence worth an operator's
|
|
534
|
+
* attention: once this gap exceeds {@link DEFAULT_PEER_RECORD_MAX_AGE_MS}, every other
|
|
535
|
+
* machine in the party is already discarding this node's address as stale.
|
|
536
|
+
*/
|
|
537
|
+
this.lastSelfRecordPublishAt = null;
|
|
538
|
+
/** Say-once latch for {@link escalateIfSelfRecordStale}; re-armed by the next publish. */
|
|
539
|
+
this.selfRecordStaleWarned = false;
|
|
336
540
|
// ── Write-while-alone re-replication (control-write-ensure-replicated) ──────
|
|
337
541
|
/**
|
|
338
542
|
* Re-replication queue for owner `authorizePeer` writes that committed while
|
|
@@ -367,6 +571,14 @@ export class CadreNode {
|
|
|
367
571
|
* exactly a "removals from before this lifetime" case).
|
|
368
572
|
*/
|
|
369
573
|
this.reissuedHeldRevocations = false;
|
|
574
|
+
/**
|
|
575
|
+
* Whether this process has seen the singleton `Revocation` ledger marker filed — its own
|
|
576
|
+
* `'opened'` or an `'already-open'` answer, in {@link openRevocationLedgerIfDue}. A
|
|
577
|
+
* record of work done, not a cached authorization answer: the marker can never be
|
|
578
|
+
* deleted (`NoDelete`), so this can only go stale if the database itself is replaced,
|
|
579
|
+
* which is why stop() clears it with the other per-process replication state.
|
|
580
|
+
*/
|
|
581
|
+
this.revocationLedgerOpened = false;
|
|
370
582
|
/** This node's own `CadrePeer` self-write committed local-only (re-touched on growth). */
|
|
371
583
|
this.pendingSelfPeerWrite = false;
|
|
372
584
|
/** This node's own `DeviceToken` self-write committed local-only (re-touched on growth). */
|
|
@@ -502,6 +714,25 @@ export class CadreNode {
|
|
|
502
714
|
getStrands() {
|
|
503
715
|
return this.strandManager.getInstances();
|
|
504
716
|
}
|
|
717
|
+
/**
|
|
718
|
+
* Strands the control network advertises that no local sAppConfig claims —
|
|
719
|
+
* the backlog behind `strand:discovered`.
|
|
720
|
+
*
|
|
721
|
+
* The event fires once per strand, and it can fire before the app has
|
|
722
|
+
* subscribed (the watcher's first poll runs inside `start()`). So an app that
|
|
723
|
+
* auto-joins discovered strands must subscribe FIRST and then drain this map,
|
|
724
|
+
* not rely on the event alone. Entries leave the map when the strand is
|
|
725
|
+
* claimed ({@link addStrand}) or its control row disappears.
|
|
726
|
+
*
|
|
727
|
+
* Bounded by the number of strands this party has that this node does not run
|
|
728
|
+
* — the control database holds only this party's rows — so there is no cap and
|
|
729
|
+
* no eviction policy to reason about.
|
|
730
|
+
*
|
|
731
|
+
* Returns a snapshot; mutating it does not affect the node.
|
|
732
|
+
*/
|
|
733
|
+
getDiscoveredStrands() {
|
|
734
|
+
return new Map(this.discoveredStrands);
|
|
735
|
+
}
|
|
505
736
|
/**
|
|
506
737
|
* Get a specific strand instance
|
|
507
738
|
*/
|
|
@@ -547,6 +778,13 @@ export class CadreNode {
|
|
|
547
778
|
// closed before any network bring-up, and the retained dial targets are
|
|
548
779
|
// loaded before the first reconcile pass could consult them.
|
|
549
780
|
this.initializeBootstrapPeerStore();
|
|
781
|
+
// Read the party's enrolled-machine count out of its node-local record and
|
|
782
|
+
// capture it for buildControlNodeOptions below. This MUST precede
|
|
783
|
+
// createControlNode: Optimystic freezes the cluster policy when the node is
|
|
784
|
+
// built, and the ControlDatabase that could answer the question live does not
|
|
785
|
+
// exist until after that. Remembering the number across the restart is the
|
|
786
|
+
// only way to declare it at all — see `enrolled-machine-store.ts`.
|
|
787
|
+
this.initializeEnrolledMachineStore();
|
|
550
788
|
// Arm the connection gate's BRING-UP QUIET PERIOD before the libp2p node
|
|
551
789
|
// exists, so no connection can form ahead of it: `@libp2p/bootstrap` emits
|
|
552
790
|
// its discovery events ~1 s after `libp2p.start()` and the connection
|
|
@@ -610,6 +848,9 @@ export class CadreNode {
|
|
|
610
848
|
// Committed-delete seam → write-while-alone revocation queue. Same wiring
|
|
611
849
|
// window argument as the membership listener above.
|
|
612
850
|
this.controlDatabase.setGuardedDeleteListener((revocation) => this.noteGuardedDelete(revocation));
|
|
851
|
+
// Abandoned-write seam. Wired here, before the first control write can run, so a
|
|
852
|
+
// write lost during the rest of bring-up is reported like any other.
|
|
853
|
+
this.controlDatabase.setControlWriteAbandonedListener((abandonment) => this.noteControlWriteAbandoned(abandonment));
|
|
613
854
|
// Create strand queryable using the control database
|
|
614
855
|
const queryable = this.createStrandQueryable();
|
|
615
856
|
// Create and start the strand watcher with sAppId lookup
|
|
@@ -660,6 +901,12 @@ export class CadreNode {
|
|
|
660
901
|
// delayed refresh path) so no early `connection:open` is missed — the
|
|
661
902
|
// write-while-alone re-replication drain must fire on the first 0→≥1 edge.
|
|
662
903
|
this.wireControlConnectionListeners();
|
|
904
|
+
// Arm the control network's peer-join block catch-up now that the control
|
|
905
|
+
// DB (which the push-time membership gate reads) is up. Armed even when
|
|
906
|
+
// this node is alone — like the strand one, it only does work when the
|
|
907
|
+
// control libp2p node reports a peer connection, and arming it at start is
|
|
908
|
+
// what closes the "genesised alone, headers never replicate" hole.
|
|
909
|
+
this.startControlBackfill();
|
|
663
910
|
// Seed the per-stream gate's materialized authorized set now that the
|
|
664
911
|
// control DB is up (non-blocking; the helper never rejects). Until it
|
|
665
912
|
// lands the snapshot is empty, which the stream gate treats as cold
|
|
@@ -670,7 +917,7 @@ export class CadreNode {
|
|
|
670
917
|
// scheduleSelfRegistration, so this node's first published `CadrePeer` row
|
|
671
918
|
// already carries the `/p2p-circuit` address the reservation earns it.
|
|
672
919
|
// Throws on a relay that will not have us — start() is fail-fast for a node
|
|
673
|
-
// whose operator named a relay.
|
|
920
|
+
// whose operator named a relay, unless network.requireRelay is false.
|
|
674
921
|
await this.driveControlRelayReservation();
|
|
675
922
|
// Schedule self-registration in background
|
|
676
923
|
this.scheduleSelfRegistration();
|
|
@@ -847,10 +1094,50 @@ export class CadreNode {
|
|
|
847
1094
|
this.bootstrapPeerStore = store;
|
|
848
1095
|
}
|
|
849
1096
|
/**
|
|
850
|
-
*
|
|
851
|
-
*
|
|
852
|
-
*
|
|
853
|
-
*
|
|
1097
|
+
* Construct (or adopt) the node-local enrolled-machine store and capture the
|
|
1098
|
+
* count this run will declare into {@link declaredEnrolledMachines}.
|
|
1099
|
+
*
|
|
1100
|
+
* Synchronous for the same reason {@link initializeBootstrapPeerStore} is: an
|
|
1101
|
+
* injected store has already loaded its persisted count by the time it is handed
|
|
1102
|
+
* in (its `open` is the async part), and the in-memory default has nothing to
|
|
1103
|
+
* load. Must run BEFORE {@link createControlNode} — that is the whole point of
|
|
1104
|
+
* the record; see `enrolled-machine-store.ts`.
|
|
1105
|
+
*
|
|
1106
|
+
* The store instance is kept across stop()→start() (so a count recorded during
|
|
1107
|
+
* the previous run is not lost with the ephemeral default), but the DECLARED
|
|
1108
|
+
* count is re-read every time — which is how a restart applies a number the
|
|
1109
|
+
* previous run learned.
|
|
1110
|
+
*
|
|
1111
|
+
* A store scoped to a different party is a configuration error, fail closed. It
|
|
1112
|
+
* is only a repair hint, but a hint sized by a foreign party's membership is a
|
|
1113
|
+
* number nobody chose, and the mismatch always means a miswired embedder.
|
|
1114
|
+
*/
|
|
1115
|
+
initializeEnrolledMachineStore() {
|
|
1116
|
+
const partyId = this.config.controlNetwork.partyId;
|
|
1117
|
+
const store = this.enrolledMachineStore
|
|
1118
|
+
?? this.config.enrolledMachines?.store
|
|
1119
|
+
?? new MemoryEnrolledMachineStore(partyId);
|
|
1120
|
+
if (store.partyId !== partyId) {
|
|
1121
|
+
throw new Error(`CadreNodeConfig: enrolledMachines.store is scoped to party ${store.partyId}, ` +
|
|
1122
|
+
`but this node serves party ${partyId} — refusing to size a repair yardstick from a foreign party`);
|
|
1123
|
+
}
|
|
1124
|
+
this.enrolledMachineStore = store;
|
|
1125
|
+
this.declaredEnrolledMachines = store.count();
|
|
1126
|
+
log('control repair yardstick will be declared from %o enrolled machine(s)', this.declaredEnrolledMachines);
|
|
1127
|
+
}
|
|
1128
|
+
/**
|
|
1129
|
+
* The node-local enrolled-machine store (null before {@link start}) — what this
|
|
1130
|
+
* node last knew about its party's size. Exposed for diagnostics and for a host
|
|
1131
|
+
* that wants to show which repair yardstick the next launch will declare.
|
|
1132
|
+
*/
|
|
1133
|
+
getEnrolledMachineStore() {
|
|
1134
|
+
return this.enrolledMachineStore;
|
|
1135
|
+
}
|
|
1136
|
+
/**
|
|
1137
|
+
* The node-local bootstrap-peer store (null before {@link start}) — the dial
|
|
1138
|
+
* targets learned out of band that {@link reconcileControlCohort} falls back to.
|
|
1139
|
+
* Exposed for diagnostics and for a host that wants to show "what would this
|
|
1140
|
+
* node dial if it is stranded?".
|
|
854
1141
|
*/
|
|
855
1142
|
getBootstrapPeerStore() {
|
|
856
1143
|
return this.bootstrapPeerStore;
|
|
@@ -892,6 +1179,90 @@ export class CadreNode {
|
|
|
892
1179
|
async createControlNode() {
|
|
893
1180
|
return await createLibp2pNode(this.buildControlNodeOptions());
|
|
894
1181
|
}
|
|
1182
|
+
/**
|
|
1183
|
+
* The control libp2p node's Optimystic network name — ONE binding for every
|
|
1184
|
+
* derivation from it (`db-p2p` namespaces all of the node's protocol ids as
|
|
1185
|
+
* `/optimystic/<networkName>/...`), so the node options and the block-transfer
|
|
1186
|
+
* protocol prefix the control backfill dials can never drift apart.
|
|
1187
|
+
*/
|
|
1188
|
+
controlNetworkName() {
|
|
1189
|
+
return `control-${this.config.controlNetwork.partyId}`;
|
|
1190
|
+
}
|
|
1191
|
+
/**
|
|
1192
|
+
* Arm the CONTROL network's peer-join block catch-up: push every block in the
|
|
1193
|
+
* control database's own raw store to each newly connected AUTHORIZED member
|
|
1194
|
+
* this runtime has not yet caught up. This is what physically replicates
|
|
1195
|
+
* control blocks committed while the writer was alone — the named
|
|
1196
|
+
* collection-header blocks written once at genesis above all, whose revision
|
|
1197
|
+
* never moves again, so no later commit ever carries them to a member that
|
|
1198
|
+
* joined after them. Without it, such a member that restarts offline reads
|
|
1199
|
+
* the affected control tables as EMPTY, silently (`isMember()` answers false
|
|
1200
|
+
* for peers it knew about before the restart).
|
|
1201
|
+
*
|
|
1202
|
+
* Unlike the per-strand instances (see `strand-instance-manager.ts`), pushes
|
|
1203
|
+
* are gated on {@link isAuthorizedMember}, judged at push time: the control
|
|
1204
|
+
* network's inbound connection gate deliberately admits non-members in
|
|
1205
|
+
* several states (seed delivery to an un-enrolled node, an open enrollment
|
|
1206
|
+
* window, an outstanding invitation, configured bootstrap/relay peers), and
|
|
1207
|
+
* pushing the whole control store to such a peer would hand a stranger the
|
|
1208
|
+
* party's entire membership, addresses and strand list. The receiving side
|
|
1209
|
+
* needs no work of its own: `createLibp2pNode` registers the block-transfer
|
|
1210
|
+
* handler on every node it builds, control node included, and this node's
|
|
1211
|
+
* per-stream gate (`authorizeInboundControlStream`) covers the inbound
|
|
1212
|
+
* direction. A denied peer is retried on its next `connection:open`, and —
|
|
1213
|
+
* because the production join order is connect-then-authorize — on every
|
|
1214
|
+
* committed membership change via {@link refreshAuthorizedControlPeers}'s
|
|
1215
|
+
* `scheduleConnectedPeers()` call.
|
|
1216
|
+
*
|
|
1217
|
+
* No-ops (logged) when the embedder configured no control storage or the
|
|
1218
|
+
* node exposes no key network — the backfill would have nothing to read or
|
|
1219
|
+
* no way to dial.
|
|
1220
|
+
*/
|
|
1221
|
+
startControlBackfill() {
|
|
1222
|
+
if (this.config.controlBackfill?.enabled === false || !this.controlNode) {
|
|
1223
|
+
return;
|
|
1224
|
+
}
|
|
1225
|
+
// Through the resolver, not the {@link controlStorage} memo field: the resolver is
|
|
1226
|
+
// idempotent (it returns the same wrapped store `buildControlNodeOptions` already
|
|
1227
|
+
// resolved), so the catch-up cannot be silently disarmed by a future reordering that
|
|
1228
|
+
// moves this call ahead of the one that populated the field.
|
|
1229
|
+
const storage = this.resolveControlStorage();
|
|
1230
|
+
const keyNetwork = this.controlNode.keyNetwork;
|
|
1231
|
+
if (!storage || !keyNetwork) {
|
|
1232
|
+
log('Control peer-join block catch-up is inert: %s', !storage ? 'no control storage configured' : 'control node exposes no keyNetwork');
|
|
1233
|
+
return;
|
|
1234
|
+
}
|
|
1235
|
+
const networkName = this.controlNetworkName();
|
|
1236
|
+
this.controlBackfill = new PeerJoinBackfill({
|
|
1237
|
+
label: networkName,
|
|
1238
|
+
libp2p: this.controlNode,
|
|
1239
|
+
peerNetwork: keyNetwork,
|
|
1240
|
+
storage,
|
|
1241
|
+
// The same prefix the receiver registered its block-transfer handler
|
|
1242
|
+
// under — derived from the same networkName binding the node options
|
|
1243
|
+
// used, never re-spelled here.
|
|
1244
|
+
protocolPrefix: `/optimystic/${networkName}`,
|
|
1245
|
+
authorizePeer: (peerId) => this.isAuthorizedMember(peerId)
|
|
1246
|
+
}, {
|
|
1247
|
+
// A shorter settle window than the strand default (1000 ms): the control
|
|
1248
|
+
// store is small (a party's membership — dozens of blocks), a re-push is
|
|
1249
|
+
// idempotent, and the window a joiner is connected before going offline
|
|
1250
|
+
// can be short (converge one row, then stop) — a slow debounce is the
|
|
1251
|
+
// difference between that joiner holding the collection headers and
|
|
1252
|
+
// reading its own membership as empty after an offline restart.
|
|
1253
|
+
//
|
|
1254
|
+
// NOTE: 250 is reasoned, not swept — the delete-while-alone scenario's
|
|
1255
|
+
// phases 1-3 finish in about a second, so the 1000 ms strand default left
|
|
1256
|
+
// the push racing the joiner's stop, and 250 gives margin. Nobody measured
|
|
1257
|
+
// the shape of the curve either side of it. If the control gates start
|
|
1258
|
+
// flaking on a joiner that stopped before being caught up, lower this
|
|
1259
|
+
// before looking anywhere else; if catch-up pushes start showing up as
|
|
1260
|
+
// connection-churn noise, raise it.
|
|
1261
|
+
debounceMs: 250,
|
|
1262
|
+
...this.config.controlBackfill
|
|
1263
|
+
});
|
|
1264
|
+
this.controlBackfill.start();
|
|
1265
|
+
}
|
|
895
1266
|
/**
|
|
896
1267
|
* The control database's cache-wrapped raw storage, resolved from
|
|
897
1268
|
* `config.storage.provider` on first use and then held in {@link controlStorage}
|
|
@@ -918,8 +1289,12 @@ export class CadreNode {
|
|
|
918
1289
|
if (!provider) {
|
|
919
1290
|
return undefined;
|
|
920
1291
|
}
|
|
921
|
-
|
|
922
|
-
|
|
1292
|
+
// Party-scoped, not the bare literal 'control': the control database holds THIS
|
|
1293
|
+
// party's records, so two parties on one device must not land in one store.
|
|
1294
|
+
// Same key as the cache label, which also makes the shared pool's `stats()` readable.
|
|
1295
|
+
const scope = controlStorageScope(this.config.controlNetwork.partyId);
|
|
1296
|
+
const resolved = typeof provider === 'function' ? provider(scope) : provider;
|
|
1297
|
+
this.controlStorage = wrapStorageWithCache(resolved, scope);
|
|
923
1298
|
return this.controlStorage;
|
|
924
1299
|
}
|
|
925
1300
|
/**
|
|
@@ -933,9 +1308,8 @@ export class CadreNode {
|
|
|
933
1308
|
* operator whose relay slot is decorative may genuinely want only the announced
|
|
934
1309
|
* address, so this reports and proceeds.
|
|
935
1310
|
*
|
|
936
|
-
* Keyed off
|
|
937
|
-
*
|
|
938
|
-
* longer route — is caught too.
|
|
1311
|
+
* Keyed off `relayAddrs` AND `listenAddrs`, so a hand-written `/p2p-circuit` entry
|
|
1312
|
+
* in `listenAddrs` — a reservation by the longer route — is caught too.
|
|
939
1313
|
*
|
|
940
1314
|
* NOTE: the only direct `console.*` in this library — a boot-time operator warning, not a
|
|
941
1315
|
* diagnostic trace (those use `debug`, which an operator never sees without `DEBUG=`). If a
|
|
@@ -947,9 +1321,13 @@ export class CadreNode {
|
|
|
947
1321
|
if (!replacesAdvertisedAddrs(network)) {
|
|
948
1322
|
return;
|
|
949
1323
|
}
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
1324
|
+
// Read off the raw config, not the resolved listen set: the question is only
|
|
1325
|
+
// "does this config name a relay", and the resolution throws on a hand-written
|
|
1326
|
+
// `<relay>/p2p-circuit` listen entry — which still warrants this warning, and
|
|
1327
|
+
// gets its own refusal a few lines later in `buildControlNodeOptions`.
|
|
1328
|
+
const namesRelay = (network?.relayAddrs?.length ?? 0) > 0
|
|
1329
|
+
|| (network?.listenAddrs ?? []).some((addr) => addr.includes('/p2p-circuit'));
|
|
1330
|
+
if (!namesRelay) {
|
|
953
1331
|
return;
|
|
954
1332
|
}
|
|
955
1333
|
console.warn('network.announceAddrs is set alongside a circuit-relay listener (from network.relayAddrs, ' +
|
|
@@ -969,11 +1347,16 @@ export class CadreNode {
|
|
|
969
1347
|
buildControlNodeOptions() {
|
|
970
1348
|
const { controlNetwork, network, profile } = this.config;
|
|
971
1349
|
const identityKey = this.identityKey;
|
|
972
|
-
// `
|
|
973
|
-
//
|
|
974
|
-
//
|
|
975
|
-
//
|
|
976
|
-
const listenAddrs = resolveListenAddrs(network
|
|
1350
|
+
// `network.relayAddrs` contributes the bare `/p2p-circuit` SEARCH entry, which
|
|
1351
|
+
// opens no connection — the reservation is driven explicitly at the END of
|
|
1352
|
+
// `start()`, so the control database is built while this node holds zero
|
|
1353
|
+
// control connections. See `relay-addrs.ts` and `start()`.
|
|
1354
|
+
const listenAddrs = resolveListenAddrs(network);
|
|
1355
|
+
// The transports those listen entries imply. A `/ws` entry switches db-p2p's
|
|
1356
|
+
// WebSocket transport on (without it the entry binds nothing and libp2p reports
|
|
1357
|
+
// nothing); anything outside {tcp, ws/wss, p2p-circuit} throws here rather than
|
|
1358
|
+
// being silently dropped at bring-up. See `relay-addrs.ts`.
|
|
1359
|
+
const transportOptions = resolveTransportOptions(network, listenAddrs);
|
|
977
1360
|
const enableRelay = this.relayServerEnabled();
|
|
978
1361
|
// The control node's own storage, resolved once per runtime (see
|
|
979
1362
|
// {@link resolveControlStorage}) rather than per call.
|
|
@@ -981,7 +1364,7 @@ export class CadreNode {
|
|
|
981
1364
|
const nodeOptions = {
|
|
982
1365
|
port: 0,
|
|
983
1366
|
bootstrapNodes: controlNetwork.bootstrapNodes,
|
|
984
|
-
networkName:
|
|
1367
|
+
networkName: this.controlNetworkName(),
|
|
985
1368
|
storage: controlStorageProvider,
|
|
986
1369
|
fretProfile: profile === 'storage' ? 'core' : 'edge',
|
|
987
1370
|
relay: enableRelay,
|
|
@@ -992,10 +1375,22 @@ export class CadreNode {
|
|
|
992
1375
|
// CONTROL_CLUSTER_POLICY: only the admission gate defaults to 2, while the read-repair
|
|
993
1376
|
// corroboration floor would otherwise fall back to this clusterSize of 16.
|
|
994
1377
|
clusterSize: CONTROL_REPLICATION_BREADTH,
|
|
995
|
-
|
|
1378
|
+
// The base control policy with the block-repair corroboration yardstick declared
|
|
1379
|
+
// from the machines this party had enrolled at this node's last look
|
|
1380
|
+
// ({@link declaredEnrolledMachines}, read in start() before this runs). Handed
|
|
1381
|
+
// `undefined` — a brand-new node, an unreadable record, or an embedder that
|
|
1382
|
+
// injected no store — this returns the frozen CONTROL_CLUSTER_POLICY object
|
|
1383
|
+
// itself, so the unknown case is provably byte-for-byte the old behaviour.
|
|
1384
|
+
// The yardstick moves ALONE: `assumedClusterSize` stays pinned at 2, because a
|
|
1385
|
+
// party of phones cannot promise three quarters of its machines are awake.
|
|
1386
|
+
clusterPolicy: controlClusterPolicy(this.declaredEnrolledMachines),
|
|
996
1387
|
arachnode: { enableRingZulu: profile === 'storage' },
|
|
997
1388
|
...(identityKey && { privateKey: identityKey }),
|
|
998
1389
|
...(network?.transports && { transports: network.transports }),
|
|
1390
|
+
// `{ wsPort }` when a listen entry names WebSocket, otherwise `{}` — and always
|
|
1391
|
+
// `{}` when `network.transports` is set, since the embedder owns transport policy
|
|
1392
|
+
// then. Spread NEXT to `transports` because the two answer the same question.
|
|
1393
|
+
...transportOptions,
|
|
999
1394
|
// Configured `listenAddrs`, plus the bare `/p2p-circuit` search listener when
|
|
1000
1395
|
// `network.relayAddrs` is set — that listener registers the pending reservation
|
|
1001
1396
|
// `driveControlRelayReservation` fills after bring-up.
|
|
@@ -1325,6 +1720,68 @@ export class CadreNode {
|
|
|
1325
1720
|
const members = await this.listAuthorizedMembers(false);
|
|
1326
1721
|
this.authorizedControlPeers = new Set(members.map((m) => m.peerId));
|
|
1327
1722
|
log('refreshAuthorizedControlPeers(%s): %d authorized peer(s)', reason, this.authorizedControlPeers.size);
|
|
1723
|
+
// Remember the party's size for the NEXT launch's control-node repair yardstick
|
|
1724
|
+
// (see `enrolled-machine-store.ts`): this is the only place the count is known,
|
|
1725
|
+
// and the control node that needs it was built long before this ran. Recorded
|
|
1726
|
+
// from the snapshot just materialized — no second membership query.
|
|
1727
|
+
//
|
|
1728
|
+
// CONTROL-NETWORK ONLY, and that scope is load-bearing. Every enrolled machine
|
|
1729
|
+
// runs the control node by construction, so here the party's machine count IS
|
|
1730
|
+
// the count of machines serving the network — the quantity
|
|
1731
|
+
// `resolveRepairYardstick` asks for. It is NOT that quantity for a strand, which
|
|
1732
|
+
// launches only on machines whose embedder registered its sApp config (see
|
|
1733
|
+
// {@link addStrand}), so this number must never be routed to a strand node:
|
|
1734
|
+
// over-declaring pins Optimystic's repair corroboration floor at two peers and a
|
|
1735
|
+
// strand that can field only one can then never repair. `launchStrand` therefore
|
|
1736
|
+
// declares nothing; see the NOTE there.
|
|
1737
|
+
//
|
|
1738
|
+
// `+ 1` for this node, which is not in its own authorized set. An EMPTY snapshot
|
|
1739
|
+
// is recorded as 1 rather than skipped. That is deliberate, and the reason
|
|
1740
|
+
// is a party that genuinely SHRANK: if empty meant "record nothing", a party
|
|
1741
|
+
// whose other machines were all revoked would keep declaring its old, larger
|
|
1742
|
+
// number forever, and over-declaring is the unsafe direction — at a yardstick of
|
|
1743
|
+
// 3 or more Optimystic pins the repair corroboration floor at two peers, so a
|
|
1744
|
+
// cohort that can only field one peer could never repair at all. Recording 1
|
|
1745
|
+
// lets the number come back down; the floor then turns it into 2, which is
|
|
1746
|
+
// exactly what declaring nothing resolves to.
|
|
1747
|
+
//
|
|
1748
|
+
// NOTE: the cost of that choice is that a TRANSIENT empty read overwrites a good
|
|
1749
|
+
// remembered count with 1 — a membership snapshot is legitimately empty early in
|
|
1750
|
+
// a run on a node whose rows have not replicated yet or whose trusted-owner
|
|
1751
|
+
// anchor is still unseeded (`listAuthorizedMembers` authorizes no one without an
|
|
1752
|
+
// anchor). Bounded and safe: the next refresh once rows land re-records the real
|
|
1753
|
+
// count, and the only node that loses anything is one stopped inside that window,
|
|
1754
|
+
// which then declares 2 — today's behaviour — on its next launch. If a node ever
|
|
1755
|
+
// needs to declare the right number on its FIRST post-restart launch, the fix is
|
|
1756
|
+
// to distinguish "no rows yet" from "no members" at this site, not to skip empty.
|
|
1757
|
+
//
|
|
1758
|
+
// A removed peer stops counting at REVOCATION, not at reap: `queryCadrePeers`
|
|
1759
|
+
// drops rows whose `StampId` is retired, and `listAuthorizedMembers` reads
|
|
1760
|
+
// through it. That is why there is no reap hook writing this record.
|
|
1761
|
+
//
|
|
1762
|
+
// `void`: the store never rejects and logs its own persist failures, and this
|
|
1763
|
+
// method's contract is never-rejects. A failed write leaves the in-memory count
|
|
1764
|
+
// correct for this session and re-lands on the next refresh.
|
|
1765
|
+
void this.enrolledMachineStore?.record(this.authorizedControlPeers.size + 1);
|
|
1766
|
+
// Membership moved: give the control backfill another look at every
|
|
1767
|
+
// connected peer. The production join order is connect-then-authorize, so
|
|
1768
|
+
// a joiner's first catch-up pass was denied at the gate while its
|
|
1769
|
+
// connection stays up — without this re-arm it would wait for a
|
|
1770
|
+
// reconnect. Cheap and idempotent: caught-up peers are skipped before any
|
|
1771
|
+
// timer is set, a peer whose pass is in flight is deferred to the end of
|
|
1772
|
+
// that pass (the pass may already be past its gate check — this very
|
|
1773
|
+
// commit is what would authorize it), and the gate re-judges at push time.
|
|
1774
|
+
//
|
|
1775
|
+
// NOTE: every refresh re-arms every connected peer that is not yet caught
|
|
1776
|
+
// up, and each re-armed pass costs one `CadrePeer` query at the gate. A
|
|
1777
|
+
// connection that is admitted but never authorized — a configured
|
|
1778
|
+
// bootstrap or relay peer, the steady state for a NAT'd node — is
|
|
1779
|
+
// therefore re-judged on every membership write and every timed reconcile
|
|
1780
|
+
// tick, forever. Negligible at a cadre's handful of connections; if a node
|
|
1781
|
+
// ever holds many such connections, or refreshes get frequent, skip
|
|
1782
|
+
// re-arming peers whose last pass was denied under the SAME membership
|
|
1783
|
+
// snapshot instead of re-arming unconditionally.
|
|
1784
|
+
this.controlBackfill?.scheduleConnectedPeers();
|
|
1328
1785
|
}
|
|
1329
1786
|
catch (error) {
|
|
1330
1787
|
log('refreshAuthorizedControlPeers(%s) failed — keeping previous snapshot: %o', reason, error);
|
|
@@ -1468,6 +1925,7 @@ export class CadreNode {
|
|
|
1468
1925
|
if (await this.seedBootstrapService.insertSelfPeerRecord(record)) {
|
|
1469
1926
|
if (this.committedAlone())
|
|
1470
1927
|
this.pendingSelfPeerWrite = true;
|
|
1928
|
+
this.noteSelfRecordPublished();
|
|
1471
1929
|
log('registerSelf: inserted own CadrePeer record (owner-signed, updatedAt=%d, %d addrs, sig=%s…)', record.updatedAt, addrs.length, record.sig.slice(0, 16));
|
|
1472
1930
|
return 'inserted';
|
|
1473
1931
|
}
|
|
@@ -1505,9 +1963,81 @@ export class CadreNode {
|
|
|
1505
1963
|
await this.controlDatabase.updateSelfPeerRecord(record);
|
|
1506
1964
|
if (this.committedAlone())
|
|
1507
1965
|
this.pendingSelfPeerWrite = true;
|
|
1966
|
+
this.noteSelfRecordPublished();
|
|
1508
1967
|
log('registerSelf: refreshed own CadrePeer record (updatedAt=%d, %d addrs, sig=%s…)', record.updatedAt, addrs.length, record.sig.slice(0, 16));
|
|
1509
1968
|
return 'refreshed';
|
|
1510
1969
|
}
|
|
1970
|
+
/**
|
|
1971
|
+
* Record that this node's own address record just landed, and re-arm the stale-record
|
|
1972
|
+
* warning. Called from BOTH publishing paths of {@link publishSelfRecord} and nowhere
|
|
1973
|
+
* else — a `skipped` publish wrote nothing, so it must not refresh the stamp.
|
|
1974
|
+
*/
|
|
1975
|
+
noteSelfRecordPublished() {
|
|
1976
|
+
this.lastSelfRecordPublishAt = Date.now();
|
|
1977
|
+
this.selfRecordStaleWarned = false;
|
|
1978
|
+
}
|
|
1979
|
+
/**
|
|
1980
|
+
* Report an abandoned control write to the embedding app.
|
|
1981
|
+
*
|
|
1982
|
+
* Every control write funnels through `ControlDatabase.lockedWithRetry`, which now tells
|
|
1983
|
+
* this node when it gives one up. Foreground writes ALSO reject to their caller; the
|
|
1984
|
+
* background ones ({@link startRecordRefresh}'s republish and the two replication
|
|
1985
|
+
* drains) are fired unawaited with a `debug`-only catch, so this event is the only thing
|
|
1986
|
+
* that reaches an app at all.
|
|
1987
|
+
*
|
|
1988
|
+
* Reporting only — the one operator-visible escalation lives on the republish path
|
|
1989
|
+
* ({@link escalateIfSelfRecordStale}), where the consequence is measurable.
|
|
1990
|
+
*/
|
|
1991
|
+
noteControlWriteAbandoned(abandonment) {
|
|
1992
|
+
log('Control write abandoned [%s] after %d/%d attempt(s) in %dms (%s): %o', abandonment.label ?? 'unlabelled', abandonment.attemptsMade, abandonment.attemptsAllowed, abandonment.elapsedMs, abandonment.reason, abandonment.error);
|
|
1993
|
+
this.emit('control:write-abandoned', abandonment);
|
|
1994
|
+
}
|
|
1995
|
+
/**
|
|
1996
|
+
* Escalate ONE failed self-address republish to the operator — once — when the node has
|
|
1997
|
+
* not published its own record for longer than a record stays fresh.
|
|
1998
|
+
*
|
|
1999
|
+
* The bar is the CONSEQUENCE, not a failure count: a resolver discards a `CadrePeer`
|
|
2000
|
+
* record older than {@link DEFAULT_PEER_RECORD_MAX_AGE_MS} (15 minutes) and the heartbeat
|
|
2001
|
+
* re-stamps at half that, so by the time this gap opens every other machine in the party
|
|
2002
|
+
* has already stopped accepting this node's address. A count would fire far too early —
|
|
2003
|
+
* one failed heartbeat is already half the budget, and a single miss costs nothing.
|
|
2004
|
+
*
|
|
2005
|
+
* Say-once, re-armed by the next successful publish ({@link noteSelfRecordPublished}), so
|
|
2006
|
+
* a node that stays broken warns once rather than every 7.5 minutes.
|
|
2007
|
+
*
|
|
2008
|
+
* A node that has never published in this session is skipped: it has no record out there
|
|
2009
|
+
* to go stale, and the reason it has none (not a member yet, no signing key, revoked) is
|
|
2010
|
+
* already logged by `registerSelf` itself.
|
|
2011
|
+
*
|
|
2012
|
+
* Only a FAILED republish is checked. A heartbeat whose publish reports `skipped` neither
|
|
2013
|
+
* stamps nor warns, so a node that stops being able to publish without erroring — revoked,
|
|
2014
|
+
* or its row removed — goes quiet here. That is the intended reading: a revoked node has
|
|
2015
|
+
* nothing to publish and its unreachability is the point, not a fault to report.
|
|
2016
|
+
*
|
|
2017
|
+
* NOTE: the second `console.*` in this library, and the reason
|
|
2018
|
+
* {@link warnIfAnnounceAddrsDiscardRelay}'s note says a third should not simply be added:
|
|
2019
|
+
* both are operator warnings about a configuration/health condition an embedder cannot
|
|
2020
|
+
* see otherwise, and both have an event beside them (`control:write-abandoned` carries
|
|
2021
|
+
* the underlying write failure). A third such condition should surface through
|
|
2022
|
+
* {@link CadreNodeEvents} alone unless it likewise degrades the whole party in silence.
|
|
2023
|
+
*/
|
|
2024
|
+
escalateIfSelfRecordStale(reason, error) {
|
|
2025
|
+
const publishedAt = this.lastSelfRecordPublishAt;
|
|
2026
|
+
if (publishedAt === null || this.selfRecordStaleWarned) {
|
|
2027
|
+
return;
|
|
2028
|
+
}
|
|
2029
|
+
const staleForMs = Date.now() - publishedAt;
|
|
2030
|
+
if (staleForMs < DEFAULT_PEER_RECORD_MAX_AGE_MS) {
|
|
2031
|
+
return;
|
|
2032
|
+
}
|
|
2033
|
+
this.selfRecordStaleWarned = true;
|
|
2034
|
+
console.warn(`[sereus] this machine has not published its own address record for ${Math.round(staleForMs / 60000)} `
|
|
2035
|
+
+ `minutes. The latest attempt (${reason}) failed with: `
|
|
2036
|
+
+ `${error instanceof Error ? error.message : String(error)}. `
|
|
2037
|
+
+ 'Other machines in the party discard an address record older than '
|
|
2038
|
+
+ `${Math.round(DEFAULT_PEER_RECORD_MAX_AGE_MS / 60000)} minutes, so any of them not already `
|
|
2039
|
+
+ 'connected to this one can no longer reach it. Publishing keeps retrying; this is reported once.');
|
|
2040
|
+
}
|
|
1511
2041
|
/**
|
|
1512
2042
|
* Sign this node's address record for publication, stamped strictly later than
|
|
1513
2043
|
* `existing` (the row it is about to replace, or null for a first insert) so a
|
|
@@ -1544,6 +2074,29 @@ export class CadreNode {
|
|
|
1544
2074
|
}
|
|
1545
2075
|
return null;
|
|
1546
2076
|
}
|
|
2077
|
+
/**
|
|
2078
|
+
* Does the `Strand` row name THIS machine as its founder — i.e. is its
|
|
2079
|
+
* `FounderOwnerKey` this node's own owner key (the key behind its PeerId)?
|
|
2080
|
+
* A null/absent column (a consent-seated strand, or a hand-built row) and a
|
|
2081
|
+
* node with no owner key both derive `false`: without a positive match this
|
|
2082
|
+
* machine must attach, never bootstrap. Pure key derivation, no I/O — cheap
|
|
2083
|
+
* enough for {@link launchStrand}'s tracked-instance early return.
|
|
2084
|
+
*
|
|
2085
|
+
* NOTE: reads "one owner key per machine" — the reference model, where the owner
|
|
2086
|
+
* key IS the key behind the PeerId. Two machines running the SAME identity key
|
|
2087
|
+
* would both derive `true` and each bootstrap on its own replica, which is the
|
|
2088
|
+
* double-`Header` hazard this derivation exists to avoid. Unreachable today: that
|
|
2089
|
+
* configuration also gives both machines one PeerId, which already breaks control
|
|
2090
|
+
* networking well before any strand launches. Revisit if machines ever share an
|
|
2091
|
+
* owner key while holding distinct transport identities — the derivation would
|
|
2092
|
+
* then need a per-machine discriminator (the `CadrePeer` PeerId) on the row.
|
|
2093
|
+
*/
|
|
2094
|
+
isSelfFoundedRow(strand) {
|
|
2095
|
+
if (strand.FounderOwnerKey == null) {
|
|
2096
|
+
return false;
|
|
2097
|
+
}
|
|
2098
|
+
return strand.FounderOwnerKey === this.getSelfSigningKey()?.publicKeyB64;
|
|
2099
|
+
}
|
|
1547
2100
|
/**
|
|
1548
2101
|
* The owner keypair (base64url Ed25519) derived from this node's resolved
|
|
1549
2102
|
* identity key. In the single-key reference model the owner signing key is
|
|
@@ -1591,7 +2144,12 @@ export class CadreNode {
|
|
|
1591
2144
|
return;
|
|
1592
2145
|
}
|
|
1593
2146
|
const republish = (reason) => {
|
|
1594
|
-
void this.registerSelf().catch((error) =>
|
|
2147
|
+
void this.registerSelf().catch((error) => {
|
|
2148
|
+
log('Record refresh (%s) failed: %o', reason, error);
|
|
2149
|
+
// The only failure in this file that degrades the whole party on its own, so it is
|
|
2150
|
+
// the only one that escalates past `debug`. See escalateIfSelfRecordStale.
|
|
2151
|
+
this.escalateIfSelfRecordStale(reason, error);
|
|
2152
|
+
});
|
|
1595
2153
|
};
|
|
1596
2154
|
const reconcile = (reason) => {
|
|
1597
2155
|
void this.reconcileControlCohort().catch((error) => log('Control-cohort reconcile (%s) failed: %o', reason, error));
|
|
@@ -1609,7 +2167,10 @@ export class CadreNode {
|
|
|
1609
2167
|
// `CadrePeer` row, so its boot-time publish does nothing, and the two triggers
|
|
1610
2168
|
// wired here are the only ones after that: `self:peer:update` needs an ADDRESS
|
|
1611
2169
|
// CHANGE, and a relay-only node's addresses stop changing the moment its
|
|
1612
|
-
// reservation lands — inside `start()`, before this listener exists
|
|
2170
|
+
// reservation lands — inside `start()`, before this listener exists (on the
|
|
2171
|
+
// `network.requireRelay: false` posture the reservation may instead land LATER,
|
|
2172
|
+
// from the retry supervisor, and then this listener is what publishes the
|
|
2173
|
+
// `/p2p-circuit` address the boot-time publish could not carry). Harmless
|
|
1613
2174
|
// where a late-authorized node's addresses keep churning (NAT/relay rotation
|
|
1614
2175
|
// fire the event anyway), and invisible while parties authorize members before
|
|
1615
2176
|
// they boot. If late enrollment becomes the normal path — an invited phone is
|
|
@@ -1671,10 +2232,16 @@ export class CadreNode {
|
|
|
1671
2232
|
this.hasControlConnection = false;
|
|
1672
2233
|
this.reconstructedLocalOnlyWrites = false;
|
|
1673
2234
|
this.reissuedHeldRevocations = false;
|
|
2235
|
+
this.revocationLedgerOpened = false;
|
|
1674
2236
|
this.pendingPeerWrites.clear();
|
|
1675
2237
|
this.pendingRevocations.clear();
|
|
1676
2238
|
this.pendingSelfPeerWrite = false;
|
|
1677
2239
|
this.pendingSelfDeviceWrite = false;
|
|
2240
|
+
// Self-record publication history is per-session too: a restarted node re-publishes
|
|
2241
|
+
// from scratch, and carrying the previous lifetime's stamp forward would either
|
|
2242
|
+
// suppress the stale-record warning or fire it on a node that just came up.
|
|
2243
|
+
this.lastSelfRecordPublishAt = null;
|
|
2244
|
+
this.selfRecordStaleWarned = false;
|
|
1678
2245
|
}
|
|
1679
2246
|
/**
|
|
1680
2247
|
* Resolve a peer's current, signed, trust-checkable multiaddrs from only its
|
|
@@ -1762,7 +2329,7 @@ export class CadreNode {
|
|
|
1762
2329
|
* Applied by ALL THREE sources of control-dial candidates, so no call site
|
|
1763
2330
|
* invents its own rule and none can produce a mixed list: the signed record
|
|
1764
2331
|
* ({@link resolvePeerAddrs}), the libp2p address book
|
|
1765
|
-
* ({@link peerStoreAddrs}), and a
|
|
2332
|
+
* ({@link peerStoreAddrs}), and a retained out-of-band dial target
|
|
1766
2333
|
* ({@link bootstrapDialAddrs}).
|
|
1767
2334
|
*
|
|
1768
2335
|
* An address naming a DIFFERENT trailing peer id is dropped: it does not reach
|
|
@@ -1788,21 +2355,26 @@ export class CadreNode {
|
|
|
1788
2355
|
* One address bound to `peerId`, or `null` when it cannot be — logged either
|
|
1789
2356
|
* way, never thrown, so every caller's list-shaping stays total.
|
|
1790
2357
|
*
|
|
2358
|
+
* Used both to normalize addresses this node DIALS (a `CadrePeer` row, a
|
|
2359
|
+
* retained bootstrap addr) and to normalize the ones it ANNOUNCES
|
|
2360
|
+
* ({@link getStrandMultiaddrs}); the rule is the same either way — an address
|
|
2361
|
+
* that does not reach `peerId` is not an address for `peerId`.
|
|
2362
|
+
*
|
|
1791
2363
|
* `withTrailingPeerId` encapsulates `/p2p/<peerId>`, which throws on a peer id
|
|
1792
2364
|
* that does not parse. Unreachable for a `CadrePeer` row (the binding gate
|
|
1793
|
-
* above parsed it already), reachable for a
|
|
1794
|
-
* ({@link bootstrapDialAddrs}).
|
|
2365
|
+
* above parsed it already) and for our own node's id, reachable for a
|
|
2366
|
+
* retained one ({@link bootstrapDialAddrs}).
|
|
1795
2367
|
*/
|
|
1796
2368
|
bindAddrToPeer(addr, peerId) {
|
|
1797
2369
|
try {
|
|
1798
2370
|
const bound = withTrailingPeerId(addr, peerId);
|
|
1799
2371
|
if (!bound) {
|
|
1800
|
-
log('
|
|
2372
|
+
log('bindAddrToPeer: dropping addr %s — it names a peer other than %s', addr.toString(), peerId);
|
|
1801
2373
|
}
|
|
1802
2374
|
return bound;
|
|
1803
2375
|
}
|
|
1804
2376
|
catch (error) {
|
|
1805
|
-
log('
|
|
2377
|
+
log('bindAddrToPeer: cannot bind %s to %s: %o', addr.toString(), peerId, error);
|
|
1806
2378
|
return null;
|
|
1807
2379
|
}
|
|
1808
2380
|
}
|
|
@@ -1841,9 +2413,15 @@ export class CadreNode {
|
|
|
1841
2413
|
* `self:peer:update` (all wired in {@link startRecordRefresh}).
|
|
1842
2414
|
*
|
|
1843
2415
|
* Concurrent triggers collapse into a single in-flight pass
|
|
1844
|
-
* (see {@link reconcileControlCohortInFlight}) so two passes never double-dial
|
|
2416
|
+
* (see {@link reconcileControlCohortInFlight}) so two passes never double-dial;
|
|
2417
|
+
* a call that joins an in-flight pass resolves to THAT pass's result.
|
|
1845
2418
|
* Best-effort throughout: a failure to resolve/dial any one sibling is logged
|
|
1846
2419
|
* and the pass continues; the whole pass is a no-op when the node is alone.
|
|
2420
|
+
*
|
|
2421
|
+
* Resolves to the peers the pass dialled ({@link ControlCohortReconcileResult}),
|
|
2422
|
+
* so a caller can tell a link this pass opened from one something else opened —
|
|
2423
|
+
* the pass skips a peer that is already connected. The timer and event triggers
|
|
2424
|
+
* discard it.
|
|
1847
2425
|
*/
|
|
1848
2426
|
async reconcileControlCohort() {
|
|
1849
2427
|
if (this.reconcileControlCohortInFlight) {
|
|
@@ -1852,7 +2430,7 @@ export class CadreNode {
|
|
|
1852
2430
|
const op = this.runReconcileControlCohort();
|
|
1853
2431
|
this.reconcileControlCohortInFlight = op;
|
|
1854
2432
|
try {
|
|
1855
|
-
await op;
|
|
2433
|
+
return await op;
|
|
1856
2434
|
}
|
|
1857
2435
|
finally {
|
|
1858
2436
|
this.reconcileControlCohortInFlight = null;
|
|
@@ -1863,7 +2441,7 @@ export class CadreNode {
|
|
|
1863
2441
|
// Shutdown / not-yet-started guard (mirrors publishSelfRecord). A pass that
|
|
1864
2442
|
// fires after stop() began must early-return rather than touch a torn-down node.
|
|
1865
2443
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
1866
|
-
return;
|
|
2444
|
+
return { dialed: [] };
|
|
1867
2445
|
}
|
|
1868
2446
|
// Ride this pass's cadence to refresh the per-stream gate's materialized
|
|
1869
2447
|
// authorized set — membership changes that ARRIVED BY REPLICATION (rather
|
|
@@ -1873,11 +2451,15 @@ export class CadreNode {
|
|
|
1873
2451
|
// CadrePeer query (two reads per pass), plus a third from
|
|
1874
2452
|
// `refreshStrandPeerAddrs` on the passes where a strand is due AND this node
|
|
1875
2453
|
// holds a control connection (one read for the whole pass, not one per
|
|
1876
|
-
// strand)
|
|
1877
|
-
//
|
|
2454
|
+
// strand). Each also reads Revocation first. Before the Revocation ledger
|
|
2455
|
+
// marker exists (filed below, once connected) that block is missing, so every
|
|
2456
|
+
// one of those reads consults the cohort about it; once the marker exists every
|
|
2457
|
+
// block they touch is held and none of them does (both states pinned in
|
|
2458
|
+
// control-founding-consult-budget.spec.ts). If those reads ever get costly,
|
|
2459
|
+
// share one row-set across all three.
|
|
1878
2460
|
await this.refreshMembershipGate('reconcile');
|
|
1879
2461
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
1880
|
-
return;
|
|
2462
|
+
return { dialed: [] };
|
|
1881
2463
|
}
|
|
1882
2464
|
// Refresh relay delegate grants BEFORE the sibling enumeration below: a
|
|
1883
2465
|
// solo cadre with a party relay has no siblings to dial but must still keep
|
|
@@ -1885,7 +2467,7 @@ export class CadreNode {
|
|
|
1885
2467
|
// the reservation and faces the connection gate again).
|
|
1886
2468
|
await this.refreshDelegateGrants();
|
|
1887
2469
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
1888
|
-
return;
|
|
2470
|
+
return { dialed: [] };
|
|
1889
2471
|
}
|
|
1890
2472
|
// Then re-warm each running strand's own address book from its siblings, on
|
|
1891
2473
|
// its own (much longer) throttle. Same reasoning as warmSiblingAddrBook
|
|
@@ -1893,7 +2475,7 @@ export class CadreNode {
|
|
|
1893
2475
|
// layer under cadre-core dials strand peers by bare peer id.
|
|
1894
2476
|
await this.refreshStrandPeerAddrs();
|
|
1895
2477
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
1896
|
-
return;
|
|
2478
|
+
return { dialed: [] };
|
|
1897
2479
|
}
|
|
1898
2480
|
const selfPeerId = this.controlNode.peerId.toString();
|
|
1899
2481
|
// Reap guarded rows this node still holds for incarnations an already-committed
|
|
@@ -1935,7 +2517,27 @@ export class CadreNode {
|
|
|
1935
2517
|
log('reconcileControlCohort: reap pass failed (continuing): %o', error);
|
|
1936
2518
|
}
|
|
1937
2519
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
1938
|
-
return;
|
|
2520
|
+
return { dialed: [] };
|
|
2521
|
+
}
|
|
2522
|
+
// File the Revocation ledger marker once, so that table stops being a never-written
|
|
2523
|
+
// block, which the storage layer re-checks with the cohort on every read (and every
|
|
2524
|
+
// membership lookup and guarded insert reads it). Connected-only for the reap's
|
|
2525
|
+
// reason above: a marker committed alone is local-only, and one filed by a
|
|
2526
|
+
// disconnected owner while another machine creates the same collection is exactly
|
|
2527
|
+
// that fork. Owner-only, and at most once per process — see openRevocationLedgerIfDue.
|
|
2528
|
+
//
|
|
2529
|
+
// NOTE: accepted tradeoff — a solo founder does not file the marker until it holds a
|
|
2530
|
+
// control connection (normally its first sibling; a relay or bootstrap connection also
|
|
2531
|
+
// counts — see getControlConnectionCount — and is the reap's same residual), so until
|
|
2532
|
+
// then it keeps paying one local findCluster per read of
|
|
2533
|
+
// the missing Revocation block (0.009 ms each, measured upstream; no network work on
|
|
2534
|
+
// a cohort of one). Fork safety weighed over that and kept; revisit if findCluster
|
|
2535
|
+
// ever shows up as material in a device profile, or if a solo-founding marker can be
|
|
2536
|
+
// made fork-safe (e.g. filed inside the genesis transaction, before any other machine
|
|
2537
|
+
// can hold the party's collections).
|
|
2538
|
+
await this.openRevocationLedgerIfDue();
|
|
2539
|
+
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
2540
|
+
return { dialed: [] };
|
|
1939
2541
|
}
|
|
1940
2542
|
}
|
|
1941
2543
|
// 1. Enumerate known siblings. This membership read is itself a pull-on-read
|
|
@@ -1949,12 +2551,11 @@ export class CadreNode {
|
|
|
1949
2551
|
// because filling this table needs a connection and getting a connection
|
|
1950
2552
|
// needs this table. Fall back to the seed's bootstrap addresses; a solo
|
|
1951
2553
|
// cadre has none and this is a no-op. Must not throw or busy-loop.
|
|
1952
|
-
await this.dialColdStartBootstrap();
|
|
1953
|
-
return;
|
|
2554
|
+
return { dialed: await this.dialColdStartBootstrap() };
|
|
1954
2555
|
}
|
|
1955
2556
|
// Re-guard after the await: a stop() may have raced the membership read.
|
|
1956
2557
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
1957
|
-
return;
|
|
2558
|
+
return { dialed: [] };
|
|
1958
2559
|
}
|
|
1959
2560
|
// 2. Classify backbone (owner) members and select a bounded dial set.
|
|
1960
2561
|
// NOTE: deliberately the REPLICATED table, not the node-local anchor — this
|
|
@@ -1964,7 +2565,7 @@ export class CadreNode {
|
|
|
1964
2565
|
// the anchor if owner status here ever gates something trusted.
|
|
1965
2566
|
const ownerKeys = await this.controlDatabase.getOwnerKeys();
|
|
1966
2567
|
if (!this._running || !this.controlNode) {
|
|
1967
|
-
return;
|
|
2568
|
+
return { dialed: [] };
|
|
1968
2569
|
}
|
|
1969
2570
|
const targetDegree = this.config.network?.controlCohort?.targetDegree
|
|
1970
2571
|
?? DEFAULT_CONTROL_COHORT_TARGET_DEGREE;
|
|
@@ -1977,25 +2578,51 @@ export class CadreNode {
|
|
|
1977
2578
|
// keeping what resolved for the dial loop below.
|
|
1978
2579
|
const resolved = await this.warmSiblingAddrBook(siblings);
|
|
1979
2580
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
1980
|
-
return;
|
|
2581
|
+
return { dialed: [] };
|
|
1981
2582
|
}
|
|
1982
2583
|
// 4. Skip already-connected peers (no re-dial / churn for live connections).
|
|
1983
2584
|
const connected = new Set(this.controlNode.getConnections().map((c) => c.remotePeer.toString()));
|
|
1984
2585
|
// 5. Dial each selected, not-yet-connected sibling, best-effort, from the
|
|
1985
2586
|
// addresses step 3 already resolved for it.
|
|
1986
|
-
|
|
2587
|
+
const dialed = [];
|
|
1987
2588
|
for (const sibling of dials) {
|
|
1988
2589
|
if (!this._running || !this.controlNode) {
|
|
1989
|
-
return;
|
|
2590
|
+
return { dialed };
|
|
1990
2591
|
}
|
|
1991
2592
|
if (connected.has(sibling.peerId)) {
|
|
1992
2593
|
continue;
|
|
1993
2594
|
}
|
|
1994
2595
|
if (await this.dialControlSibling(sibling, resolved.get(sibling.peerId) ?? [])) {
|
|
1995
|
-
dialed
|
|
2596
|
+
dialed.push(sibling.peerId);
|
|
1996
2597
|
}
|
|
1997
2598
|
}
|
|
1998
|
-
log('reconcileControlCohort: pass complete (siblings=%d, selected=%d, dialed=%d)', siblings.length, dials.length, dialed);
|
|
2599
|
+
log('reconcileControlCohort: pass complete (siblings=%d, selected=%d, dialed=%d)', siblings.length, dials.length, dialed.length);
|
|
2600
|
+
return { dialed };
|
|
2601
|
+
}
|
|
2602
|
+
/**
|
|
2603
|
+
* The owner-only half of the reconcile pass's ledger-marker step: file the singleton
|
|
2604
|
+
* `Revocation` marker ({@link SeedBootstrapService.openRevocationLedger}) unless this
|
|
2605
|
+
* process has already seen it filed. The connectivity gate is the caller's
|
|
2606
|
+
* ({@link runReconcileControlCohort}); why the marker exists is on
|
|
2607
|
+
* {@link ControlDatabase.openRevocationLedger}.
|
|
2608
|
+
*
|
|
2609
|
+
* Sets {@link revocationLedgerOpened} on `'opened'` or `'already-open'`. A node that
|
|
2610
|
+
* cannot sign as an owner does nothing: it picks the owner's marker up the first time it
|
|
2611
|
+
* reads the table, like any other replicated row. Best-effort like every step of the
|
|
2612
|
+
* pass — a failure is logged and leaves the flag clear, so the next connected pass retries.
|
|
2613
|
+
*/
|
|
2614
|
+
async openRevocationLedgerIfDue() {
|
|
2615
|
+
if (this.revocationLedgerOpened || !this.seedBootstrapService?.canAuthorize()) {
|
|
2616
|
+
return;
|
|
2617
|
+
}
|
|
2618
|
+
try {
|
|
2619
|
+
const outcome = await this.seedBootstrapService.openRevocationLedger();
|
|
2620
|
+
this.revocationLedgerOpened = true;
|
|
2621
|
+
log('reconcileControlCohort: revocation ledger marker %s', outcome);
|
|
2622
|
+
}
|
|
2623
|
+
catch (error) {
|
|
2624
|
+
log('reconcileControlCohort: filing the revocation ledger marker failed (retrying next pass): %o', error);
|
|
2625
|
+
}
|
|
1999
2626
|
}
|
|
2000
2627
|
/**
|
|
2001
2628
|
* Resolve every sibling's signed address record once and merge what resolves
|
|
@@ -2016,6 +2643,9 @@ export class CadreNode {
|
|
|
2016
2643
|
* resolves to nothing (revoked, stale, untrusted) is not written at all, so its
|
|
2017
2644
|
* existing entry ages out on its own.
|
|
2018
2645
|
*
|
|
2646
|
+
* The same resolution also keeps a sibling's retained out-of-band dial target
|
|
2647
|
+
* current ({@link refreshDialHint}).
|
|
2648
|
+
*
|
|
2019
2649
|
* NOTE: this resolves EVERY sibling serially before the dial loop below runs,
|
|
2020
2650
|
* so it costs one record query per sibling per reconcile pass (~15s) and each
|
|
2021
2651
|
* one delays the pass's first dial. A cadre is a handful of devices, so today
|
|
@@ -2025,6 +2655,9 @@ export class CadreNode {
|
|
|
2025
2655
|
async warmSiblingAddrBook(siblings) {
|
|
2026
2656
|
const resolved = new Map();
|
|
2027
2657
|
const counts = { merged: 0, restamped: 0, skipped: 0, failed: 0 };
|
|
2658
|
+
// One copy of the store's map for the whole loop; a refresh replaces an entry
|
|
2659
|
+
// rather than mutating it, so the copy stays a consistent "before" view.
|
|
2660
|
+
const hints = this.bootstrapPeerStore?.all();
|
|
2028
2661
|
for (const sibling of siblings) {
|
|
2029
2662
|
if (!this._running || !this.controlNode) {
|
|
2030
2663
|
break;
|
|
@@ -2037,6 +2670,7 @@ export class CadreNode {
|
|
|
2037
2670
|
if (!this._running || !controlNode) {
|
|
2038
2671
|
break;
|
|
2039
2672
|
}
|
|
2673
|
+
this.refreshDialHint(sibling.peerId, addrs, hints?.get(sibling.peerId)?.addrs);
|
|
2040
2674
|
// `mergePeerAddrs` owns the whole best-effort contract — including parsing
|
|
2041
2675
|
// a malformed `CadrePeer.PeerId` — so one bad row cannot abort the pass.
|
|
2042
2676
|
counts[await mergePeerAddrs(controlNode, sibling.peerId, addrs)]++;
|
|
@@ -2044,6 +2678,40 @@ export class CadreNode {
|
|
|
2044
2678
|
log('reconcileControlCohort: address book warmed (siblings=%d, merged=%d, restamped=%d, skipped=%d, failed=%d)', resolved.size, counts.merged, counts.restamped, counts.skipped, counts.failed);
|
|
2045
2679
|
return resolved;
|
|
2046
2680
|
}
|
|
2681
|
+
/**
|
|
2682
|
+
* Replace a sibling's retained out-of-band dial target (see
|
|
2683
|
+
* {@link bootstrapPeerStore}) with the addresses its signed record just resolved
|
|
2684
|
+
* to, when the two differ — so an address change this node saw while the record
|
|
2685
|
+
* was fresh (a new port, a new LAN address) is what it dials after a relaunch
|
|
2686
|
+
* that outlives the record's freshness window.
|
|
2687
|
+
*
|
|
2688
|
+
* Only a sibling that already HAS an entry (`retained`) is refreshed: the store
|
|
2689
|
+
* holds peers learned out of band and must not grow into a copy of every
|
|
2690
|
+
* sibling's addresses. An empty resolution leaves the entry alone — a stale or
|
|
2691
|
+
* not-yet-signed record is exactly when the entry is needed. The comparison is
|
|
2692
|
+
* order-insensitive and runs on the retained addresses after binding them to the
|
|
2693
|
+
* peer id, so an entry recorded without `/p2p/` suffixes is not rewritten merely
|
|
2694
|
+
* for lacking them.
|
|
2695
|
+
*
|
|
2696
|
+
* NOTE: replaces rather than merges, so the entry becomes exactly what the
|
|
2697
|
+
* sibling announces. If a node's signed record ever lists fewer addresses this
|
|
2698
|
+
* node can reach than it was added with (say an announce override naming only a
|
|
2699
|
+
* public address, for a node added by its LAN address), a relaunch dials the
|
|
2700
|
+
* worse set; merge the two lists here if that shows up.
|
|
2701
|
+
*/
|
|
2702
|
+
refreshDialHint(peerId, resolved, retained) {
|
|
2703
|
+
if (!retained || resolved.length === 0) {
|
|
2704
|
+
return;
|
|
2705
|
+
}
|
|
2706
|
+
// Both lists are de-duplicated by `normalizeDialAddrs`, so equal size plus
|
|
2707
|
+
// containment is set equality.
|
|
2708
|
+
const current = resolved.map((addr) => addr.toString());
|
|
2709
|
+
const previous = new Set(this.bootstrapDialAddrs(peerId, retained).map((addr) => addr.toString()));
|
|
2710
|
+
if (current.length === previous.size && current.every((addr) => previous.has(addr))) {
|
|
2711
|
+
return;
|
|
2712
|
+
}
|
|
2713
|
+
this.retainDialTarget(peerId, current, 'refreshDialHint');
|
|
2714
|
+
}
|
|
2047
2715
|
/**
|
|
2048
2716
|
* {@link resolvePeerAddrs} for one sibling, best-effort: a control-DB read
|
|
2049
2717
|
* failure yields `[]` (and the cold-start fallback then gets its turn) rather
|
|
@@ -2059,25 +2727,31 @@ export class CadreNode {
|
|
|
2059
2727
|
}
|
|
2060
2728
|
}
|
|
2061
2729
|
/**
|
|
2062
|
-
*
|
|
2730
|
+
* Time limits for dialing ONE control-network peer from its candidate
|
|
2731
|
+
* addresses ({@link dialPeerAddrs}): the reconcile pass's steady-state sibling
|
|
2063
2732
|
* ({@link dialControlSibling}) and cold-start bootstrap peer
|
|
2064
|
-
* ({@link dialBootstrapPeer})
|
|
2065
|
-
*
|
|
2733
|
+
* ({@link dialBootstrapPeer}) dials, and — handed to every
|
|
2734
|
+
* {@link SeedBootstrapService} this node builds — `applySeed`'s owner dials and
|
|
2735
|
+
* `dialInvite`.
|
|
2066
2736
|
*
|
|
2067
|
-
* See {@link DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS} for why
|
|
2068
|
-
*
|
|
2737
|
+
* See {@link DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS} for why a peer's dial is
|
|
2738
|
+
* bounded as a whole, and {@link DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS}
|
|
2739
|
+
* for why each address is bounded as well.
|
|
2069
2740
|
*/
|
|
2070
|
-
|
|
2071
|
-
|
|
2072
|
-
|
|
2741
|
+
controlDialBudget() {
|
|
2742
|
+
const cohort = this.config.network?.controlCohort;
|
|
2743
|
+
return {
|
|
2744
|
+
perAddressMs: cohort?.perAddressDialTimeoutMs ?? DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS,
|
|
2745
|
+
totalMs: cohort?.dialTimeoutMs ?? DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS,
|
|
2746
|
+
};
|
|
2073
2747
|
}
|
|
2074
2748
|
/**
|
|
2075
2749
|
* Dial one sibling from its already-resolved addresses, best-effort. Returns
|
|
2076
|
-
* whether
|
|
2750
|
+
* whether the dial resolved (false when no address resolves or the dial fails).
|
|
2077
2751
|
*
|
|
2078
2752
|
* A per-peer failure (NAT, offline, relay down, connection-gater denial, or
|
|
2079
|
-
* the {@link
|
|
2080
|
-
*
|
|
2753
|
+
* the {@link controlDialBudget} expiring) is logged and swallowed so one
|
|
2754
|
+
* unreachable sibling never aborts the pass — exactly like
|
|
2081
2755
|
* {@link SeedBootstrapService.applySeed}'s owner-dial loop. A failed dial is
|
|
2082
2756
|
* simply retried on the next pass.
|
|
2083
2757
|
*/
|
|
@@ -2093,7 +2767,7 @@ export class CadreNode {
|
|
|
2093
2767
|
}
|
|
2094
2768
|
try {
|
|
2095
2769
|
log('reconcileControlCohort: dialing sibling %s (%d addr(s))', sibling.peerId, addrs.length);
|
|
2096
|
-
await
|
|
2770
|
+
await dialPeerAddrs(controlNode, addrs, this.controlDialBudget(), `reconcileControlCohort dial of sibling ${sibling.peerId}`);
|
|
2097
2771
|
return true;
|
|
2098
2772
|
}
|
|
2099
2773
|
catch (error) {
|
|
@@ -2102,20 +2776,65 @@ export class CadreNode {
|
|
|
2102
2776
|
}
|
|
2103
2777
|
}
|
|
2104
2778
|
/**
|
|
2105
|
-
* A sibling's control-network dial addresses for the reconcile pass
|
|
2779
|
+
* A sibling's control-network dial addresses for the reconcile pass, from the
|
|
2780
|
+
* first of three sources that yields any:
|
|
2781
|
+
*
|
|
2782
|
+
* 1. the signed, fresh, trust-gated control addresses `resolved` for it by
|
|
2783
|
+
* {@link warmSiblingAddrBook} — passed in rather than re-resolved, so the
|
|
2784
|
+
* pass makes one record query per sibling;
|
|
2785
|
+
* 2. the libp2p address book ({@link peerStoreAddrs}) — the entries `applySeed`
|
|
2786
|
+
* and identify put there, until they age out;
|
|
2787
|
+
* 3. the sibling's retained out-of-band dial target ({@link retainedDialAddrs}).
|
|
2788
|
+
* For a node that cannot listen, this is the only source for a sibling it
|
|
2789
|
+
* added but has not yet connected to: the added node's row stays unsigned
|
|
2790
|
+
* until it self-publishes, which needs the connection this dial opens. It is
|
|
2791
|
+
* also what a relaunch that outlived the record's freshness window uses.
|
|
2792
|
+
*
|
|
2793
|
+
* Returns `[]` (never throws) when none yields an address — that sibling is
|
|
2794
|
+
* skipped this pass.
|
|
2795
|
+
*
|
|
2796
|
+
* The list may name transports this node cannot dial (a lent node reports TCP
|
|
2797
|
+
* and `/ws` addresses to a phone that dials WebSockets only). That needs no
|
|
2798
|
+
* filtering here: libp2p's dial queue (`calculateMultiaddrs`, libp2p 3.1.3)
|
|
2799
|
+
* rejects an address no transport can dial before touching the network, so
|
|
2800
|
+
* such an address costs a log line.
|
|
2106
2801
|
*
|
|
2107
|
-
*
|
|
2108
|
-
*
|
|
2109
|
-
*
|
|
2110
|
-
*
|
|
2111
|
-
*
|
|
2112
|
-
* yields an address — that sibling is skipped this pass.
|
|
2802
|
+
* The list is NOT handed to one `dial()`. libp2p tries a multi-address dial's
|
|
2803
|
+
* addresses one after another under a single deadline, with no limit per
|
|
2804
|
+
* address, and sorts loopback addresses last — so one or two addresses that
|
|
2805
|
+
* never answer used up the whole deadline before the address that worked was
|
|
2806
|
+
* tried. {@link dialPeerAddrs} dials each address on its own time limit instead.
|
|
2113
2807
|
*/
|
|
2114
2808
|
async resolveControlDialAddrs(peerId, resolved) {
|
|
2115
2809
|
if (resolved.length > 0) {
|
|
2116
2810
|
return resolved;
|
|
2117
2811
|
}
|
|
2118
|
-
|
|
2812
|
+
const booked = await this.peerStoreAddrs(peerId);
|
|
2813
|
+
if (booked.length > 0) {
|
|
2814
|
+
return booked;
|
|
2815
|
+
}
|
|
2816
|
+
return this.retainedDialAddrs(peerId);
|
|
2817
|
+
}
|
|
2818
|
+
/**
|
|
2819
|
+
* A sibling's retained out-of-band dial target (see {@link bootstrapPeerStore}),
|
|
2820
|
+
* bound to its peer id through {@link bootstrapDialAddrs}; `[]` when it has none.
|
|
2821
|
+
*
|
|
2822
|
+
* These addresses are never merged into the libp2p address book: they are
|
|
2823
|
+
* unverified, and the address book takes only verified ones (see
|
|
2824
|
+
* `mergePeerAddrs`). Layers that dial by bare peer id use the connection this
|
|
2825
|
+
* dial opens, and after it drops the next reconcile pass dials again.
|
|
2826
|
+
*
|
|
2827
|
+
* NOTE: an entry is consulted here only for a current sibling, so a peer revoked
|
|
2828
|
+
* by another owner is not dialed from it, even though the entry stays (only this
|
|
2829
|
+
* node's own {@link removePeer} forgets one). The cold-start branch
|
|
2830
|
+
* ({@link dialColdStartBootstrap}) still dials every entry while there are no
|
|
2831
|
+
* siblings at all; if entries for revoked peers ever cause dial churn there,
|
|
2832
|
+
* prune entries whose peer has a retired `CadrePeer` row during the membership
|
|
2833
|
+
* refresh.
|
|
2834
|
+
*/
|
|
2835
|
+
retainedDialAddrs(peerId) {
|
|
2836
|
+
const entry = this.bootstrapPeerStore?.all().get(peerId);
|
|
2837
|
+
return entry ? this.bootstrapDialAddrs(peerId, entry.addrs) : [];
|
|
2119
2838
|
}
|
|
2120
2839
|
/**
|
|
2121
2840
|
* Cold-start fallback: the libp2p peerStore multiaddrs for `peerId` (seeded by
|
|
@@ -2163,37 +2882,55 @@ export class CadreNode {
|
|
|
2163
2882
|
* signature-checked against a trust-anchored signer before this runs, and the
|
|
2164
2883
|
* flag only *selects a dial target*; a dial grants no authority.
|
|
2165
2884
|
*
|
|
2166
|
-
*
|
|
2167
|
-
* that applies a seed minted after it joined finds itself in the owner list;
|
|
2168
|
-
* retaining that would make the cold-start pass dial this node forever (the
|
|
2169
|
-
* steady-state pass filters self out of its sibling list for the same reason).
|
|
2885
|
+
* Addressless peers and self are skipped by {@link retainDialTarget}.
|
|
2170
2886
|
*/
|
|
2171
2887
|
recordSeedBootstrapPeers(seed) {
|
|
2888
|
+
for (const peer of seed.peers) {
|
|
2889
|
+
if (peer.isOwner) {
|
|
2890
|
+
this.retainDialTarget(peer.peerId, peer.multiaddrs, 'recordSeedBootstrapPeers');
|
|
2891
|
+
}
|
|
2892
|
+
}
|
|
2893
|
+
}
|
|
2894
|
+
/**
|
|
2895
|
+
* Retain (or replace) one peer's out-of-band dial target in
|
|
2896
|
+
* {@link bootstrapPeerStore}. Shared by every writer: seed intake, {@link addDrone}
|
|
2897
|
+
* and {@link refreshDialHint}.
|
|
2898
|
+
*
|
|
2899
|
+
* Fire-and-log: the entry is visible synchronously by the store's contract, and
|
|
2900
|
+
* the promise tracks durability only. A persist failure costs restart survival,
|
|
2901
|
+
* never this session's dial set — the same trade
|
|
2902
|
+
* `SeedBootstrapService.anchorAcceptedSigner` makes — so it is logged rather than
|
|
2903
|
+
* failing a seed or an add that has already been accepted.
|
|
2904
|
+
*
|
|
2905
|
+
* Two peers are never retained. One with no address: there is nothing to dial.
|
|
2906
|
+
* And self: `createSeed` projects EVERY `CadrePeer` row, so an owner that applies
|
|
2907
|
+
* a seed minted after it joined finds itself in the owner list, and `addDrone`
|
|
2908
|
+
* could be handed this node's own id. Retaining self would make the cold-start
|
|
2909
|
+
* pass dial this node forever (the steady-state pass filters self out of its
|
|
2910
|
+
* sibling list for the same reason).
|
|
2911
|
+
*/
|
|
2912
|
+
retainDialTarget(peerId, addrs, caller) {
|
|
2172
2913
|
const store = this.bootstrapPeerStore;
|
|
2173
2914
|
if (!store) {
|
|
2174
2915
|
// Unreachable in production: start() builds the store before any network
|
|
2175
|
-
// bring-up, and
|
|
2176
|
-
// SeedBootstrapService rejects
|
|
2177
|
-
|
|
2916
|
+
// bring-up, and every caller needs a started node (an uninitialized
|
|
2917
|
+
// SeedBootstrapService rejects a seed or an add, and the reconcile pass
|
|
2918
|
+
// needs a running one).
|
|
2919
|
+
log('%s: no bootstrap-peer store yet; not retaining %s', caller, peerId);
|
|
2178
2920
|
return;
|
|
2179
2921
|
}
|
|
2180
|
-
|
|
2181
|
-
|
|
2182
|
-
if (!peer.isOwner || peer.multiaddrs.length === 0 || peer.peerId === selfPeerId) {
|
|
2183
|
-
continue;
|
|
2184
|
-
}
|
|
2185
|
-
// Sync-visible by contract; the promise tracks durability only. A persist
|
|
2186
|
-
// failure costs restart survival, never this session's retry set — the same
|
|
2187
|
-
// trade `SeedBootstrapService.anchorAcceptedSigner` makes — so log and carry
|
|
2188
|
-
// on rather than failing the seed that was already accepted.
|
|
2189
|
-
void store.record(peer.peerId, peer.multiaddrs).catch((error) => {
|
|
2190
|
-
log('recordSeedBootstrapPeers: persisting bootstrap peer %s failed (retained in memory): %o', peer.peerId, error);
|
|
2191
|
-
});
|
|
2922
|
+
if (addrs.length === 0 || peerId === this.controlNode?.peerId.toString()) {
|
|
2923
|
+
return;
|
|
2192
2924
|
}
|
|
2925
|
+
void store.record(peerId, addrs).catch((error) => {
|
|
2926
|
+
log('%s: persisting dial target %s failed (retained in memory): %o', caller, peerId, error);
|
|
2927
|
+
});
|
|
2193
2928
|
}
|
|
2194
2929
|
/**
|
|
2195
|
-
* Cold-start branch of the reconcile pass: re-dial
|
|
2196
|
-
*
|
|
2930
|
+
* Cold-start branch of the reconcile pass: re-dial every retained out-of-band
|
|
2931
|
+
* dial target (see {@link bootstrapPeerStore}) — in practice the owner peers of
|
|
2932
|
+
* the seeds this node applied — while the control database still has no
|
|
2933
|
+
* siblings to dial.
|
|
2197
2934
|
*
|
|
2198
2935
|
* `SeedBootstrapService.applySeed` dials those owners exactly ONCE, best-effort.
|
|
2199
2936
|
* When that single dial fails — owner momentarily down, relay reservation not
|
|
@@ -2220,14 +2957,14 @@ export class CadreNode {
|
|
|
2220
2957
|
// cannot mutate what we are iterating.
|
|
2221
2958
|
const targets = this.bootstrapPeerStore?.all();
|
|
2222
2959
|
if (!controlNode || !targets || targets.size === 0) {
|
|
2223
|
-
return;
|
|
2960
|
+
return [];
|
|
2224
2961
|
}
|
|
2225
2962
|
// Same skip rule as step 3 of the steady-state pass: no churn on live links.
|
|
2226
2963
|
const connected = new Set(controlNode.getConnections().map((c) => c.remotePeer.toString()));
|
|
2227
|
-
|
|
2964
|
+
const dialed = [];
|
|
2228
2965
|
for (const [peerId, entry] of targets) {
|
|
2229
2966
|
if (!this._running || !this.controlNode) {
|
|
2230
|
-
return;
|
|
2967
|
+
return dialed;
|
|
2231
2968
|
}
|
|
2232
2969
|
// NOTE: a connection this node still holds in `status: 'open'` after the remote
|
|
2233
2970
|
// has already aborted it counts as connected here, so the cold-start retry stays
|
|
@@ -2240,19 +2977,21 @@ export class CadreNode {
|
|
|
2240
2977
|
continue;
|
|
2241
2978
|
}
|
|
2242
2979
|
if (await this.dialBootstrapPeer(peerId, entry.addrs)) {
|
|
2243
|
-
dialed
|
|
2980
|
+
dialed.push(peerId);
|
|
2244
2981
|
}
|
|
2245
2982
|
}
|
|
2246
|
-
log('reconcileControlCohort: cold-start pass complete (bootstrap=%d, connected=%d, dialed=%d)', targets.size, connected.size, dialed);
|
|
2983
|
+
log('reconcileControlCohort: cold-start pass complete (bootstrap=%d, connected=%d, dialed=%d)', targets.size, connected.size, dialed.length);
|
|
2984
|
+
return dialed;
|
|
2247
2985
|
}
|
|
2248
2986
|
/**
|
|
2249
|
-
* Bind a
|
|
2250
|
-
* authenticates the peer it is aiming at rather than
|
|
2987
|
+
* Bind a retained dial target's addresses (see {@link bootstrapPeerStore}) to
|
|
2988
|
+
* its peer id, so the dial authenticates the peer it is aiming at rather than
|
|
2989
|
+
* trusting whoever answers.
|
|
2251
2990
|
*
|
|
2252
2991
|
* The same {@link normalizeDialAddrs} rule the resolved and address-book paths
|
|
2253
2992
|
* use, on the third and last source of control-dial candidates: an address that
|
|
2254
2993
|
* already terminates in `/p2p/<id>` must carry THIS peer's id or it is dropped
|
|
2255
|
-
* (
|
|
2994
|
+
* (an entry that disagrees with itself is not a dial target); one that does not —
|
|
2256
2995
|
* a bare listen addr, or a relay hop with the destination missing — gets the id
|
|
2257
2996
|
* encapsulated; an unencapsulatable one is dropped rather than dialed bare.
|
|
2258
2997
|
*/
|
|
@@ -2272,7 +3011,7 @@ export class CadreNode {
|
|
|
2272
3011
|
}
|
|
2273
3012
|
try {
|
|
2274
3013
|
log('reconcileControlCohort(cold-start): dialing bootstrap peer %s (%d addr(s))', peerId, parsed.length);
|
|
2275
|
-
await
|
|
3014
|
+
await dialPeerAddrs(controlNode, parsed, this.controlDialBudget(), `reconcileControlCohort cold-start dial of bootstrap peer ${peerId}`);
|
|
2276
3015
|
return true;
|
|
2277
3016
|
}
|
|
2278
3017
|
catch (error) {
|
|
@@ -2939,6 +3678,13 @@ export class CadreNode {
|
|
|
2939
3678
|
// (register a config + addStrand); the strand-agnostic seam keeps this
|
|
2940
3679
|
// class free of any app's join policy. A self-configured strand (config
|
|
2941
3680
|
// already present) keeps auto-starting below, unchanged.
|
|
3681
|
+
//
|
|
3682
|
+
// Recorded BEFORE the emit so a handler that synchronously drains
|
|
3683
|
+
// `getDiscoveredStrands()` sees this strand too. The watcher never offers
|
|
3684
|
+
// the same strand twice (its `knownStrands` retains the id for the life of
|
|
3685
|
+
// the process), so this map — not the event — is what a late subscriber
|
|
3686
|
+
// reads. See the `strand:discovered` doc in types.ts.
|
|
3687
|
+
this.discoveredStrands.set(strand.Id, strand);
|
|
2942
3688
|
this.emit('strand:discovered', { strandId: strand.Id, strand });
|
|
2943
3689
|
return;
|
|
2944
3690
|
}
|
|
@@ -3044,20 +3790,31 @@ export class CadreNode {
|
|
|
3044
3790
|
}
|
|
3045
3791
|
// Stop all strand instances
|
|
3046
3792
|
await this.strandManager.stopAll();
|
|
3047
|
-
// Clear sApp configs
|
|
3793
|
+
// Clear sApp configs, recorded launch refusals and the unclaimed-strand backlog
|
|
3048
3794
|
this.sAppConfigs.clear();
|
|
3795
|
+
this.strandLaunchRefusals.clear();
|
|
3796
|
+
this.discoveredStrands.clear();
|
|
3049
3797
|
// Drop delegate-admission state: the grants are scoped to the session that
|
|
3050
3798
|
// recorded them, so a stop()/start() cycle on this object must not keep
|
|
3051
3799
|
// admitting delegates announced under the previous one, nor carry stale
|
|
3052
3800
|
// announce timestamps that would suppress the next session's refresh.
|
|
3053
3801
|
this.delegateAdmission.clear();
|
|
3054
3802
|
this.delegateAnnounceAt.clear();
|
|
3803
|
+
// Control backfill first — before the database closes and the control node
|
|
3804
|
+
// stops — so no NEW catch-up push is issued against a torn-down transport.
|
|
3805
|
+
// A push already in flight is not awaited; it fails into the module's own
|
|
3806
|
+
// per-chunk catch. (Mirrors releaseRuntime in strand-instance-manager.ts.)
|
|
3807
|
+
if (this.controlBackfill) {
|
|
3808
|
+
this.controlBackfill.stop();
|
|
3809
|
+
this.controlBackfill = null;
|
|
3810
|
+
}
|
|
3055
3811
|
// Close control database (this also shuts down the collection factory).
|
|
3056
3812
|
// Detach the membership hub first: nothing this teardown does should drive a
|
|
3057
3813
|
// gate refresh, and a notification arriving after this point is a no-op.
|
|
3058
3814
|
if (this.controlDatabase) {
|
|
3059
3815
|
this.controlDatabase.setMembershipChangeListener(null);
|
|
3060
3816
|
this.controlDatabase.setGuardedDeleteListener(null);
|
|
3817
|
+
this.controlDatabase.setControlWriteAbandonedListener(null);
|
|
3061
3818
|
await this.controlDatabase.close();
|
|
3062
3819
|
this.controlDatabase = null;
|
|
3063
3820
|
}
|
|
@@ -3084,6 +3841,21 @@ export class CadreNode {
|
|
|
3084
3841
|
// healthy and make the embedder's retry a no-op ("already running").
|
|
3085
3842
|
this._running = false;
|
|
3086
3843
|
}
|
|
3844
|
+
/**
|
|
3845
|
+
* A gated joiner's database was just published (`StartStrandConfig.onWritable`): the
|
|
3846
|
+
* instance is `'active'` now, so re-arm its idle timer as any activity would. Nothing
|
|
3847
|
+
* records activity on a gated joiner, so its idle timer may already have fired — the
|
|
3848
|
+
* instance read `'idle'` with a hibernate timer pending — and without this the flip
|
|
3849
|
+
* back to `'active'` would leave that timer to quiesce a strand the app just started
|
|
3850
|
+
* using. Emits `strand:writable` after the timers are settled.
|
|
3851
|
+
*/
|
|
3852
|
+
handleStrandWritable(strandId) {
|
|
3853
|
+
const instance = this.strandManager.getInstance(strandId);
|
|
3854
|
+
if (instance) {
|
|
3855
|
+
this.hibernationManager.recordActivity(instance);
|
|
3856
|
+
}
|
|
3857
|
+
this.emit('strand:writable', { strandId });
|
|
3858
|
+
}
|
|
3087
3859
|
// Hibernation callbacks
|
|
3088
3860
|
async handleStrandIdle(strandId) {
|
|
3089
3861
|
const instance = this.strandManager.getInstance(strandId);
|
|
@@ -3133,9 +3905,10 @@ export class CadreNode {
|
|
|
3133
3905
|
log('handleStrandWake: strand %s not found', strandId);
|
|
3134
3906
|
return;
|
|
3135
3907
|
}
|
|
3136
|
-
// Still live (idle wake, or defensive double-wake): no rebuild needed.
|
|
3908
|
+
// Still live (idle wake, or defensive double-wake): no rebuild needed. A joiner
|
|
3909
|
+
// still behind its first-sync gate wakes back to `'syncing'`, not `'active'`.
|
|
3137
3910
|
if (instance.libp2pNode || instance.database) {
|
|
3138
|
-
instance.status =
|
|
3911
|
+
instance.status = liveStrandStatus(instance);
|
|
3139
3912
|
instance.lastActivity = new Date();
|
|
3140
3913
|
log('Strand %s woke (already live)', strandId);
|
|
3141
3914
|
this.emit('strand:waking', { strandId });
|
|
@@ -3152,8 +3925,8 @@ export class CadreNode {
|
|
|
3152
3925
|
/**
|
|
3153
3926
|
* Rebuild a quiesced strand's runtime, re-resolving the volatile cohort input
|
|
3154
3927
|
* first: the discovery seed may have grown since the strand last ran. Shared by
|
|
3155
|
-
* the wake (`handleStrandWake`) and check-in (`handleStrandCheckIn`) paths so
|
|
3156
|
-
*
|
|
3928
|
+
* the wake (`handleStrandWake`) and check-in (`handleStrandCheckIn`) paths so both
|
|
3929
|
+
* apply the same fresh resolution. `resumeStrand` is idempotent
|
|
3157
3930
|
* (returns the live instance unchanged) as a backstop against double-resume.
|
|
3158
3931
|
*/
|
|
3159
3932
|
async resumeStrandRuntime(strandId) {
|
|
@@ -3164,6 +3937,9 @@ export class CadreNode {
|
|
|
3164
3937
|
? peerIdFromPrivateKey(await strandTransportKey(this.identityKey, strandId)).toString()
|
|
3165
3938
|
: undefined;
|
|
3166
3939
|
const bootstrapNodes = await this.resolveCohortSeed(strandId, delegatePeerId);
|
|
3940
|
+
// NOTE: no `servingMachines` override — see `launchStrand` for why a strand node
|
|
3941
|
+
// declares no repair yardstick, and `StartStrandConfig.servingMachines` for the
|
|
3942
|
+
// count that would legitimately go here once one exists.
|
|
3167
3943
|
const instance = await this.strandManager.resumeStrand(strandId, { bootstrapNodes });
|
|
3168
3944
|
// Same reason as the launch path: `bootstrapNodes` only reaches the address
|
|
3169
3945
|
// book through @libp2p/bootstrap discovery, so merge it directly as well.
|
|
@@ -3250,7 +4026,7 @@ export class CadreNode {
|
|
|
3250
4026
|
await this.holdWakeWindow(instance, windowMs);
|
|
3251
4027
|
const sawActivity = instance.lastActivity !== activityMark;
|
|
3252
4028
|
if (sawActivity) {
|
|
3253
|
-
instance.status =
|
|
4029
|
+
instance.status = liveStrandStatus(instance);
|
|
3254
4030
|
this.emit('strand:waking', { strandId });
|
|
3255
4031
|
log('Wake window: strand %s saw activity during the window; staying active', strandId);
|
|
3256
4032
|
return true;
|
|
@@ -3297,59 +4073,298 @@ export class CadreNode {
|
|
|
3297
4073
|
* Add a strand with its sApp configuration.
|
|
3298
4074
|
* The hosting application must provide the sApp schema when creating a strand.
|
|
3299
4075
|
*
|
|
4076
|
+
* This is the ATTACH half only — it starts the local instance and never publishes the
|
|
4077
|
+
* `Strand` row. A joiner (the row arrived over the control network) wants exactly this; a
|
|
4078
|
+
* FOUNDER wants {@link foundStrand}, which publishes and attaches in one resumable call.
|
|
4079
|
+
* With no explicit `founder` flag, founder-ness is derived from the row's
|
|
4080
|
+
* `FounderOwnerKey` (see {@link StrandConfig.founder}) — so attaching a row THIS machine
|
|
4081
|
+
* published (e.g. re-attaching its own orphan after a restart) founds it, and attaching
|
|
4082
|
+
* anyone else's row joins, without the caller needing to know which it is.
|
|
4083
|
+
*
|
|
3300
4084
|
* A rejected call leaves nothing running but DOES leave the sApp config
|
|
3301
4085
|
* registered, deliberately: both an explicit retry and the {@link StrandWatcher}'s
|
|
3302
|
-
* automatic relaunch need it.
|
|
3303
|
-
*
|
|
3304
|
-
*
|
|
3305
|
-
*
|
|
3306
|
-
*
|
|
4086
|
+
* automatic relaunch need it. That background relaunch covers only a strand the
|
|
4087
|
+
* WATCHER launched: it forgets a row whose `onStrandAdded` threw and re-attempts it
|
|
4088
|
+
* on a later poll (each failure re-emitting `strand:error`) until it succeeds. A
|
|
4089
|
+
* strand claimed here after it was DISCOVERED gets no such retry — the watcher
|
|
4090
|
+
* already recorded it in its `knownStrands` when it offered it as
|
|
4091
|
+
* `strand:discovered`, and no later poll re-offers a strand it knows. This call has
|
|
4092
|
+
* also already dropped it from {@link getDiscoveredStrands}, so a failed claim
|
|
4093
|
+
* leaves the strand dormant until the caller retries or the node restarts (see
|
|
4094
|
+
* `backlog/bug-discovered-strand-lost-when-claim-fails`).
|
|
4095
|
+
* {@link detachStrand} (reached via {@link stopStrand}) is what abandons a strand
|
|
4096
|
+
* for good.
|
|
3307
4097
|
*/
|
|
3308
4098
|
async addStrand(config) {
|
|
3309
4099
|
if (!this._running) {
|
|
3310
4100
|
throw new Error('CadreNode not running');
|
|
3311
4101
|
}
|
|
3312
|
-
const { strandRow, sAppConfig, founder } = config;
|
|
3313
|
-
|
|
4102
|
+
const { strandRow, sAppConfig, founder, partyMemberPrivateKey } = config;
|
|
4103
|
+
if (partyMemberPrivateKey !== undefined && strandRow.Type !== 'c') {
|
|
4104
|
+
// Only a closed strand seats a Member/Manager, so an open launch would drop this
|
|
4105
|
+
// key silently. Refuse instead: the caller has confused the party identity key
|
|
4106
|
+
// with something an open strand uses, and a silent drop hides that.
|
|
4107
|
+
throw new Error(`addStrand(${strandRow.Id}): partyMemberPrivateKey belongs to a CLOSED strand ` +
|
|
4108
|
+
`(Type 'c'), but this row is Type '${strandRow.Type}' — an open strand seats no ` +
|
|
4109
|
+
'Member/Manager, so the key would be ignored.');
|
|
4110
|
+
}
|
|
4111
|
+
// Store sApp config for this strand. The strand is claimed now, so it leaves the
|
|
4112
|
+
// unclaimed backlog — a later `getDiscoveredStrands()` drain must not re-offer it.
|
|
3314
4113
|
this.sAppConfigs.set(strandRow.Id, sAppConfig);
|
|
3315
|
-
|
|
3316
|
-
|
|
3317
|
-
//
|
|
3318
|
-
|
|
4114
|
+
this.discoveredStrands.delete(strandRow.Id);
|
|
4115
|
+
log('Registered sAppConfig for strand %s (sApp: %s, founder: %s)', strandRow.Id, sAppConfig.id, founder ?? 'derived');
|
|
4116
|
+
// An unset `founder` is DERIVED from the row inside launchStrand (this node founds
|
|
4117
|
+
// iff the row's FounderOwnerKey is its own owner key); an explicit flag wins — the
|
|
4118
|
+
// formation/responder flows pass one deliberately, since their consent-seated rows
|
|
4119
|
+
// carry a null column. See StrandConfig.founder. Same rule for the explicit
|
|
4120
|
+
// partyMemberPrivateKey: it wins over the StrandPartyKey control-row read.
|
|
4121
|
+
const instance = await this.launchStrand(strandRow, sAppConfig, founder, partyMemberPrivateKey);
|
|
4122
|
+
// The first-sync write gate: a JOINING machine's database is withheld until it has
|
|
4123
|
+
// received the strand's Header from another member, because a write before that
|
|
4124
|
+
// forks every table it touches (see strand-first-sync-gate.ts). Waiting here is what
|
|
4125
|
+
// lets an app write straight after `addStrand` resolves — the reference chat apps
|
|
4126
|
+
// do — so the default is to wait, bounded by `strandFirstSync.timeoutMs`. A timeout
|
|
4127
|
+
// rejects with the retryable `StrandAwaitingFirstSyncError` and leaves the launch
|
|
4128
|
+
// up: this same call, made again once a member is reachable, completes the attach
|
|
4129
|
+
// (the tracked-instance path above returns the still-syncing instance, and this
|
|
4130
|
+
// wait picks it up). Founders and machines that already hold the Header resolve at
|
|
4131
|
+
// once. Only a LIVE gated instance is waited on — a hibernating one is returned as
|
|
4132
|
+
// it always was (no database either; a wake publishes it).
|
|
4133
|
+
if (config.awaitFirstSync !== false && this.strandManager.isAwaitingFirstSync(strandRow.Id)) {
|
|
4134
|
+
await this.strandManager.whenWritable(strandRow.Id, { timeoutMs: this.config.strandFirstSync?.timeoutMs });
|
|
4135
|
+
}
|
|
4136
|
+
return instance;
|
|
4137
|
+
}
|
|
4138
|
+
/**
|
|
4139
|
+
* Resolve once a launched strand is writable — its database published to the app and
|
|
4140
|
+
* its status `'active'` — or reject with the retryable `StrandAwaitingFirstSyncError`
|
|
4141
|
+
* after `timeoutMs` (default `strandFirstSync.timeoutMs`, else
|
|
4142
|
+
* `DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS`). The promise form of the `strand:writable`
|
|
4143
|
+
* event, for a caller that attached without waiting ({@link StrandConfig.awaitFirstSync}
|
|
4144
|
+
* `false`) or that holds a strand the watcher launched. Resolves immediately for a
|
|
4145
|
+
* strand that is already writable; rejects immediately for one this node does not run.
|
|
4146
|
+
* A hibernating strand is not woken — wake it first if that is what is wanted.
|
|
4147
|
+
*/
|
|
4148
|
+
async whenStrandWritable(strandId, options) {
|
|
4149
|
+
return await this.strandManager.whenWritable(strandId, {
|
|
4150
|
+
timeoutMs: options?.timeoutMs ?? this.config.strandFirstSync?.timeoutMs
|
|
4151
|
+
});
|
|
3319
4152
|
}
|
|
3320
4153
|
/**
|
|
3321
4154
|
* Publish a strand row to the shared control database under this node's own
|
|
3322
4155
|
* owner identity, so other cadre members discover it via control-network
|
|
3323
4156
|
* sync (their {@link StrandWatcher} fires `strand:discovered`).
|
|
3324
4157
|
*
|
|
3325
|
-
*
|
|
3326
|
-
*
|
|
3327
|
-
*
|
|
3328
|
-
*
|
|
3329
|
-
* `addStrand` (the row already exists).
|
|
4158
|
+
* **Founding a strand is TWO steps** — this one (publish the row cadre-wide) and
|
|
4159
|
+
* {@link addStrand} (start the local instance, `founder: true` to write the strand's
|
|
4160
|
+
* `Header`/founding membership). `addStrand` deliberately omits the insert: it only
|
|
4161
|
+
* starts the LOCAL instance, whereas publishing makes the strand visible cadre-wide. A
|
|
4162
|
+
* discovering peer only does `addStrand` (the row already exists). Callers founding a
|
|
4163
|
+
* strand should use {@link foundStrand}, which performs both steps and is safe to
|
|
4164
|
+
* re-run — hand-rolling the pair is what left strands half-founded when an app was
|
|
4165
|
+
* killed between them.
|
|
4166
|
+
*
|
|
4167
|
+
* **Idempotent for identical content.** A live row whose `(Type, MemberPrivateKey)` match
|
|
4168
|
+
* the arguments is the state this call would have produced, so the call logs and returns
|
|
4169
|
+
* that row instead of writing — a publish interrupted after it committed can be repeated.
|
|
4170
|
+
* DIFFERENT content on the same id throws, naming the columns that differ, rather than
|
|
4171
|
+
* surfacing the raw `UNIQUE constraint failed: Strand.Id`: silently accepting it would
|
|
4172
|
+
* let a retry reopen a closed strand or swap the key gating its reads. The read and the
|
|
4173
|
+
* insert are not atomic, so a concurrent founder (two machines of one party, same id) is
|
|
4174
|
+
* caught on the insert's uniqueness rejection and resolved the same way — re-read, no-op
|
|
4175
|
+
* on a match, rethrow otherwise. Never an overwrite either way.
|
|
4176
|
+
*
|
|
4177
|
+
* A tombstoned strand is NOT resurrected here: {@link unpublishStrand} deletes the row, so
|
|
4178
|
+
* the idempotent branch is unreachable and the ordinary publish path re-seats it, exactly
|
|
4179
|
+
* as that method documents.
|
|
4180
|
+
*
|
|
4181
|
+
* **Already stuck on `UNIQUE constraint failed: Strand.Id`?** The row is already published
|
|
4182
|
+
* — attach, do NOT republish: `addStrand` (or {@link foundStrand}) with the id and, for a
|
|
4183
|
+
* closed strand, the `MemberPrivateKey` read back from the row. `unpublishStrand` + a
|
|
4184
|
+
* fresh publish also clears it for an OPEN strand, but is destructive for a closed one
|
|
4185
|
+
* (the key exists nowhere else). No app-data wipe is needed for either.
|
|
3330
4186
|
*
|
|
3331
4187
|
* The insert is signed with the ed25519 key behind this node's PeerId — which
|
|
3332
4188
|
* {@link ed25519KeyPairFromLibp2p} also exposes as the node's owner keypair,
|
|
3333
|
-
* so peer identity and owner key are one and the same. That key
|
|
3334
|
-
*
|
|
4189
|
+
* so peer identity and owner key are one and the same. That key is also persisted on the
|
|
4190
|
+
* row as `FounderOwnerKey` (the schema pins the column to the verified signer), which is
|
|
4191
|
+
* what later lets any launch derive "this machine is the founder" from the row alone.
|
|
4192
|
+
* The key must be enrolled in `OwnerKey` (e.g. via {@link ControlDatabase.ensureOwnerKey}
|
|
3335
4193
|
* at genesis) or the schema's `Strand.AuthorizedInsert` constraint rejects the write.
|
|
3336
4194
|
* Failing loudly here is intentional: a silently-unpublished strand would run
|
|
3337
4195
|
* as a local-only island that no peer could ever discover or join.
|
|
3338
4196
|
*
|
|
4197
|
+
* A CLOSED strand's publish also seats this party's own membership identity key —
|
|
4198
|
+
* the `StrandPartyKey` row {@link ensureStrandPartyKey} documents — which is what the
|
|
4199
|
+
* founder bootstrap derives `Member.Key`/`Manager.MemberKey` from. The
|
|
4200
|
+
* `memberPrivateKey` ARGUMENT stays the strand-wide read secret formation hands to
|
|
4201
|
+
* joiners; it derives nobody's identity.
|
|
4202
|
+
*
|
|
3339
4203
|
* @param strandId - Unique strand identifier (typically the same id passed to
|
|
3340
4204
|
* {@link addStrand}).
|
|
3341
4205
|
* @param type - `'o'` for open (default) or `'c'` for closed.
|
|
3342
|
-
* @param memberPrivateKey - Optional membership key for a closed strand.
|
|
3343
|
-
* @
|
|
3344
|
-
*
|
|
4206
|
+
* @param memberPrivateKey - Optional shared membership (read) key for a closed strand.
|
|
4207
|
+
* @returns The live `Strand` row — the one just inserted, or the matching one already
|
|
4208
|
+
* there. A closed strand's caller should carry THIS row's `MemberPrivateKey` forward:
|
|
4209
|
+
* on a repeat it is the stored key, not the argument.
|
|
4210
|
+
* @throws if the node is not started, exposes no owner signing key, the id is blank, a
|
|
4211
|
+
* row with the same id holds different content, or the control DB rejects the
|
|
4212
|
+
* (unauthorized) insert.
|
|
3345
4213
|
*/
|
|
3346
4214
|
async publishStrand(strandId, type = 'o', memberPrivateKey) {
|
|
3347
4215
|
const signingKey = this.requireOwnerSigningKey(`publish strand ${strandId}`);
|
|
3348
4216
|
// Trim/reject here so the id that lands matches the one unpublishStrand looks up: it
|
|
3349
4217
|
// trims too, and an untrimmed row would be unreachable by the same string.
|
|
3350
4218
|
const trimmed = requireNonBlank(strandId, 'strand id');
|
|
3351
|
-
|
|
4219
|
+
// FounderOwnerKey records THIS machine as the row's publisher (the schema pins it to
|
|
4220
|
+
// the signing owner), which is what later lets a relaunch derive founder-ness.
|
|
4221
|
+
const desired = {
|
|
4222
|
+
Id: trimmed,
|
|
4223
|
+
Type: type,
|
|
4224
|
+
MemberPrivateKey: memberPrivateKey ?? null,
|
|
4225
|
+
FounderOwnerKey: signingKey.publicKeyB64,
|
|
4226
|
+
};
|
|
4227
|
+
// Read first rather than leaning on the insert's collision for BOTH readings. The
|
|
4228
|
+
// ordinary resume (an app killed after its own publish committed) then needs no error
|
|
4229
|
+
// matching at all: the collision path can only recognise itself by the engine's message
|
|
4230
|
+
// TEXT (see isStrandIdConflict), and an upstream reword there must degrade the RARE race,
|
|
4231
|
+
// not the common resume. It also spends no stamp or signature on a known no-op.
|
|
4232
|
+
const existing = await this.controlDatabase.queryStrand(trimmed);
|
|
4233
|
+
if (existing) {
|
|
4234
|
+
// NOTE: reads local converged state, and `queryStrand` does not filter rows whose
|
|
4235
|
+
// StampId a Revocation has retired. So on a sibling that still physically holds a row
|
|
4236
|
+
// deleted elsewhere while ALONE (the delete-while-alone gap in docs/architecture.md),
|
|
4237
|
+
// an owner-signed republish now no-ops onto that doomed row instead of re-seating the
|
|
4238
|
+
// id under a fresh stamp. Harmless while that gap is itself unresolved — both
|
|
4239
|
+
// behaviours end at the same unconverged state — but if delete-while-alone ever gains
|
|
4240
|
+
// real replay, filter retired stamps here (queryStrandStampId + the Revocation read)
|
|
4241
|
+
// so a re-seat is not mistaken for a repeat.
|
|
4242
|
+
const matched = requireMatchingStrandRow(existing, desired, 'is already published');
|
|
4243
|
+
log('publishStrand(%s): already published (type %s) with identical content — no-op', trimmed, type);
|
|
4244
|
+
// A closed strand THIS machine published must hold a party key (its own membership
|
|
4245
|
+
// identity — see ensureStrandPartyKey). Insert-if-absent, so the ordinary repeat
|
|
4246
|
+
// is a read-only no-op and a publish interrupted between the row insert and the
|
|
4247
|
+
// mint heals here. A row another machine published is that machine's to mint for.
|
|
4248
|
+
if (type === 'c' && this.isSelfFoundedRow(matched)) {
|
|
4249
|
+
await this.ensureStrandPartyKey(trimmed);
|
|
4250
|
+
}
|
|
4251
|
+
return matched;
|
|
4252
|
+
}
|
|
4253
|
+
try {
|
|
4254
|
+
await this.controlDatabase.insertStrand(trimmed, type, signingKey.publicKeyB64, signMessageWith(signingKey.privateKeyB64), memberPrivateKey);
|
|
4255
|
+
}
|
|
4256
|
+
catch (error) {
|
|
4257
|
+
// Only the uniqueness collision is a candidate for the idempotent reading; every
|
|
4258
|
+
// other rejection (unauthorized signer, retired stamp) must keep surfacing.
|
|
4259
|
+
if (!isStrandIdConflict(error)) {
|
|
4260
|
+
throw error;
|
|
4261
|
+
}
|
|
4262
|
+
const landed = await this.controlDatabase.queryStrand(trimmed);
|
|
4263
|
+
if (!landed) {
|
|
4264
|
+
// Collided, yet no row is readable: not the race this branch handles (a row
|
|
4265
|
+
// deleted between the rejection and the re-read, or a stale local view). Surface
|
|
4266
|
+
// the original rather than guessing.
|
|
4267
|
+
throw error;
|
|
4268
|
+
}
|
|
4269
|
+
log('publishStrand(%s): lost a concurrent founding race; re-read the landed row', trimmed);
|
|
4270
|
+
return requireMatchingStrandRow(landed, desired, 'landed concurrently from another founder');
|
|
4271
|
+
}
|
|
3352
4272
|
log('Published strand %s (type %s) to control DB under owner %s', trimmed, type, signingKey.publicKeyB64);
|
|
4273
|
+
// A closed strand's publish also mints this party's own membership identity key
|
|
4274
|
+
// (StrandPartyKey) — the key the founder bootstrap derives Member/Manager from,
|
|
4275
|
+
// deliberately distinct from the shared memberPrivateKey argument above. A failure
|
|
4276
|
+
// here surfaces (the strand row has already committed, so a retried publish takes
|
|
4277
|
+
// the idempotent branch and heals the mint; so does the next founder launch).
|
|
4278
|
+
if (type === 'c') {
|
|
4279
|
+
await this.ensureStrandPartyKey(trimmed);
|
|
4280
|
+
}
|
|
4281
|
+
return desired;
|
|
4282
|
+
}
|
|
4283
|
+
/**
|
|
4284
|
+
* Found a strand — publish its row cadre-wide AND start the local instance as its
|
|
4285
|
+
* founder — in one resumable call. The single entry point for creating a strand; a
|
|
4286
|
+
* caller joining one someone else founded uses {@link addStrand} alone.
|
|
4287
|
+
*
|
|
4288
|
+
* Safe to re-run from any point of interruption, which hand-rolling
|
|
4289
|
+
* {@link publishStrand} + {@link addStrand} is not. Either half can already have
|
|
4290
|
+
* happened:
|
|
4291
|
+
*
|
|
4292
|
+
* - **row already published** → the stored row is adopted rather than re-published, so a
|
|
4293
|
+
* run killed after the insert committed no longer dies on
|
|
4294
|
+
* `UNIQUE constraint failed: Strand.Id`. For a closed strand the row's STORED
|
|
4295
|
+
* `MemberPrivateKey` wins over `config.memberPrivateKey`: a caller that mints a key per
|
|
4296
|
+
* attempt (as the reference apps do) would otherwise present a key that does not match
|
|
4297
|
+
* the membership already seated in the strand.
|
|
4298
|
+
* - **instance already running** → the tracked instance is returned, and — because the
|
|
4299
|
+
* `Strand` row records its publishing machine (`FounderOwnerKey`) — a founder request
|
|
4300
|
+
* against an instance that was first launched as a joiner now runs the (idempotent,
|
|
4301
|
+
* insert-if-absent) founder bootstrap on it rather than silently skipping it
|
|
4302
|
+
* ({@link StrandInstanceManager.foundExistingStrand}). So a strand that something else
|
|
4303
|
+
* attached first — the reference RN app's `strand:discovered` handler after a restart,
|
|
4304
|
+
* or this node's own {@link StrandWatcher} poll winning `launchStrand`'s
|
|
4305
|
+
* `resolveCohortSeed` window — still ends up with its `Strand.Header` written.
|
|
4306
|
+
*
|
|
4307
|
+
* Founder-ness is resolved FROM THE ROW, not assumed: this machine founds iff the
|
|
4308
|
+
* resolved row's `FounderOwnerKey` is its own owner key. A fresh publish records this
|
|
4309
|
+
* machine, so the common path founds; a machine that lost a concurrent founding race to
|
|
4310
|
+
* a sibling adopts the sibling's row and ATTACHES instead — deliberately, since two
|
|
4311
|
+
* machines bootstrapping the same strand on separate replicas is the double-`Header`
|
|
4312
|
+
* hazard. The returned {@link FoundStrandResult.founded} says which happened.
|
|
4313
|
+
*
|
|
4314
|
+
* `Type` is compared, not adopted: a stored row of the other type means the caller and
|
|
4315
|
+
* the control plane disagree about what this strand IS, so it throws.
|
|
4316
|
+
*
|
|
4317
|
+
* @returns The instance AND the row the strand actually runs under — read the membership
|
|
4318
|
+
* key from the returned row, not from a freshly minted one ({@link FoundStrandResult}).
|
|
4319
|
+
* @throws if the node is not started, the id is blank, a published row of the same id has
|
|
4320
|
+
* a different `Type`, or either half rejects.
|
|
4321
|
+
*/
|
|
4322
|
+
async foundStrand(config) {
|
|
4323
|
+
const { strandId, type = 'o', memberPrivateKey, sAppConfig, awaitFirstSync } = config;
|
|
4324
|
+
if (!this._running || !this.controlDatabase) {
|
|
4325
|
+
throw new Error(`CadreNode must be started before attempting to found strand ${strandId}`);
|
|
4326
|
+
}
|
|
4327
|
+
const controlDatabase = this.controlDatabase;
|
|
4328
|
+
const trimmed = requireNonBlank(strandId, 'strand id');
|
|
4329
|
+
const timed = (step, op) => timedStep('foundStrand', trimmed, step, op);
|
|
4330
|
+
const tTotal = performance.now();
|
|
4331
|
+
const published = await timed('queryStrand', () => controlDatabase.queryStrand(trimmed));
|
|
4332
|
+
const strandRow = published
|
|
4333
|
+
? this.adoptPublishedStrand(published, type, memberPrivateKey)
|
|
4334
|
+
: await timed('publishStrand', () => this.publishStrand(trimmed, type, memberPrivateKey));
|
|
4335
|
+
// Derived, not hardcoded `true`: adopting a row another machine published means that
|
|
4336
|
+
// machine runs the bootstrap — this one must attach or it would write a second Header.
|
|
4337
|
+
const founded = this.isSelfFoundedRow(strandRow);
|
|
4338
|
+
if (!founded) {
|
|
4339
|
+
log('foundStrand(%s): the stored row was published by a different machine ' +
|
|
4340
|
+
'(FounderOwnerKey is not this node\'s owner key) — attaching as a joiner; ' +
|
|
4341
|
+
'the founder bootstrap runs on the publishing machine', trimmed);
|
|
4342
|
+
}
|
|
4343
|
+
const instance = await timed('addStrand', () => this.addStrand({ strandRow, sAppConfig, founder: founded, awaitFirstSync }));
|
|
4344
|
+
timing(`[foundStrand:${trimmed}] total: ${Math.round(performance.now() - tTotal)}ms`);
|
|
4345
|
+
return { instance, strandRow, founded };
|
|
4346
|
+
}
|
|
4347
|
+
/**
|
|
4348
|
+
* Resume onto an already-published `Strand` row: keep its stored content, reject a type
|
|
4349
|
+
* disagreement. The stored `MemberPrivateKey` is authoritative — see {@link foundStrand}
|
|
4350
|
+
* for why a caller-minted key must lose here (and why that is NOT the same rule as
|
|
4351
|
+
* {@link publishStrand}'s, which throws on a key mismatch because it is being asked to
|
|
4352
|
+
* WRITE that key, not to resume onto what already exists).
|
|
4353
|
+
*/
|
|
4354
|
+
adoptPublishedStrand(published, type, memberPrivateKey) {
|
|
4355
|
+
if (published.Type !== type) {
|
|
4356
|
+
throw new Error(`Cannot found strand ${published.Id} as type '${type}': it is already published as ` +
|
|
4357
|
+
`type '${published.Type}'. Reconcile deliberately rather than re-founding — a ` +
|
|
4358
|
+
'published open strand cannot be closed, nor a closed one opened, by re-publishing.');
|
|
4359
|
+
}
|
|
4360
|
+
if (memberPrivateKey !== undefined && memberPrivateKey !== published.MemberPrivateKey) {
|
|
4361
|
+
log('foundStrand(%s): already published — resuming on the STORED MemberPrivateKey; ' +
|
|
4362
|
+
'the supplied key is discarded (read it back from the returned instance)', published.Id);
|
|
4363
|
+
}
|
|
4364
|
+
else {
|
|
4365
|
+
log('foundStrand(%s): already published (type %s) — adopting the stored row', published.Id, type);
|
|
4366
|
+
}
|
|
4367
|
+
return published;
|
|
3353
4368
|
}
|
|
3354
4369
|
/**
|
|
3355
4370
|
* Remove this party's `Strand` row from the shared control database — the owner-signed
|
|
@@ -3364,10 +4379,12 @@ export class CadreNode {
|
|
|
3364
4379
|
* is an owner-signed control-plane write like every other, not "destroy the network".
|
|
3365
4380
|
*
|
|
3366
4381
|
* Irreversible for a closed strand (`Type='c'`): the row carries `MemberPrivateKey`,
|
|
3367
|
-
* this party's
|
|
3368
|
-
*
|
|
3369
|
-
*
|
|
3370
|
-
*
|
|
4382
|
+
* this party's read secret for that network, and the strand's `StrandPartyKey` row —
|
|
4383
|
+
* this party's own membership identity, removed (and tombstoned) in the SAME
|
|
4384
|
+
* transaction — is stored nowhere else either. With both gone the party can never
|
|
4385
|
+
* again sign as its seated member/manager there, and a re-published row mints a fresh
|
|
4386
|
+
* identity that does not match the membership already written into the strand's RBAC
|
|
4387
|
+
* layer. The strand id itself is NOT
|
|
3371
4388
|
* blacklisted: a fresh owner-signed {@link publishStrand} re-seats it under a new
|
|
3372
4389
|
* stamp — only the unsigned consent re-seat is permanently foreclosed (the removal's
|
|
3373
4390
|
* `Revocation` tombstone names the id, which the consent branch of
|
|
@@ -3418,6 +4435,26 @@ export class CadreNode {
|
|
|
3418
4435
|
'row deletion itself cannot be replayed, so other nodes may keep running the strand ' +
|
|
3419
4436
|
'until the collection converges.', trimmed);
|
|
3420
4437
|
}
|
|
4438
|
+
// A refused pre-split launch left no tracked instance, so the stop below never
|
|
4439
|
+
// reaches detachStrand for it — clear the refusal here so a re-founded id starts clean.
|
|
4440
|
+
this.strandLaunchRefusals.delete(trimmed);
|
|
4441
|
+
// Clear THIS machine's own MemberPeer binding while the strand runtime — and the
|
|
4442
|
+
// retained party key — is still live: the delete above already destroyed the
|
|
4443
|
+
// party's StrandPartyKey row, so once this process forgets the key nothing can
|
|
4444
|
+
// ever sign the removal again. Runs AFTER the control delete so a rejected
|
|
4445
|
+
// unpublish (unauthorized signer) never strips a binding the party still needs;
|
|
4446
|
+
// best-effort by contract (never throws) — a hibernating or unreachable strand
|
|
4447
|
+
// keeps the stale binding, which grants nothing today. Sibling machines observing
|
|
4448
|
+
// this removal via their watchers do NOT clear their own bindings (see the NOTE on
|
|
4449
|
+
// StrandInstanceManager.clearOwnMemberPeerBinding).
|
|
4450
|
+
//
|
|
4451
|
+
// NOTE: this makes unpublishStrand await a STRAND-NETWORK write it never used to —
|
|
4452
|
+
// however long that write takes to reject or commit is now added to unpublish's
|
|
4453
|
+
// wall-clock. Fine while strand writes fail fast on an unreachable quorum; if
|
|
4454
|
+
// unpublish ever starts hanging on an isolated strand, give this call its own
|
|
4455
|
+
// deadline rather than dropping it (a binding cleared late is still worth more than
|
|
4456
|
+
// one never cleared).
|
|
4457
|
+
await this.strandManager.clearOwnMemberPeerBinding(trimmed);
|
|
3421
4458
|
// Converge locally now rather than waiting up to a poll interval. The watcher fires
|
|
3422
4459
|
// onStrandRemoved for a strand it tracked; the explicit stop below covers a node whose
|
|
3423
4460
|
// strandFilter never admitted this strand (the watcher never knew it, so it will never
|
|
@@ -3429,6 +4466,101 @@ export class CadreNode {
|
|
|
3429
4466
|
}
|
|
3430
4467
|
log('Unpublished strand %s from control DB under owner %s', trimmed, signingKey.publicKeyB64);
|
|
3431
4468
|
}
|
|
4469
|
+
/**
|
|
4470
|
+
* Seat this party's own strand membership identity key — the `StrandPartyKey` row —
|
|
4471
|
+
* for `strandId`, minting a fresh one when none is given, and adopting the stored one
|
|
4472
|
+
* when a row already exists. Insert-if-absent and stable thereafter: the founding
|
|
4473
|
+
* `Member.Key` must not change across restarts, or the bootstrap's insert-if-absent
|
|
4474
|
+
* guards stop matching.
|
|
4475
|
+
*
|
|
4476
|
+
* This is the identity half of the closed-strand key split: the row's `PrivateKey` —
|
|
4477
|
+
* NOT the strand row's shared `MemberPrivateKey`, which formation hands to every
|
|
4478
|
+
* joining party — is what the founder bootstrap derives `Member.Key`/`Manager.MemberKey`
|
|
4479
|
+
* from. It replicates to every machine this party owns (same plaintext-at-rest stance
|
|
4480
|
+
* as `MemberPrivateKey`; docs/strands.md → "Closed-Strand Member Key Handling") and is
|
|
4481
|
+
* never put on the formation wire.
|
|
4482
|
+
*
|
|
4483
|
+
* Called internally by {@link publishStrand} (closed strands mint at publish) and by a
|
|
4484
|
+
* founder launch that finds no row (a publish interrupted before its mint); public so a
|
|
4485
|
+
* test harness — or a joiner flow that persists a formation-issued identity — can seat
|
|
4486
|
+
* a specific key deliberately.
|
|
4487
|
+
*
|
|
4488
|
+
* @param strandId - The strand the key is this party's identity for.
|
|
4489
|
+
* @param partyMemberPrivateKey - Optional specific key (base64 protobuf, as
|
|
4490
|
+
* `generateStrandMemberKey` mints). When a row already holds a DIFFERENT key this
|
|
4491
|
+
* throws — a party has one identity per strand; rotation is a deliberate
|
|
4492
|
+
* remove-then-insert, not a silent swap.
|
|
4493
|
+
* @returns The live party key: the one just seated, or the stored one.
|
|
4494
|
+
*/
|
|
4495
|
+
async ensureStrandPartyKey(strandId, partyMemberPrivateKey) {
|
|
4496
|
+
const trimmed = requireNonBlank(strandId, 'strand id');
|
|
4497
|
+
const signingKey = this.requireOwnerSigningKey(`seat a party key for strand ${trimmed}`);
|
|
4498
|
+
const existing = await this.controlDatabase.queryStrandPartyKey(trimmed);
|
|
4499
|
+
if (existing !== null) {
|
|
4500
|
+
if (partyMemberPrivateKey !== undefined && partyMemberPrivateKey !== existing) {
|
|
4501
|
+
// Neither key is included: the party key is this party's membership identity secret.
|
|
4502
|
+
throw new Error(`Strand ${trimmed} already has a party key and the supplied one differs. The stored ` +
|
|
4503
|
+
'key is the identity the strand membership was seated under; to rotate it, remove ' +
|
|
4504
|
+
'the row deliberately (deleteStrandPartyKey) rather than overwriting it.');
|
|
4505
|
+
}
|
|
4506
|
+
return existing;
|
|
4507
|
+
}
|
|
4508
|
+
const key = partyMemberPrivateKey ?? await generateStrandMemberKey();
|
|
4509
|
+
try {
|
|
4510
|
+
await this.controlDatabase.insertStrandPartyKey(trimmed, key, signingKey.publicKeyB64, signMessageWith(signingKey.privateKeyB64));
|
|
4511
|
+
}
|
|
4512
|
+
catch (error) {
|
|
4513
|
+
// The read and the insert are not atomic; a concurrent seat of the same strand's
|
|
4514
|
+
// key can land in between. Adopt the landed row when it satisfies this request —
|
|
4515
|
+
// the same resolution publishStrand applies to its own read-then-insert window —
|
|
4516
|
+
// and rethrow every other rejection (unauthorized signer, retired stamp).
|
|
4517
|
+
const landed = await this.controlDatabase.queryStrandPartyKey(trimmed);
|
|
4518
|
+
if (landed === null || (partyMemberPrivateKey !== undefined && partyMemberPrivateKey !== landed)) {
|
|
4519
|
+
throw error;
|
|
4520
|
+
}
|
|
4521
|
+
log('ensureStrandPartyKey(%s): lost a concurrent seat race; adopting the landed key', trimmed);
|
|
4522
|
+
return landed;
|
|
4523
|
+
}
|
|
4524
|
+
log('Seated strand party key for %s under owner %s', trimmed, signingKey.publicKeyB64);
|
|
4525
|
+
return key;
|
|
4526
|
+
}
|
|
4527
|
+
/**
|
|
4528
|
+
* Resolve the party membership key a closed strand's launch threads into the founder
|
|
4529
|
+
* bootstrap: the explicit attach-time key when given, else the party's persisted
|
|
4530
|
+
* `StrandPartyKey` row, else — on the one machine whose owner key the row names as
|
|
4531
|
+
* founder — a freshly minted-and-persisted key, which seats the identity for a publish
|
|
4532
|
+
* that was interrupted before its mint. Everyone else resolves undefined: non-founding
|
|
4533
|
+
* machines never mint (no mint race between a party's machines), and only a founder
|
|
4534
|
+
* bootstrap needs the key at all.
|
|
4535
|
+
*
|
|
4536
|
+
* Minting does NOT repair a strand founded before the key split: its founding
|
|
4537
|
+
* membership was seated under the shared `MemberPrivateKey`, and nothing re-seats it —
|
|
4538
|
+
* the founder bootstrap refuses such a strand (`PreSplitStrandIdentityError`) instead.
|
|
4539
|
+
*
|
|
4540
|
+
* NOTE: "founding machine" here is the ROW's provenance, not the launch's resolved
|
|
4541
|
+
* `founder` flag — so an explicit `founder: false` over a row this machine published
|
|
4542
|
+
* still mints, an owner-signed write a caller that said "I am not founding" did not ask
|
|
4543
|
+
* for. Harmless (the party needs that identity for the strand either way, and the mint
|
|
4544
|
+
* is insert-if-absent) and unreachable today: the one explicit `founder: false` caller
|
|
4545
|
+
* passes a row with a null `FounderOwnerKey`. If a caller ever pairs `founder: false`
|
|
4546
|
+
* with its own published row, gate the mint on the resolved flag too.
|
|
4547
|
+
*/
|
|
4548
|
+
async resolveStrandPartyKey(strand, explicitKey) {
|
|
4549
|
+
if (explicitKey !== undefined) {
|
|
4550
|
+
return explicitKey;
|
|
4551
|
+
}
|
|
4552
|
+
if (!this.controlDatabase) {
|
|
4553
|
+
return undefined;
|
|
4554
|
+
}
|
|
4555
|
+
const stored = await this.controlDatabase.queryStrandPartyKey(strand.Id);
|
|
4556
|
+
if (stored !== null) {
|
|
4557
|
+
return stored;
|
|
4558
|
+
}
|
|
4559
|
+
if (!this.isSelfFoundedRow(strand)) {
|
|
4560
|
+
return undefined;
|
|
4561
|
+
}
|
|
4562
|
+
return await this.ensureStrandPartyKey(strand.Id);
|
|
4563
|
+
}
|
|
3432
4564
|
/**
|
|
3433
4565
|
* Publish an owner-signed `FormationInvite` (open-invitation token) to the
|
|
3434
4566
|
* shared control database, so a later {@link formStrand} redemption can be
|
|
@@ -3554,21 +4686,97 @@ export class CadreNode {
|
|
|
3554
4686
|
* seed, starts the strand, and registers it with the hibernation manager
|
|
3555
4687
|
* before emitting `strand:started`.
|
|
3556
4688
|
*
|
|
3557
|
-
*
|
|
3558
|
-
*
|
|
3559
|
-
*
|
|
3560
|
-
*
|
|
3561
|
-
*
|
|
4689
|
+
* Founder-ness: an explicit `founder` argument wins (the formation/responder flows pass
|
|
4690
|
+
* one deliberately — their consent-seated rows carry a null `FounderOwnerKey`); when
|
|
4691
|
+
* unset it is DERIVED from the row, so the control-discovered path
|
|
4692
|
+
* (`handleStrandAdded`) and a restart's re-attach found this machine's own strands
|
|
4693
|
+
* without the caller having to know. The derivation is pure key comparison — no I/O.
|
|
4694
|
+
*
|
|
4695
|
+
* Idempotent when the strand manager already tracks `strand.Id` — the watcher
|
|
4696
|
+
* rediscovers a row this node already started and calls this again via
|
|
4697
|
+
* `handleStrandAdded`; without the guard that re-entry would resolve a fresh
|
|
3562
4698
|
* (already-connected) cohort seed and re-emit `strand:started` for an
|
|
3563
4699
|
* instance that never stopped. The guard runs before the cohort-seed RPC
|
|
3564
|
-
* fan-out, so a rediscovery costs nothing beyond the map lookup
|
|
4700
|
+
* fan-out, so a rediscovery costs nothing beyond the map lookup plus the (pure)
|
|
4701
|
+
* founder derivation. NOT a silent no-op any more when the launch resolves as a
|
|
4702
|
+
* FOUNDER: the tracked instance may have been launched first as a joiner (an app
|
|
4703
|
+
* attach, or this node's own watcher poll winning the `resolveCohortSeed` window
|
|
4704
|
+
* below), which used to drop the founder request and leave the strand headerless —
|
|
4705
|
+
* now the tracked instance is founded in place
|
|
4706
|
+
* ({@link StrandInstanceManager.foundExistingStrand}), waking it first if quiesced
|
|
4707
|
+
* so the bootstrap actually runs before this resolves.
|
|
4708
|
+
*
|
|
4709
|
+
* A founder launch refused as pre-split (`PreSplitStrandIdentityError`) is recorded in
|
|
4710
|
+
* {@link strandLaunchRefusals} for the formation arm, then rethrown; a founder launch
|
|
4711
|
+
* that succeeds clears the record (it ran the bootstrap's pre-split check and passed).
|
|
4712
|
+
*/
|
|
4713
|
+
async launchStrand(strand, sAppConfig, founder, explicitPartyKey) {
|
|
4714
|
+
const resolvedFounder = founder ?? this.isSelfFoundedRow(strand);
|
|
4715
|
+
try {
|
|
4716
|
+
const instance = await this.startOrFoundStrand(strand, sAppConfig, resolvedFounder, explicitPartyKey);
|
|
4717
|
+
if (resolvedFounder) {
|
|
4718
|
+
this.strandLaunchRefusals.delete(strand.Id);
|
|
4719
|
+
}
|
|
4720
|
+
return instance;
|
|
4721
|
+
}
|
|
4722
|
+
catch (error) {
|
|
4723
|
+
if (error instanceof PreSplitStrandIdentityError) {
|
|
4724
|
+
this.strandLaunchRefusals.set(strand.Id, error);
|
|
4725
|
+
}
|
|
4726
|
+
throw error;
|
|
4727
|
+
}
|
|
4728
|
+
}
|
|
4729
|
+
/**
|
|
4730
|
+
* The launch itself, for {@link launchStrand} (which documents the behaviour): found a
|
|
4731
|
+
* tracked instance in place, or start a fresh one.
|
|
3565
4732
|
*/
|
|
3566
|
-
async
|
|
4733
|
+
async startOrFoundStrand(strand, sAppConfig, resolvedFounder, explicitPartyKey) {
|
|
4734
|
+
const timed = (step, op) => timedStep('startOrFoundStrand', strand.Id, step, op);
|
|
3567
4735
|
const existing = this.strandManager.getInstance(strand.Id);
|
|
3568
4736
|
if (existing) {
|
|
3569
|
-
|
|
4737
|
+
if (resolvedFounder) {
|
|
4738
|
+
// The resolver runs only when the retained config lacks a party key for a
|
|
4739
|
+
// closed strand (see foundExistingStrand), so the common watcher re-entry
|
|
4740
|
+
// ('already-founder') still costs no control read.
|
|
4741
|
+
const outcome = await timed('foundExistingStrand', () => this.strandManager.foundExistingStrand(strand.Id, () => this.resolveStrandPartyKey(strand, explicitPartyKey)));
|
|
4742
|
+
if (outcome === 'needs-resume') {
|
|
4743
|
+
try {
|
|
4744
|
+
// Quiesced instance: the retained config now founds, but founding promises
|
|
4745
|
+
// the bootstrap has RUN by the time the caller resolves — wake through the
|
|
4746
|
+
// hibernation manager (coalesced with any in-flight wake, timer-aware) so
|
|
4747
|
+
// the rebuild executes it now rather than at some eventual wake.
|
|
4748
|
+
await timed('wakeStrand', () => this.wakeStrand(strand.Id));
|
|
4749
|
+
// The wake's rebuild founds — UNLESS a wake was already in flight when the
|
|
4750
|
+
// config flipped, in which case it had already read the pre-flip config and
|
|
4751
|
+
// rebuilt as a joiner, and `wakeStrand` merely coalesced onto it. Re-run the
|
|
4752
|
+
// (insert-if-absent) bootstrap so founding never resolves headerless.
|
|
4753
|
+
await timed('ensureFounderBootstrap', () => this.strandManager.ensureFounderBootstrap(strand.Id));
|
|
4754
|
+
}
|
|
4755
|
+
catch (error) {
|
|
4756
|
+
// The founding did not happen (e.g. the rebuild refused a pre-split strand and
|
|
4757
|
+
// rolled back, leaving the instance tracked with no runtime): withdraw the flip
|
|
4758
|
+
// so the next attempt re-runs the founding instead of resolving
|
|
4759
|
+
// 'already-founder' over an instance that never founded.
|
|
4760
|
+
this.strandManager.withdrawFounderRequest(strand.Id);
|
|
4761
|
+
throw error;
|
|
4762
|
+
}
|
|
4763
|
+
}
|
|
4764
|
+
log('launchStrand: strand %s already tracked — founder request honored (%s)', strand.Id, outcome);
|
|
4765
|
+
}
|
|
4766
|
+
else {
|
|
4767
|
+
log('launchStrand: strand %s already tracked locally — skipping re-launch', strand.Id);
|
|
4768
|
+
}
|
|
3570
4769
|
return existing;
|
|
3571
4770
|
}
|
|
4771
|
+
// A closed strand's launch carries the party's OWN membership identity key: the
|
|
4772
|
+
// explicit attach-time key, else the control-layer StrandPartyKey row — minted here
|
|
4773
|
+
// when this machine is the row-derived founder and no row exists yet (a publish
|
|
4774
|
+
// interrupted before its mint). A joiner with no persisted key threads undefined;
|
|
4775
|
+
// only a FOUNDER bootstrap needs the key, and that path throws loudly without one
|
|
4776
|
+
// (StrandDatabase).
|
|
4777
|
+
const partyMemberPrivateKey = strand.Type === 'c'
|
|
4778
|
+
? await timed('resolveStrandPartyKey', () => this.resolveStrandPartyKey(strand, explicitPartyKey))
|
|
4779
|
+
: undefined;
|
|
3572
4780
|
// Each strand node gets its own transport identity, derived from the cadre
|
|
3573
4781
|
// identity key + strandId (see strand-transport-key.ts). Sharing the
|
|
3574
4782
|
// control node's key here gave every node one peerId, which collides at a
|
|
@@ -3584,16 +4792,17 @@ export class CadreNode {
|
|
|
3584
4792
|
// identity is ever supported, fall back to `undefined` here — libp2p then
|
|
3585
4793
|
// generates a random per-strand key, which still avoids the collision but
|
|
3586
4794
|
// gives up peerId stability across restarts.
|
|
3587
|
-
const
|
|
3588
|
-
|
|
4795
|
+
const identityKey = this.identityKey;
|
|
4796
|
+
const transportKey = identityKey
|
|
4797
|
+
? await timed('strandTransportKey', () => strandTransportKey(identityKey, strand.Id))
|
|
3589
4798
|
: undefined;
|
|
3590
4799
|
// Derived BEFORE seed resolution so the seed pass doubles as the delegate
|
|
3591
4800
|
// announcement and every grant is recorded before `startStrand` runs
|
|
3592
4801
|
// `libp2p.start()` (the responder records the grant before replying; the
|
|
3593
4802
|
// client awaits the replies).
|
|
3594
4803
|
const delegatePeerId = transportKey ? peerIdFromPrivateKey(transportKey).toString() : undefined;
|
|
3595
|
-
const bootstrapNodes = await this.resolveCohortSeed(strand.Id, delegatePeerId);
|
|
3596
|
-
const instance = await this.strandManager.startStrand({
|
|
4804
|
+
const bootstrapNodes = await timed('resolveCohortSeed', () => this.resolveCohortSeed(strand.Id, delegatePeerId));
|
|
4805
|
+
const instance = await timed('strandManager.startStrand', () => this.strandManager.startStrand({
|
|
3597
4806
|
strandRow: strand,
|
|
3598
4807
|
sAppConfig,
|
|
3599
4808
|
storage: this.config.storage,
|
|
@@ -3604,15 +4813,45 @@ export class CadreNode {
|
|
|
3604
4813
|
bootstrapNodes,
|
|
3605
4814
|
requireSignedSchemas: this.config.requireSignedSchemas,
|
|
3606
4815
|
clusterSize: this.config.strandClusterSize,
|
|
4816
|
+
// NOTE: deliberately NO `servingMachines`, so the strand node declares no repair
|
|
4817
|
+
// yardstick and runs the frozen STRAND_CLUSTER_POLICY. The only count this node holds
|
|
4818
|
+
// is the party's enrolled machines (`authorizedControlPeers`), and passing that here
|
|
4819
|
+
// is the regression `bug-strand-yardstick-counts-party-machines` removed — a strand
|
|
4820
|
+
// runs on a subset of the party, and over-declaring makes repair impossible rather
|
|
4821
|
+
// than merely weak. The full argument, and the count that will legitimately go here,
|
|
4822
|
+
// are on `StartStrandConfig.servingMachines`.
|
|
3607
4823
|
backfill: this.config.strandBackfill,
|
|
3608
|
-
|
|
3609
|
-
|
|
4824
|
+
revocationEnforcement: this.config.strandRevocationEnforcement,
|
|
4825
|
+
membershipReconciliation: this.config.strandMembershipReconciliation,
|
|
4826
|
+
onSelfRevoked: (revokedStrandId) => this.emit('strand:revoked', { strandId: revokedStrandId }),
|
|
4827
|
+
// The joiner's first-sync write gate (strand-first-sync-gate.ts): a launch that
|
|
4828
|
+
// comes up `'syncing'` announces the moment its database is published.
|
|
4829
|
+
firstSync: this.config.strandFirstSync,
|
|
4830
|
+
onWritable: (writableStrandId) => this.handleStrandWritable(writableStrandId),
|
|
4831
|
+
// Re-announce the delegate to ONE relay before the strand node's reservation
|
|
4832
|
+
// supervisor re-drives it (see announceDelegateToRelay). Retained with the
|
|
4833
|
+
// launch config, so a hibernation wake's rebuilt supervisors get it too.
|
|
4834
|
+
announceDelegateToRelay: (announcedStrandId, relayAddr, announcedDelegatePeerId) => this.announceDelegateToRelay(announcedStrandId, relayAddr, announcedDelegatePeerId),
|
|
4835
|
+
// The staged formation invitation seam for the bring-up membership
|
|
4836
|
+
// reconciler: read lazily per pass (a re-formation replaces the entry) and
|
|
4837
|
+
// cleared once spent, burned, or dead — see pendingMembershipInvites.
|
|
4838
|
+
pendingMembershipInvite: {
|
|
4839
|
+
get: () => this.pendingMembershipInvites.get(strand.Id),
|
|
4840
|
+
clear: () => { this.pendingMembershipInvites.delete(strand.Id); }
|
|
4841
|
+
},
|
|
4842
|
+
// The RESOLVED flag, never the raw argument — see the doc comment above.
|
|
4843
|
+
founder: resolvedFounder,
|
|
4844
|
+
// Retained with the launch config, so a hibernation wake rebuilds under the
|
|
4845
|
+
// same identity without re-reading the control DB.
|
|
4846
|
+
partyMemberPrivateKey
|
|
4847
|
+
}));
|
|
3610
4848
|
// The seed reached the node as `bootstrapNodes`, which only enters the
|
|
3611
4849
|
// address book via @libp2p/bootstrap discovery. Merge it directly too, so a
|
|
3612
4850
|
// sibling is dialable by bare peer id from the first moment (see
|
|
3613
4851
|
// mergeStrandPeerAddrs; refreshStrandPeerAddrs keeps it warm from here on).
|
|
3614
|
-
|
|
3615
|
-
|
|
4852
|
+
const strandNode = instance.libp2pNode;
|
|
4853
|
+
if (strandNode) {
|
|
4854
|
+
await timed('mergeStrandPeerAddrs', () => this.mergeStrandPeerAddrs(strandNode, bootstrapNodes, strand.Id));
|
|
3616
4855
|
}
|
|
3617
4856
|
this.hibernationManager.trackStrand(instance);
|
|
3618
4857
|
this.emit('strand:started', { strandId: strand.Id });
|
|
@@ -3645,6 +4884,20 @@ export class CadreNode {
|
|
|
3645
4884
|
* for a relay we are not yet connected to.
|
|
3646
4885
|
*/
|
|
3647
4886
|
async resolveCohortSeed(strandId, delegatePeerId) {
|
|
4887
|
+
const siblings = await this.resolveSiblingSeed(strandId, delegatePeerId);
|
|
4888
|
+
// Sibling answers FIRST: they were resolved just now, while a contact-map entry is as
|
|
4889
|
+
// old as the formation that produced it. `unionAddrs` appends only what the siblings
|
|
4890
|
+
// did not already name, so a fresher answer is never displaced and each source keeps
|
|
4891
|
+
// its own signaling-first ordering.
|
|
4892
|
+
return unionAddrs(siblings, this.crossPartyStrandAddrs.get(strandId) ?? []);
|
|
4893
|
+
}
|
|
4894
|
+
/**
|
|
4895
|
+
* The own-party half of {@link resolveCohortSeed}: strand-network addresses resolved
|
|
4896
|
+
* from CONNECTED cohort siblings over the control-mesh strand-addr RPC, doubling as
|
|
4897
|
+
* the delegate announcement when `delegatePeerId` is given. Empty when the control DB
|
|
4898
|
+
* or node is absent (not yet started / torn down).
|
|
4899
|
+
*/
|
|
4900
|
+
async resolveSiblingSeed(strandId, delegatePeerId) {
|
|
3648
4901
|
if (!this.controlDatabase || !this.controlNode) {
|
|
3649
4902
|
return [];
|
|
3650
4903
|
}
|
|
@@ -3744,10 +4997,10 @@ export class CadreNode {
|
|
|
3744
4997
|
* Recorded OPTIMISTICALLY at announce time: `collectStrandAddrs` folds
|
|
3745
4998
|
* per-peer failure to `[]` and reports no per-peer success, and threading
|
|
3746
4999
|
* success out would change its API for little gain. A failed INITIAL announce
|
|
3747
|
-
*
|
|
3748
|
-
*
|
|
3749
|
-
*
|
|
3750
|
-
* (15 min), still inside the 30 min TTL.
|
|
5000
|
+
* costs the strand supervisor its first attempt only (the relay denies the
|
|
5001
|
+
* reservation; every re-drive re-announces first through
|
|
5002
|
+
* {@link announceDelegateToRelay}); a failed REFRESH retries within
|
|
5003
|
+
* `DELEGATE_GRANT_TTL_MS / 2` (15 min), still inside the 30 min TTL.
|
|
3751
5004
|
*/
|
|
3752
5005
|
recordDelegateAnnounces(relayPeerIds, strandId, now = Date.now()) {
|
|
3753
5006
|
for (const relayPeerId of relayPeerIds) {
|
|
@@ -3789,6 +5042,49 @@ export class CadreNode {
|
|
|
3789
5042
|
}
|
|
3790
5043
|
await Promise.all([...running].map(([strandId, delegatePeerId]) => this.announceDelegateToDueRelays(strandId, delegatePeerId, relays, now)));
|
|
3791
5044
|
}
|
|
5045
|
+
/**
|
|
5046
|
+
* Announce ONE strand's delegate peerId to ONE relay, UNTHROTTLED — the
|
|
5047
|
+
* `beforeRedrive` hook of that strand node's per-relay reservation supervisor
|
|
5048
|
+
* (`strand-instance-manager.ts` → `buildStrandRuntime`), run before every
|
|
5049
|
+
* re-drive after the first attempt.
|
|
5050
|
+
*
|
|
5051
|
+
* Why a re-drive must re-announce first: a party control node running the relay
|
|
5052
|
+
* server admits the strand node's derived peerId on an in-memory delegate grant
|
|
5053
|
+
* (`delegate-admission.ts`), and a relay restart drops every grant it held
|
|
5054
|
+
* without telling the announcer. {@link refreshDelegateGrants} would re-announce
|
|
5055
|
+
* on its own at most every `DELEGATE_GRANT_TTL_MS / 2` (15 min) per (relay,
|
|
5056
|
+
* strand) — far slower than the supervisor's backoff — so without this the first
|
|
5057
|
+
* re-drives after a relay restart would be denied at the relay's connection gate.
|
|
5058
|
+
* The throttle map is updated afterwards, so the periodic pass does not announce
|
|
5059
|
+
* again right away.
|
|
5060
|
+
*
|
|
5061
|
+
* Against a dedicated ops relay (no strand-addr RPC) the request fails per-peer
|
|
5062
|
+
* and `collectStrandAddrs` folds it to `[]`: one wasted protocol negotiation per
|
|
5063
|
+
* re-drive attempt, bounded by the supervisor's backoff. Never throws on that
|
|
5064
|
+
* path; a relay addr that names no peer id is logged and skipped (the hook's
|
|
5065
|
+
* caller runs the drive regardless).
|
|
5066
|
+
*
|
|
5067
|
+
* NOTE: against a relay that is DOWN this hook costs up to two strand-addr
|
|
5068
|
+
* timeouts (10 s each: dial by peer id, then by addr) before the 10 s drive even
|
|
5069
|
+
* starts, so one failed re-drive can hold the supervisor `driving` for ~30 s.
|
|
5070
|
+
* Bounded and harmless while the relay is unreachable anyway; if recovery
|
|
5071
|
+
* latency after a relay comes back ever matters, skip the announce when the
|
|
5072
|
+
* control node holds no connection to the relay (the drive's own dial fails
|
|
5073
|
+
* faster) rather than shortening the strand-addr timeout.
|
|
5074
|
+
*/
|
|
5075
|
+
async announceDelegateToRelay(strandId, relayAddr, delegatePeerId) {
|
|
5076
|
+
const controlNode = this.controlNode;
|
|
5077
|
+
if (!controlNode) {
|
|
5078
|
+
return;
|
|
5079
|
+
}
|
|
5080
|
+
const relayPeerId = trailingPeerId(multiaddr(relayAddr));
|
|
5081
|
+
if (relayPeerId === null) {
|
|
5082
|
+
log('announceDelegateToRelay: relay addr %s names no peer id; strand %s not announced', relayAddr, strandId);
|
|
5083
|
+
return;
|
|
5084
|
+
}
|
|
5085
|
+
await collectStrandAddrs(controlNode, [{ peerId: relayPeerId, addrs: [multiaddr(relayAddr)] }], strandId, { delegatePeerId });
|
|
5086
|
+
this.recordDelegateAnnounces([relayPeerId], strandId);
|
|
5087
|
+
}
|
|
3792
5088
|
/** One strand's share of {@link refreshDelegateGrants}: announce to the relays whose grant is due. */
|
|
3793
5089
|
async announceDelegateToDueRelays(strandId, delegatePeerId, relays, now) {
|
|
3794
5090
|
const controlNode = this.controlNode;
|
|
@@ -3851,35 +5147,41 @@ export class CadreNode {
|
|
|
3851
5147
|
if (due.length === 0) {
|
|
3852
5148
|
return;
|
|
3853
5149
|
}
|
|
3854
|
-
// A due strand with nobody to ask leaves its throttle unstamped (see below),
|
|
3855
|
-
// so this pass re-enters on EVERY 15 s reconcile tick for as long as the node
|
|
3856
|
-
// is alone — which is the steady state of a solo node running a strand.
|
|
3857
5150
|
// `connectedSiblingTargets`' membership read is unbounded, and with zero
|
|
3858
|
-
// connections its answer is empty whatever the table holds, so decide it
|
|
3859
|
-
//
|
|
3860
|
-
|
|
3861
|
-
|
|
3862
|
-
|
|
5151
|
+
// connections its answer is empty whatever the table holds, so decide it from the
|
|
5152
|
+
// connection list instead of paying for the read once a tick. Note this only skips
|
|
5153
|
+
// the RPC: a strand carrying cross-party contact addresses still refreshes below,
|
|
5154
|
+
// because those never came from a sibling in the first place.
|
|
5155
|
+
//
|
|
3863
5156
|
// NOTE: one strand-addr RPC per (due strand × connected sibling) per refresh
|
|
3864
5157
|
// interval — each a tiny request/response on an already-open control
|
|
3865
5158
|
// connection. If a node ever runs MANY strands at once, batch the RPC to
|
|
3866
5159
|
// carry several strand ids per request rather than one fan-out per strand.
|
|
3867
|
-
const targets =
|
|
3868
|
-
|
|
3869
|
-
|
|
3870
|
-
|
|
3871
|
-
|
|
3872
|
-
|
|
3873
|
-
//
|
|
3874
|
-
if (
|
|
5160
|
+
const targets = this.controlNode.getConnections().length === 0
|
|
5161
|
+
? []
|
|
5162
|
+
: await this.connectedSiblingTargets().catch((error) => {
|
|
5163
|
+
log('refreshStrandPeerAddrs: sibling enumeration failed (skipping pass): %o', error);
|
|
5164
|
+
return [];
|
|
5165
|
+
});
|
|
5166
|
+
// A shutdown landed mid-enumeration.
|
|
5167
|
+
if (!this._running || !this.controlNode) {
|
|
3875
5168
|
return;
|
|
3876
5169
|
}
|
|
3877
5170
|
await Promise.all(due.map((strandId) => this.refreshOneStrandPeerAddrs(strandId, targets, now)));
|
|
3878
5171
|
}
|
|
3879
5172
|
/**
|
|
3880
|
-
* One strand's share of {@link refreshStrandPeerAddrs}: RPC the siblings,
|
|
3881
|
-
*
|
|
3882
|
-
* swallowed so one strand's failure never
|
|
5173
|
+
* One strand's share of {@link refreshStrandPeerAddrs}: RPC the siblings, union their
|
|
5174
|
+
* answers with this strand's cross-party contact addresses, and merge the lot into the
|
|
5175
|
+
* strand's address book. Errors are logged and swallowed so one strand's failure never
|
|
5176
|
+
* costs the others their refresh.
|
|
5177
|
+
*
|
|
5178
|
+
* `targets` may be EMPTY — a solo node, or one whose only control peers are strangers.
|
|
5179
|
+
* The cross-party addresses still have to be re-merged in that case: they were learned
|
|
5180
|
+
* at formation and can never be re-resolved, so this periodic re-merge is the ONLY
|
|
5181
|
+
* thing standing between them and the peerStore's one-hour expiry. A strand with
|
|
5182
|
+
* neither a sibling to ask nor a contact entry does nothing and leaves its throttle
|
|
5183
|
+
* unstamped, so the next reconcile tick retries rather than sitting out the whole
|
|
5184
|
+
* refresh interval having done nothing.
|
|
3883
5185
|
*/
|
|
3884
5186
|
async refreshOneStrandPeerAddrs(strandId, targets, now) {
|
|
3885
5187
|
const controlNode = this.controlNode;
|
|
@@ -3887,13 +5189,19 @@ export class CadreNode {
|
|
|
3887
5189
|
if (!controlNode || !strandNode) {
|
|
3888
5190
|
return;
|
|
3889
5191
|
}
|
|
5192
|
+
const contactAddrs = this.crossPartyStrandAddrs.get(strandId) ?? [];
|
|
5193
|
+
if (targets.length === 0 && contactAddrs.length === 0) {
|
|
5194
|
+
return;
|
|
5195
|
+
}
|
|
3890
5196
|
try {
|
|
3891
5197
|
// The running strand node's own peerId is the delegate to announce — see
|
|
3892
5198
|
// the relationship note on refreshStrandPeerAddrs.
|
|
3893
|
-
const
|
|
3894
|
-
|
|
3895
|
-
|
|
3896
|
-
|
|
5199
|
+
const siblingAddrs = targets.length === 0
|
|
5200
|
+
? []
|
|
5201
|
+
: await collectStrandAddrs(controlNode, [...targets], strandId, {
|
|
5202
|
+
delegatePeerId: strandNode.peerId.toString()
|
|
5203
|
+
});
|
|
5204
|
+
// Stamped on the pass having happened, not on its answer: the fan-out is
|
|
3897
5205
|
// what the throttle exists to bound, and an all-empty round is a normal
|
|
3898
5206
|
// steady state for a strand no connected sibling currently runs.
|
|
3899
5207
|
this.strandPeerAddrRefreshAt.set(strandId, now);
|
|
@@ -3902,7 +5210,7 @@ export class CadreNode {
|
|
|
3902
5210
|
if (!this._running || this.strandManager.getInstance(strandId)?.libp2pNode !== strandNode) {
|
|
3903
5211
|
return;
|
|
3904
5212
|
}
|
|
3905
|
-
await this.mergeStrandPeerAddrs(strandNode,
|
|
5213
|
+
await this.mergeStrandPeerAddrs(strandNode, unionAddrs(siblingAddrs, contactAddrs), strandId);
|
|
3906
5214
|
}
|
|
3907
5215
|
catch (error) {
|
|
3908
5216
|
log('refreshStrandPeerAddrs: strand %s refresh failed (continuing): %o', strandId, error);
|
|
@@ -3926,6 +5234,15 @@ export class CadreNode {
|
|
|
3926
5234
|
* handshake, and a bad entry costs one failed dial that ages out at the
|
|
3927
5235
|
* peerStore's one-hour expiry. No new gating here — cross-party strand trust is
|
|
3928
5236
|
* `backlog/strand-network-nat-relay-reachability`.
|
|
5237
|
+
*
|
|
5238
|
+
* NOTE: one input does NOT age out — the cross-party addresses a formation
|
|
5239
|
+
* carried back ({@link crossPartyStrandAddrs}). Nothing can re-resolve them, so
|
|
5240
|
+
* {@link refreshStrandPeerAddrs} re-merges the same stored list every pass, which
|
|
5241
|
+
* also means a junk entry from a responder survives for the node's lifetime rather
|
|
5242
|
+
* than an hour. Bounded to {@link MAX_STRAND_ADDRS} per strand, from a party this
|
|
5243
|
+
* node deliberately formed a strand with, and still authority-free — so the
|
|
5244
|
+
* exposure is a handful of failed dials, not a trust hole. Evict on
|
|
5245
|
+
* `unpublishStrand` if that ever stops being an acceptable price.
|
|
3929
5246
|
*/
|
|
3930
5247
|
async mergeStrandPeerAddrs(strandNode, addrs, strandId) {
|
|
3931
5248
|
const counts = { merged: 0, restamped: 0, skipped: 0, failed: 0 };
|
|
@@ -3967,13 +5284,39 @@ export class CadreNode {
|
|
|
3967
5284
|
if (!node) {
|
|
3968
5285
|
return [];
|
|
3969
5286
|
}
|
|
3970
|
-
|
|
5287
|
+
// Every entry is bound to THIS strand node's id, because both consumers — the
|
|
5288
|
+
// strand-addr RPC answer and the formation result's `strandAddrs` — are attributed
|
|
5289
|
+
// per peer on arrival (`groupAddrsByPeerId`) and an entry naming no destination is
|
|
5290
|
+
// dropped there. libp2p already appends the id to its announced addresses, so this
|
|
5291
|
+
// is normally a no-op; it is the guard for a bare listen addr and for a relay hop
|
|
5292
|
+
// whose trailing `/p2p/` names the RELAY. An addr terminating in a DIFFERENT id is
|
|
5293
|
+
// dropped rather than announced — it does not reach this node, and announcing it
|
|
5294
|
+
// would file our address under someone else's id in the receiver's book.
|
|
5295
|
+
//
|
|
5296
|
+
// Re-parsed through this package's own `multiaddr` rather than passed straight in:
|
|
5297
|
+
// libp2p hands back its nested `@multiformats/multiaddr` copy, a structurally
|
|
5298
|
+
// different type from the one `withTrailingPeerId` takes (see
|
|
5299
|
+
// `backlog/reference-app-web-libp2p-interface-dedup`). Re-parsing a string libp2p
|
|
5300
|
+
// itself produced cannot fail.
|
|
5301
|
+
const selfId = node.peerId.toString();
|
|
5302
|
+
const addrs = [];
|
|
5303
|
+
for (const ma of node.getMultiaddrs()) {
|
|
5304
|
+
const bound = this.bindAddrToPeer(multiaddr(ma.toString()), selfId);
|
|
5305
|
+
if (bound !== null) {
|
|
5306
|
+
addrs.push(bound.toString());
|
|
5307
|
+
}
|
|
5308
|
+
}
|
|
5309
|
+
return orderSignalingFirst(addrs);
|
|
3971
5310
|
}
|
|
3972
5311
|
/**
|
|
3973
5312
|
* Stop a strand on THIS node only: untrack it from hibernation, drop its sApp config,
|
|
3974
5313
|
* stop the local instance, and emit `strand:stopped`. The shared `Strand` row is left
|
|
3975
|
-
* intact, so on the next node
|
|
3976
|
-
*
|
|
5314
|
+
* intact, so on the next node RESTART the strand is rediscovered and surfaces as
|
|
5315
|
+
* `strand:discovered` again — not on the next watcher poll, which never re-offers a
|
|
5316
|
+
* strand it has already seen (`StrandWatcher.knownStrands` retains the id for the life
|
|
5317
|
+
* of the process). The stop also drops the strand from {@link getDiscoveredStrands},
|
|
5318
|
+
* so a drain cannot undo a deliberate stop. Party-wide removal is
|
|
5319
|
+
* {@link unpublishStrand}.
|
|
3977
5320
|
*/
|
|
3978
5321
|
async stopStrand(strandId) {
|
|
3979
5322
|
if (!this._running) {
|
|
@@ -3983,20 +5326,28 @@ export class CadreNode {
|
|
|
3983
5326
|
}
|
|
3984
5327
|
/**
|
|
3985
5328
|
* Local teardown for one strand, shared by the caller-driven {@link stopStrand} and the
|
|
3986
|
-
* watcher-driven `handleStrandRemoved`: untrack hibernation, drop the sApp config
|
|
3987
|
-
* the instance, emit `strand:stopped`. Touches no
|
|
3988
|
-
* removal the node is on is the caller's concern,
|
|
5329
|
+
* watcher-driven `handleStrandRemoved`: untrack hibernation, drop the sApp config and any
|
|
5330
|
+
* recorded launch refusal, stop the instance, emit `strand:stopped`. Touches no
|
|
5331
|
+
* control-plane row — which side of the removal the node is on is the caller's concern,
|
|
5332
|
+
* not this method's.
|
|
3989
5333
|
*
|
|
3990
5334
|
* The stop + emit are skipped when the strand manager holds no instance for `strandId` —
|
|
3991
5335
|
* e.g. a party owner that published a strand's row but never ran it locally, or an
|
|
3992
5336
|
* explicit {@link stopStrand} for an id this node never started. `hibernationManager`
|
|
3993
|
-
* untrack and the `sAppConfigs`
|
|
3994
|
-
* nothing to remove), so a launch
|
|
3995
|
-
* gets its stray
|
|
5337
|
+
* untrack and the `sAppConfigs` / {@link strandLaunchRefusals} / {@link discoveredStrands}
|
|
5338
|
+
* deletes stay unconditional (all no-ops when there is nothing to remove), so a launch
|
|
5339
|
+
* that failed before an instance was ever tracked still gets its stray entries cleared.
|
|
5340
|
+
*
|
|
5341
|
+
* Dropping the {@link discoveredStrands} entry here covers both callers, and both
|
|
5342
|
+
* readings are the intended one: `handleStrandRemoved` arrives because the control row
|
|
5343
|
+
* is gone, and an explicit {@link stopStrand} is a deliberate abandonment — neither
|
|
5344
|
+
* strand may be re-offered to a later `getDiscoveredStrands()` drain.
|
|
3996
5345
|
*/
|
|
3997
5346
|
async detachStrand(strandId) {
|
|
3998
5347
|
this.hibernationManager.untrackStrand(strandId);
|
|
3999
5348
|
this.sAppConfigs.delete(strandId);
|
|
5349
|
+
this.strandLaunchRefusals.delete(strandId);
|
|
5350
|
+
this.discoveredStrands.delete(strandId);
|
|
4000
5351
|
if (!this.strandManager.hasStrand(strandId)) {
|
|
4001
5352
|
log('detachStrand: strand %s not tracked locally — no-op (no stop, no strand:stopped)', strandId);
|
|
4002
5353
|
return;
|
|
@@ -4212,6 +5563,11 @@ export class CadreNode {
|
|
|
4212
5563
|
* relay indistinguishable from a hung start, and much shorter would fail nodes
|
|
4213
5564
|
* on links that were merely slow. The retries carry on in the background after
|
|
4214
5565
|
* this resolves, exactly as they do for a {@link reserveRelays} caller.
|
|
5566
|
+
*
|
|
5567
|
+
* `network.requireRelay === false` softens only the outcome below: a first
|
|
5568
|
+
* attempt that lands no `/p2p-circuit` address is logged instead of thrown, and
|
|
5569
|
+
* `start()` carries on with the retry supervisor already running in the
|
|
5570
|
+
* background (the same supervisor {@link reserveRelays} always starts).
|
|
4215
5571
|
*/
|
|
4216
5572
|
async driveControlRelayReservation() {
|
|
4217
5573
|
const relayAddrs = this.config.network?.relayAddrs ?? [];
|
|
@@ -4222,6 +5578,11 @@ export class CadreNode {
|
|
|
4222
5578
|
const state = await this.reserveRelays([...relayAddrs]);
|
|
4223
5579
|
timing('[start] reserveRelays: %dms', Math.round(performance.now() - t0));
|
|
4224
5580
|
if (state.status !== 'reserved') {
|
|
5581
|
+
if (this.config.network?.requireRelay === false) {
|
|
5582
|
+
log('Relay reservation did not land on the first attempt (status: %s); ' +
|
|
5583
|
+
'continuing without it because network.requireRelay is false: %o', state.status, state);
|
|
5584
|
+
return;
|
|
5585
|
+
}
|
|
4225
5586
|
throw new RelayReservationFailedError(relayAddrs, state);
|
|
4226
5587
|
}
|
|
4227
5588
|
log('Reserved a relay slot: %o', state.circuitAddrs);
|
|
@@ -4357,6 +5718,7 @@ export class CadreNode {
|
|
|
4357
5718
|
partyId: this.config.controlNetwork.partyId,
|
|
4358
5719
|
ownerPrivateKey,
|
|
4359
5720
|
inviteAddressResolver: () => this.resolveInviteAddresses(),
|
|
5721
|
+
dialBudget: this.controlDialBudget(),
|
|
4360
5722
|
trustPolicy: this.config.seedTrustPolicy,
|
|
4361
5723
|
// Seed trust anchors on the node-local store (seeded just above with this
|
|
4362
5724
|
// node's own genesis key), never on the replicated OwnerKey table.
|
|
@@ -4596,6 +5958,7 @@ export class CadreNode {
|
|
|
4596
5958
|
partyId: this.config.controlNetwork.partyId,
|
|
4597
5959
|
// No owner key - this node only receives seeds
|
|
4598
5960
|
inviteAddressResolver: () => this.resolveInviteAddresses(),
|
|
5961
|
+
dialBudget: this.controlDialBudget(),
|
|
4599
5962
|
trustPolicy: this.config.seedTrustPolicy,
|
|
4600
5963
|
// A listener-only node accepts a wire-delivered seed solely against this
|
|
4601
5964
|
// anchor (there is no per-call override on the inbound handler): with no
|
|
@@ -4670,6 +6033,25 @@ export class CadreNode {
|
|
|
4670
6033
|
await this.membershipGateDrain;
|
|
4671
6034
|
}
|
|
4672
6035
|
}
|
|
6036
|
+
/**
|
|
6037
|
+
* Re-materialize a CLOSED strand's revoked-peer deny set now, and hang up any
|
|
6038
|
+
* connected peer it newly covers — the strand-side counterpart to
|
|
6039
|
+
* {@link refreshMembershipGate}.
|
|
6040
|
+
*
|
|
6041
|
+
* Unlike that one, NOTHING calls this automatically: strand membership is
|
|
6042
|
+
* written through the strand's own `Database` handle
|
|
6043
|
+
* (`strand-membership-writer.ts`), which raises no notification this runtime
|
|
6044
|
+
* can hook, and a revocation that arrives by REPLICATION raises none either.
|
|
6045
|
+
* The gate therefore polls (default 30 s). An app that has just called
|
|
6046
|
+
* `revokeMember` or `leaveStrand` should follow the write with this call so
|
|
6047
|
+
* the cut is immediate rather than up to one poll interval late.
|
|
6048
|
+
*
|
|
6049
|
+
* Quiet no-op for a strand that is not running, is quiesced, is open, or has
|
|
6050
|
+
* the gate disabled. Never rejects; resolves once the sweep has finished.
|
|
6051
|
+
*/
|
|
6052
|
+
async refreshRevocationEnforcement(strandId) {
|
|
6053
|
+
await this.strandManager.refreshRevocationEnforcement(strandId);
|
|
6054
|
+
}
|
|
4673
6055
|
/**
|
|
4674
6056
|
* The single in-flight refresh loop behind {@link refreshMembershipGate}: read
|
|
4675
6057
|
* until the stale flag stays clear, then release the slot.
|
|
@@ -4744,6 +6126,10 @@ export class CadreNode {
|
|
|
4744
6126
|
await this.seedBootstrapService.authorizePeer({ peerId, multiaddrs });
|
|
4745
6127
|
// Queue for re-replication if this committed local-only (no connected cohort).
|
|
4746
6128
|
this.noteControlWrite(peerId, 'authorize');
|
|
6129
|
+
// NOTE: unlike addDrone, `multiaddrs` is not retained as a dial target: the
|
|
6130
|
+
// flows that vouch through here have the new peer reach this node (it applies a
|
|
6131
|
+
// seed naming the owners, or it dialed in with an invite). If one ever vouches a
|
|
6132
|
+
// peer that cannot reach this node, retain its addresses as addDrone does.
|
|
4747
6133
|
}
|
|
4748
6134
|
/**
|
|
4749
6135
|
* Remove a previously-authorized peer from the cadre.
|
|
@@ -4760,6 +6146,13 @@ export class CadreNode {
|
|
|
4760
6146
|
await this.seedBootstrapService.removePeer(peerId);
|
|
4761
6147
|
// Track + loudly flag a delete that committed local-only (security-relevant).
|
|
4762
6148
|
this.noteControlWrite(peerId, 'remove');
|
|
6149
|
+
// A removed peer is no longer a dial target, however this node learned its
|
|
6150
|
+
// address. Gone from the store synchronously; a persist failure only lets the
|
|
6151
|
+
// entry reappear after a restart, so it is logged rather than failing a
|
|
6152
|
+
// removal that has already committed.
|
|
6153
|
+
void this.bootstrapPeerStore?.forget(peerId).catch((error) => {
|
|
6154
|
+
log('removePeer: persisting the removal of dial target %s failed (forgotten in memory): %o', peerId, error);
|
|
6155
|
+
});
|
|
4763
6156
|
}
|
|
4764
6157
|
/**
|
|
4765
6158
|
* Create a seed from the current control network state.
|
|
@@ -4796,6 +6189,7 @@ export class CadreNode {
|
|
|
4796
6189
|
const tempService = new SeedBootstrapService({
|
|
4797
6190
|
partyId: seed.partyId,
|
|
4798
6191
|
trustPolicy: this.config.seedTrustPolicy,
|
|
6192
|
+
dialBudget: this.controlDialBudget(),
|
|
4799
6193
|
// The anchor is node-scoped, not service-scoped: a throwaway service
|
|
4800
6194
|
// must consult (and persist an accepted signer into) the SAME store the
|
|
4801
6195
|
// persistent one would, or a cold-start enrollment via this path would
|
|
@@ -4876,6 +6270,20 @@ export class CadreNode {
|
|
|
4876
6270
|
/**
|
|
4877
6271
|
* Add a drone to the cadre (for phone/server adding provider-hosted node).
|
|
4878
6272
|
* Creates authorization and seed for drone initialization.
|
|
6273
|
+
*
|
|
6274
|
+
* Also retains the drone's handed-over addresses as a durable dial target (see
|
|
6275
|
+
* {@link bootstrapPeerStore}). The drone cannot dial an owner that does not
|
|
6276
|
+
* listen (a phone), and its `CadrePeer` row stays unsigned — so unresolvable —
|
|
6277
|
+
* until it self-publishes over a connection; this node therefore has to open
|
|
6278
|
+
* that connection from the addresses it was handed, on this launch or a later
|
|
6279
|
+
* one.
|
|
6280
|
+
*
|
|
6281
|
+
* Nothing is dialed here: the drone has not received the seed yet. After
|
|
6282
|
+
* delivering it, call {@link reconcileControlCohort} to dial straight away;
|
|
6283
|
+
* otherwise the next timed reconcile pass does. A pass already in flight is
|
|
6284
|
+
* joined rather than restarted, and one that listed siblings before this add
|
|
6285
|
+
* does not dial the drone, so a caller waiting for the connection should allow
|
|
6286
|
+
* for one more timed pass.
|
|
4879
6287
|
*/
|
|
4880
6288
|
async addDrone(options) {
|
|
4881
6289
|
if (!this.seedBootstrapService) {
|
|
@@ -4883,7 +6291,12 @@ export class CadreNode {
|
|
|
4883
6291
|
}
|
|
4884
6292
|
// The drone's `CadrePeer` insert notifies the membership hub on the way to the
|
|
4885
6293
|
// seed, so the per-stream gate already admits it here.
|
|
4886
|
-
|
|
6294
|
+
const result = await this.seedBootstrapService.addDrone(options);
|
|
6295
|
+
// Same queueing as authorizePeer: a phone adding its first always-on node has
|
|
6296
|
+
// no control connection, so the insert usually committed local-only.
|
|
6297
|
+
this.noteControlWrite(options.dronePeerId, 'authorize');
|
|
6298
|
+
this.retainDialTarget(options.dronePeerId, options.droneMultiaddrs, 'addDrone');
|
|
6299
|
+
return result;
|
|
4887
6300
|
}
|
|
4888
6301
|
/**
|
|
4889
6302
|
* Create an invite for a phone to join the cadre.
|
|
@@ -4956,6 +6369,7 @@ export class CadreNode {
|
|
|
4956
6369
|
const tempService = new SeedBootstrapService({
|
|
4957
6370
|
partyId: invite.partyId,
|
|
4958
6371
|
trustPolicy: this.config.seedTrustPolicy,
|
|
6372
|
+
dialBudget: this.controlDialBudget(),
|
|
4959
6373
|
});
|
|
4960
6374
|
if (this.controlNode && this.controlDatabase) {
|
|
4961
6375
|
tempService.initialize(this.controlNode, this.controlDatabase, { registerHandler: false });
|
|
@@ -4981,7 +6395,15 @@ export class CadreNode {
|
|
|
4981
6395
|
this.strandSolicitationService = new StrandSolicitationService({
|
|
4982
6396
|
...options,
|
|
4983
6397
|
partyId: this.config.controlNetwork.partyId,
|
|
4984
|
-
cadrePeerAddrs: this.getMultiaddrs()
|
|
6398
|
+
cadrePeerAddrs: this.getMultiaddrs(),
|
|
6399
|
+
// Overrides any caller-supplied hook, exactly like partyId/cadrePeerAddrs: only
|
|
6400
|
+
// this node can say which strand-network addresses it is actually listening on,
|
|
6401
|
+
// and a wrong answer here seeds a joiner's mesh with addresses that reach nobody.
|
|
6402
|
+
resolveStrandAddrs: (strandId) => this.getStrandMultiaddrs(strandId),
|
|
6403
|
+
// Overridden on the same grounds: only this node holds the running strand
|
|
6404
|
+
// instance's database and this party's `StrandPartyKey` identity, and a wrong
|
|
6405
|
+
// issuer here would admit joiners under someone else's authority.
|
|
6406
|
+
issueMembershipInvite: (strandId) => this.issueStrandMembershipInvite(strandId)
|
|
4985
6407
|
});
|
|
4986
6408
|
// Register as responder on the control node
|
|
4987
6409
|
this.strandSolicitationService.registerResponder(this.controlNode);
|
|
@@ -5026,7 +6448,176 @@ export class CadreNode {
|
|
|
5026
6448
|
if (!this.strandSolicitationService) {
|
|
5027
6449
|
this.initializeStrandSolicitation();
|
|
5028
6450
|
}
|
|
5029
|
-
|
|
6451
|
+
const result = await this.strandSolicitationService.formStrand(invitation, disclosure, this.controlNode);
|
|
6452
|
+
this.recordCrossPartyStrandAddrs(result.strandId, result.strandAddrs);
|
|
6453
|
+
// A closed host strand's approval carries the joiner's own membership invitation —
|
|
6454
|
+
// persist this party's identity key and stage the invitation for bring-up to redeem.
|
|
6455
|
+
// Runs AFTER the addr recording so a persistence failure (which throws — see the
|
|
6456
|
+
// method) still leaves the cross-party seed in place for a manual recovery.
|
|
6457
|
+
if (result.membershipInvite) {
|
|
6458
|
+
await this.adoptFormationMembershipInvite(result.strandId, result.membershipInvite);
|
|
6459
|
+
}
|
|
6460
|
+
return result;
|
|
6461
|
+
}
|
|
6462
|
+
/**
|
|
6463
|
+
* Adopt an approved closed-strand formation's membership half on the JOINER:
|
|
6464
|
+
*
|
|
6465
|
+
* 1. Mint-or-reuse this party's own strand identity — {@link ensureStrandPartyKey}
|
|
6466
|
+
* with no explicit key returns the stored `StrandPartyKey` row when one exists (a
|
|
6467
|
+
* re-formation by an existing member party: lost addresses, an app reinstall with
|
|
6468
|
+
* an intact control DB) and mints + persists a fresh one otherwise. The invitation
|
|
6469
|
+
* will admit THIS key's public half as the `Strand.Member`.
|
|
6470
|
+
* 2. Stage the invitation in {@link pendingMembershipInvites} for the strand
|
|
6471
|
+
* bring-up flow (`strand-node-binds-member-peer`) to redeem via `consumeInvite`.
|
|
6472
|
+
*
|
|
6473
|
+
* Throws — failing the whole {@link formStrand} — when the identity cannot be
|
|
6474
|
+
* persisted: a joiner "joined" without a persistable identity could never become a
|
|
6475
|
+
* member, and failing loudly beats a silent half-member. The formation itself has
|
|
6476
|
+
* already succeeded by then and its one-time token is SPENT, so the error says so:
|
|
6477
|
+
* recovery is fixing the underlying cause (no owner signing key / control DB write
|
|
6478
|
+
* rejected) and redeeming a FRESH invitation.
|
|
6479
|
+
*/
|
|
6480
|
+
async adoptFormationMembershipInvite(strandId, invite) {
|
|
6481
|
+
try {
|
|
6482
|
+
await this.ensureStrandPartyKey(strandId);
|
|
6483
|
+
}
|
|
6484
|
+
catch (error) {
|
|
6485
|
+
throw new Error(`Formation for strand ${strandId} was approved (its one-time token is spent), but ` +
|
|
6486
|
+
'persisting this party\'s membership identity (StrandPartyKey) failed — the joiner ' +
|
|
6487
|
+
'cannot become a member without it. Fix the underlying cause, then redeem a fresh ' +
|
|
6488
|
+
'invitation (the delivered one dies with this error).', { cause: error });
|
|
6489
|
+
}
|
|
6490
|
+
// NOTE: staging alone is not enough on a node whose reconciler for this strand has
|
|
6491
|
+
// already finished. `StrandMembershipReconciler` latches a terminal stopped state on
|
|
6492
|
+
// its `done` path and is rebuilt only by a strand relaunch, so a RE-formation against
|
|
6493
|
+
// an already-launched strand stages an invitation no loop will ever look at. Harmless
|
|
6494
|
+
// for a party that is still a member (the invitation would only be burned), and the
|
|
6495
|
+
// live defect for one that has been REMOVED — which is exactly when an app re-forms.
|
|
6496
|
+
// Tracked as `backlog/bug-removed-party-cannot-redeem-its-way-back`; pinned by
|
|
6497
|
+
// `strand-party-removal-via-formation-e2e.integration.ts` (test 2).
|
|
6498
|
+
this.pendingMembershipInvites.set(strandId, invite);
|
|
6499
|
+
log('formStrand: staged membership invitation for strand %s (party key persisted)', strandId);
|
|
6500
|
+
}
|
|
6501
|
+
/**
|
|
6502
|
+
* The pending single-use membership invitation a closed-strand formation carried back
|
|
6503
|
+
* for `strandId`, or `undefined` when none is staged — the seam the strand bring-up
|
|
6504
|
+
* membership reconciler (`strand-membership-reconciler.ts`) reads to redeem the
|
|
6505
|
+
* joiner's `Strand.Member` seat. `undefined` therefore also means "already redeemed,
|
|
6506
|
+
* burned, or found dead": the reconciler clears the entry as soon as it settles the
|
|
6507
|
+
* invitation, so a caller polling this sees it disappear on its own. In-memory only;
|
|
6508
|
+
* see {@link pendingMembershipInvites} for lifetime and re-formation semantics.
|
|
6509
|
+
*/
|
|
6510
|
+
getPendingMembershipInvite(strandId) {
|
|
6511
|
+
return this.pendingMembershipInvites.get(strandId);
|
|
6512
|
+
}
|
|
6513
|
+
/**
|
|
6514
|
+
* Responder-side issuer behind the formation manager's `issueMembershipInvite` seam
|
|
6515
|
+
* (see `StrandFormationManagerOptions.issueMembershipInvite` for the contract this
|
|
6516
|
+
* implements): mint a single-use `Strand.Invite` against the LIVE host strand so a
|
|
6517
|
+
* validated joiner can seat its own `Strand.Member` row.
|
|
6518
|
+
*
|
|
6519
|
+
* - Open host strand → `null` (no members, nothing to invite into).
|
|
6520
|
+
* - Closed host strand whose founder launch was refused as pre-split
|
|
6521
|
+
* ({@link strandLaunchRefusals}) → rethrow the recorded `PreSplitStrandIdentityError`;
|
|
6522
|
+
* the manager maps it to the NON-retryable `HOST_STRAND_MUST_BE_RECREATED_REASON`.
|
|
6523
|
+
* - Closed host strand with no `StrandPartyKey` row → throw: this party's identity is
|
|
6524
|
+
* the invite's issuing manager, and without it nothing can sign the issuance. (The
|
|
6525
|
+
* founder's publish/launch paths mint it, so this is a not-yet-converged sibling.)
|
|
6526
|
+
* The manager maps the throw to a clean retryable rejection BEFORE the formation
|
|
6527
|
+
* token is spent.
|
|
6528
|
+
* - Closed host strand with no running local instance/database → throw, same mapping:
|
|
6529
|
+
* a joiner admitted without an invitation would look joined and never become a
|
|
6530
|
+
* member, and a responder not running the strand cannot serve its sync anyway.
|
|
6531
|
+
* - Closed host strand whose LIVE rows carry the pre-split fingerprint
|
|
6532
|
+
* (`assertNotPreSplitStrand`) → throw `PreSplitStrandIdentityError`, same mapping as
|
|
6533
|
+
* the recorded refusal. Covers the responders that never ran a refused founder launch:
|
|
6534
|
+
* a sibling machine of the founding party, or a node restarted since the refusal.
|
|
6535
|
+
*
|
|
6536
|
+
* The recorded refusal is checked first: it is an in-memory read and the only
|
|
6537
|
+
* permanent diagnosis. Identity is checked BEFORE the runtime: it is the cheaper read
|
|
6538
|
+
* and the more actionable diagnosis when both are missing (a missing runtime is
|
|
6539
|
+
* transient, a missing identity is not), and it keeps the branch reachable without
|
|
6540
|
+
* standing a strand runtime up.
|
|
6541
|
+
*
|
|
6542
|
+
* The invitation expires `MEMBERSHIP_INVITE_TTL_MS` from now — see that constant for
|
|
6543
|
+
* the slow-joiner / lost-result tradeoff.
|
|
6544
|
+
*
|
|
6545
|
+
* NOTE: the issuing identity must be a `Strand.Manager` (the schema's `InviteValid`
|
|
6546
|
+
* gate), and only the FOUNDING party's key is seated as one. Today only the founder
|
|
6547
|
+
* party can host a bound formation at all — a joining party never gets the host
|
|
6548
|
+
* strand's control `Strand` row, so `resolveStrand` reports `missing` on it — so this
|
|
6549
|
+
* never bites. If a joined party is ever able to host formations into a strand it
|
|
6550
|
+
* joined (re-invite / multi-hop join), issuance here fails the manager gate and every
|
|
6551
|
+
* such redemption rejects with the retryable-sounding
|
|
6552
|
+
* `MEMBERSHIP_INVITE_UNAVAILABLE_REASON` forever; that flow needs manager delegation,
|
|
6553
|
+
* not a retry.
|
|
6554
|
+
*/
|
|
6555
|
+
async issueStrandMembershipInvite(strandId) {
|
|
6556
|
+
if (!this.controlDatabase) {
|
|
6557
|
+
throw new Error(`Cannot issue a membership invitation for strand ${strandId}: control database unavailable`);
|
|
6558
|
+
}
|
|
6559
|
+
const row = await this.controlDatabase.queryStrand(strandId);
|
|
6560
|
+
if (!row) {
|
|
6561
|
+
// The bound resolution saw this row moments ago; a vanished row is a concurrent
|
|
6562
|
+
// unpublish — reject rather than invite into a strand this party just removed.
|
|
6563
|
+
throw new Error(`Cannot issue a membership invitation for strand ${strandId}: its Strand row is gone`);
|
|
6564
|
+
}
|
|
6565
|
+
if (row.Type !== 'c') {
|
|
6566
|
+
return null;
|
|
6567
|
+
}
|
|
6568
|
+
const refusal = this.strandLaunchRefusals.get(strandId);
|
|
6569
|
+
if (refusal) {
|
|
6570
|
+
throw refusal;
|
|
6571
|
+
}
|
|
6572
|
+
const partyKey = await this.controlDatabase.queryStrandPartyKey(strandId);
|
|
6573
|
+
if (partyKey === null) {
|
|
6574
|
+
throw new Error(`Cannot issue a membership invitation for closed strand ${strandId}: this party holds ` +
|
|
6575
|
+
'no StrandPartyKey row for it (identity not yet converged from the machine that published it)');
|
|
6576
|
+
}
|
|
6577
|
+
const db = this.strandManager.getInstance(strandId)?.database?.getDatabase();
|
|
6578
|
+
if (!db) {
|
|
6579
|
+
throw new Error(`Cannot issue a membership invitation for closed strand ${strandId}: its runtime is ` +
|
|
6580
|
+
'not live on this responder (not launched, hibernating, or quiescing)');
|
|
6581
|
+
}
|
|
6582
|
+
if (row.MemberPrivateKey) {
|
|
6583
|
+
await assertNotPreSplitStrand(db, strandId, strandMemberKeyPair(row.MemberPrivateKey).publicKeyB64);
|
|
6584
|
+
}
|
|
6585
|
+
return await issueInvite(db, {
|
|
6586
|
+
managerKeyPair: strandMemberKeyPair(partyKey),
|
|
6587
|
+
expiration: Date.now() + MEMBERSHIP_INVITE_TTL_MS
|
|
6588
|
+
});
|
|
6589
|
+
}
|
|
6590
|
+
/**
|
|
6591
|
+
* Remember the responder's strand-network addresses for a strand this node just
|
|
6592
|
+
* formed, so the strand's discovery seed has something to work with when the app
|
|
6593
|
+
* launches it (see {@link crossPartyStrandAddrs}).
|
|
6594
|
+
*
|
|
6595
|
+
* Scoped strictly to `strandId`: these addresses reach ONE strand node of ONE other
|
|
6596
|
+
* party and must never seed another strand's mesh or the control peerStore. An empty
|
|
6597
|
+
* list is recorded as no entry at all rather than an empty one, so a later formation
|
|
6598
|
+
* against the same strand that DOES disclose addresses is not shadowed.
|
|
6599
|
+
*/
|
|
6600
|
+
recordCrossPartyStrandAddrs(strandId, strandAddrs) {
|
|
6601
|
+
if (strandAddrs.length === 0) {
|
|
6602
|
+
log('formStrand: responder disclosed no strand addrs for %s — cross-party seed stays empty', strandId);
|
|
6603
|
+
return;
|
|
6604
|
+
}
|
|
6605
|
+
// Capped at the same `MAX_STRAND_ADDRS` one arriving list is, because this
|
|
6606
|
+
// ACCUMULATES: re-forming against the same strand (the documented recovery for a
|
|
6607
|
+
// responder whose relay reservation rotated) unions a fresh list onto the old one,
|
|
6608
|
+
// and nothing here can tell a dead entry from a live one. Without the cap, the seed
|
|
6609
|
+
// — and the peerStore rows every refresh pass re-merges from it — would grow by up
|
|
6610
|
+
// to 16 entries per redemption for the node's lifetime. Oldest entries fall off the
|
|
6611
|
+
// end, which is the right end to lose: `unionAddrs` keeps the freshest list first.
|
|
6612
|
+
const merged = unionAddrs(strandAddrs, this.crossPartyStrandAddrs.get(strandId) ?? [])
|
|
6613
|
+
.slice(0, MAX_STRAND_ADDRS);
|
|
6614
|
+
this.crossPartyStrandAddrs.set(strandId, merged);
|
|
6615
|
+
// A strand already running when this lands (a re-formation) would otherwise wait out
|
|
6616
|
+
// the rest of its ~10-minute refresh throttle before the new addresses reached its
|
|
6617
|
+
// address book. Clearing the stamp makes the next 15 s reconcile tick merge them,
|
|
6618
|
+
// which matters precisely because re-forming is the recovery path for a dead entry.
|
|
6619
|
+
this.strandPeerAddrRefreshAt.delete(strandId);
|
|
6620
|
+
log('formStrand: recorded %d cross-party strand addr(s) for %s', merged.length, strandId);
|
|
5030
6621
|
}
|
|
5031
6622
|
/**
|
|
5032
6623
|
* Encode an open invitation for out-of-band delivery (QR, link, etc.).
|