@serfab/cadre-core 0.12.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -8
- package/dist/bootstrap-peer-store-file.js +2 -2
- package/dist/bootstrap-peer-store.d.ts +32 -12
- package/dist/bootstrap-peer-store.js +41 -15
- package/dist/bootstrap-peer-store.js.map +1 -1
- package/dist/cadre-node.d.ts +883 -102
- package/dist/cadre-node.js +1800 -209
- package/dist/cadre-node.js.map +1 -1
- package/dist/control-authorization.d.ts +2 -2
- package/dist/control-authorization.js +1 -0
- package/dist/control-authorization.js.map +1 -1
- package/dist/control-cohort.d.ts +13 -26
- package/dist/control-cohort.js +0 -26
- package/dist/control-cohort.js.map +1 -1
- package/dist/control-database.d.ts +318 -44
- package/dist/control-database.js +516 -74
- package/dist/control-database.js.map +1 -1
- package/dist/control-read-retry.d.ts +40 -11
- package/dist/control-read-retry.js +48 -15
- package/dist/control-read-retry.js.map +1 -1
- package/dist/control-retry.d.ts +58 -3
- package/dist/control-retry.js +58 -3
- package/dist/control-retry.js.map +1 -1
- package/dist/control-schema.d.ts +1 -1
- package/dist/control-schema.js +133 -21
- package/dist/control-schema.js.map +1 -1
- package/dist/control-write-retry.d.ts +80 -11
- package/dist/control-write-retry.js +168 -29
- package/dist/control-write-retry.js.map +1 -1
- package/dist/enrolled-machine-store-file.d.ts +13 -0
- package/dist/enrolled-machine-store-file.js +41 -0
- package/dist/enrolled-machine-store-file.js.map +1 -0
- package/dist/enrolled-machine-store.d.ts +81 -0
- package/dist/enrolled-machine-store.js +225 -0
- package/dist/enrolled-machine-store.js.map +1 -0
- package/dist/file-durable-slot.js +5 -3
- package/dist/file-durable-slot.js.map +1 -1
- package/dist/index.d.ts +17 -8
- package/dist/index.js +54 -9
- package/dist/index.js.map +1 -1
- package/dist/membership-connection-gater.d.ts +15 -1
- package/dist/membership-connection-gater.js +9 -3
- package/dist/membership-connection-gater.js.map +1 -1
- package/dist/node-local-snapshot.d.ts +21 -7
- package/dist/node-local-snapshot.js +30 -9
- package/dist/node-local-snapshot.js.map +1 -1
- package/dist/peer-dial.d.ts +143 -0
- package/dist/peer-dial.js +176 -0
- package/dist/peer-dial.js.map +1 -0
- package/dist/{strand-backfill.d.ts → peer-join-backfill.d.ts} +49 -20
- package/dist/{strand-backfill.js → peer-join-backfill.js} +133 -52
- package/dist/peer-join-backfill.js.map +1 -0
- package/dist/relay-addrs.d.ts +160 -49
- package/dist/relay-addrs.js +287 -60
- package/dist/relay-addrs.js.map +1 -1
- package/dist/relay-reservation.d.ts +71 -3
- package/dist/relay-reservation.js +116 -8
- package/dist/relay-reservation.js.map +1 -1
- package/dist/seed-bootstrap.d.ts +30 -1
- package/dist/seed-bootstrap.js +62 -17
- package/dist/seed-bootstrap.js.map +1 -1
- package/dist/storage-scope.d.ts +62 -0
- package/dist/storage-scope.js +75 -0
- package/dist/storage-scope.js.map +1 -0
- package/dist/strand-database.d.ts +41 -11
- package/dist/strand-database.js +48 -18
- package/dist/strand-database.js.map +1 -1
- package/dist/strand-first-sync-gate.d.ts +152 -0
- package/dist/strand-first-sync-gate.js +208 -0
- package/dist/strand-first-sync-gate.js.map +1 -0
- package/dist/strand-formation-manager.d.ts +99 -5
- package/dist/strand-formation-manager.js +119 -8
- package/dist/strand-formation-manager.js.map +1 -1
- package/dist/strand-formation-protocol.d.ts +97 -3
- package/dist/strand-formation-protocol.js +108 -2
- package/dist/strand-formation-protocol.js.map +1 -1
- package/dist/strand-instance-manager.d.ts +389 -19
- package/dist/strand-instance-manager.js +678 -65
- package/dist/strand-instance-manager.js.map +1 -1
- package/dist/strand-member-key.d.ts +20 -12
- package/dist/strand-member-key.js +20 -12
- package/dist/strand-member-key.js.map +1 -1
- package/dist/strand-member-registry.js +3 -0
- package/dist/strand-member-registry.js.map +1 -1
- package/dist/strand-membership-reconciler.d.ts +361 -0
- package/dist/strand-membership-reconciler.js +548 -0
- package/dist/strand-membership-reconciler.js.map +1 -0
- package/dist/strand-membership-writer.d.ts +143 -11
- package/dist/strand-membership-writer.js +419 -135
- package/dist/strand-membership-writer.js.map +1 -1
- package/dist/strand-network-config.d.ts +116 -0
- package/dist/strand-network-config.js +192 -0
- package/dist/strand-network-config.js.map +1 -0
- package/dist/strand-revocation-enforcer.d.ts +392 -0
- package/dist/strand-revocation-enforcer.js +533 -0
- package/dist/strand-revocation-enforcer.js.map +1 -0
- package/dist/strand-solicitation.d.ts +17 -1
- package/dist/strand-solicitation.js +13 -2
- package/dist/strand-solicitation.js.map +1 -1
- package/dist/strand-wake-protocol.d.ts +11 -7
- package/dist/strand-wake-protocol.js +20 -52
- package/dist/strand-wake-protocol.js.map +1 -1
- package/dist/strand-watcher.js +12 -1
- package/dist/strand-watcher.js.map +1 -1
- package/dist/timed-step.d.ts +12 -0
- package/dist/timed-step.js +28 -0
- package/dist/timed-step.js.map +1 -0
- package/dist/timeout-scheduler.d.ts +19 -0
- package/dist/timeout-scheduler.js +22 -0
- package/dist/timeout-scheduler.js.map +1 -0
- package/dist/types.d.ts +472 -46
- package/dist/types.js +7 -1
- package/dist/types.js.map +1 -1
- package/package.json +16 -8
- package/src/bootstrap-peer-store-file.ts +2 -2
- package/src/bootstrap-peer-store.ts +69 -27
- package/src/cadre-node.ts +1926 -228
- package/src/control-authorization.ts +2 -1
- package/src/control-cohort.ts +122 -130
- package/src/control-database.ts +586 -74
- package/src/control-read-retry.ts +202 -167
- package/src/control-retry.ts +297 -190
- package/src/control-schema.ts +133 -21
- package/src/control-write-retry.ts +525 -383
- package/src/enrolled-machine-store-file.ts +43 -0
- package/src/enrolled-machine-store.ts +280 -0
- package/src/file-durable-slot.ts +5 -3
- package/src/index.ts +134 -15
- package/src/membership-connection-gater.ts +9 -3
- package/src/node-local-snapshot.ts +38 -9
- package/src/peer-addr-book.ts +251 -250
- package/src/peer-dial.ts +215 -0
- package/src/{strand-backfill.ts → peer-join-backfill.ts} +478 -389
- package/src/relay-addrs.ts +458 -218
- package/src/relay-reservation.ts +951 -828
- package/src/seed-bootstrap.ts +76 -18
- package/src/storage-scope.ts +78 -0
- package/src/strand-database.ts +61 -21
- package/src/strand-first-sync-gate.ts +258 -0
- package/src/strand-formation-manager.ts +162 -8
- package/src/strand-formation-protocol.ts +850 -692
- package/src/strand-instance-manager.ts +877 -74
- package/src/strand-member-key.ts +20 -12
- package/src/strand-member-registry.ts +3 -0
- package/src/strand-membership-reconciler.ts +680 -0
- package/src/strand-membership-writer.ts +550 -190
- package/src/strand-network-config.ts +250 -0
- package/src/strand-revocation-enforcer.ts +653 -0
- package/src/strand-solicitation.ts +499 -471
- package/src/strand-wake-protocol.ts +21 -66
- package/src/strand-watcher.ts +325 -314
- package/src/timed-step.ts +28 -0
- package/src/timeout-scheduler.ts +28 -0
- package/src/types.ts +487 -52
- package/dist/strand-backfill.js.map +0 -1
package/src/cadre-node.ts
CHANGED
|
@@ -11,6 +11,8 @@ import type {
|
|
|
11
11
|
StrandInstance,
|
|
12
12
|
StrandRow,
|
|
13
13
|
StrandConfig,
|
|
14
|
+
FoundStrandConfig,
|
|
15
|
+
FoundStrandResult,
|
|
14
16
|
SAppConfig,
|
|
15
17
|
CadreNodeEvents,
|
|
16
18
|
ControlNetworkSeed,
|
|
@@ -23,6 +25,7 @@ import type {
|
|
|
23
25
|
OpenInvitation,
|
|
24
26
|
FormStrandResult,
|
|
25
27
|
StrandFormationDisclosure,
|
|
28
|
+
StrandMembershipInvite,
|
|
26
29
|
ResolveOpts,
|
|
27
30
|
SelfRegistrationOutcome,
|
|
28
31
|
ServiceWakeResult,
|
|
@@ -34,14 +37,19 @@ import type {
|
|
|
34
37
|
PeerAddressRecord,
|
|
35
38
|
ResolveDeviceTokenOpts
|
|
36
39
|
} from './types.js';
|
|
37
|
-
import {
|
|
40
|
+
import { controlClusterPolicy, CONTROL_REPLICATION_BREADTH, DEFAULT_CHECKIN_WINDOW_MS } from './types.js';
|
|
38
41
|
import { sign } from '@optimystic/quereus-plugin-crypto';
|
|
39
42
|
import { ed25519KeyPairFromLibp2p, ed25519PublicKeyFromPrivate, requireEd25519PublicKeyB64, type Ed25519KeyPair } from './ed25519-key.js';
|
|
40
43
|
import { strandTransportKey } from './strand-transport-key.js';
|
|
44
|
+
import { controlStorageScope } from './storage-scope.js';
|
|
45
|
+
import { generateStrandMemberKey, strandMemberKeyPair } from './strand-member-key.js';
|
|
46
|
+
import { assertNotPreSplitStrand, issueInvite, PreSplitStrandIdentityError } from './strand-membership-writer.js';
|
|
47
|
+
import { MEMBERSHIP_INVITE_TTL_MS } from './strand-formation-manager.js';
|
|
41
48
|
import { DEFAULT_IDENTITY_KEY_ID } from './key-store.js';
|
|
42
49
|
import { loadOrCreateIdentityKey } from './identity-key.js';
|
|
43
50
|
import { MemoryTrustedOwnerStore, type TrustedOwnerStore, type TrustSource } from './trusted-owner-store.js';
|
|
44
51
|
import { MemoryBootstrapPeerStore, type BootstrapPeerStore } from './bootstrap-peer-store.js';
|
|
52
|
+
import { MemoryEnrolledMachineStore, type EnrolledMachineStore } from './enrolled-machine-store.js';
|
|
45
53
|
import { mergePeerAddrs, groupAddrsByPeerId, type MergeAddrsResult } from './peer-addr-book.js';
|
|
46
54
|
import { verifyCadrePeerVoucher } from './peer-authorization.js';
|
|
47
55
|
import { ed25519PublicKeyB64FromPeerId } from './seed-bootstrap.js';
|
|
@@ -51,6 +59,7 @@ import {
|
|
|
51
59
|
isPeerRecordFresh,
|
|
52
60
|
orderSignalingFirst,
|
|
53
61
|
isSignalingAddr,
|
|
62
|
+
trailingPeerId,
|
|
54
63
|
withTrailingPeerId,
|
|
55
64
|
currentMemberTrustPolicy,
|
|
56
65
|
DEFAULT_PEER_RECORD_MAX_AGE_MS,
|
|
@@ -62,25 +71,33 @@ import {
|
|
|
62
71
|
isPushPlatform
|
|
63
72
|
} from './device-token.js';
|
|
64
73
|
import { StrandWatcher, type StrandQueryable, type SAppIdLookup } from './strand-watcher.js';
|
|
65
|
-
import { StrandInstanceManager } from './strand-instance-manager.js';
|
|
74
|
+
import { StrandInstanceManager, liveStrandStatus } from './strand-instance-manager.js';
|
|
75
|
+
import { PeerJoinBackfill } from './peer-join-backfill.js';
|
|
66
76
|
import { deriveCohortMembers } from './strand-cohort.js';
|
|
67
77
|
import type { CohortPeerRow } from './strand-cohort.js';
|
|
68
78
|
import {
|
|
69
79
|
selectControlCohortDials,
|
|
70
|
-
DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS,
|
|
71
80
|
DEFAULT_CONTROL_COHORT_RECONCILE_MS,
|
|
72
|
-
DEFAULT_CONTROL_COHORT_TARGET_DEGREE
|
|
81
|
+
DEFAULT_CONTROL_COHORT_TARGET_DEGREE,
|
|
82
|
+
type ControlCohortReconcileResult
|
|
73
83
|
} from './control-cohort.js';
|
|
74
|
-
import {
|
|
84
|
+
import {
|
|
85
|
+
dialPeerAddrs,
|
|
86
|
+
DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS,
|
|
87
|
+
DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS,
|
|
88
|
+
type PeerDialBudget
|
|
89
|
+
} from './peer-dial.js';
|
|
75
90
|
import { EnrollmentService } from './enrollment.js';
|
|
76
91
|
import { HibernationManager, type HibernationCallbacks } from './hibernation-manager.js';
|
|
77
|
-
import { ControlDatabase, type RevokedRowRef } from './control-database.js';
|
|
92
|
+
import { ControlDatabase, isStrandIdConflict, type RevokedRowRef } from './control-database.js';
|
|
93
|
+
import type { ControlRetryAbandonment } from './control-retry.js';
|
|
78
94
|
import { SeedBootstrapService, type SeedEventCallbacks } from './seed-bootstrap.js';
|
|
79
95
|
import type { SeedTrustPolicy } from './seed-trust-policy.js';
|
|
80
96
|
import {
|
|
81
97
|
StrandSolicitationService,
|
|
82
98
|
type StrandSolicitationServiceOptions
|
|
83
99
|
} from './strand-solicitation.js';
|
|
100
|
+
import { MAX_STRAND_ADDRS } from './strand-formation-protocol.js';
|
|
84
101
|
import {
|
|
85
102
|
createMembershipConnectionGater,
|
|
86
103
|
DEFAULT_ENROLLMENT_WINDOW_MS,
|
|
@@ -97,7 +114,7 @@ import {
|
|
|
97
114
|
peerStrandKey,
|
|
98
115
|
type CircuitRelayTarget
|
|
99
116
|
} from './delegate-admission.js';
|
|
100
|
-
import { relayCircuitAddrs, resolveListenAddrs, RelayReservationFailedError } from './relay-addrs.js';
|
|
117
|
+
import { relayCircuitAddrs, resolveListenAddrs, resolveTransportOptions, RelayReservationFailedError } from './relay-addrs.js';
|
|
101
118
|
import { replacesAdvertisedAddrs, resolveAnnounceAddrs } from './announce-addrs.js';
|
|
102
119
|
import {
|
|
103
120
|
superviseRelayReservation,
|
|
@@ -115,6 +132,7 @@ import {
|
|
|
115
132
|
type ConnectionLike
|
|
116
133
|
} from './diagnostics/connection-path.js';
|
|
117
134
|
import { TurnRelayTracker } from './diagnostics/webrtc-turn-tracker.js';
|
|
135
|
+
import { timedStep } from './timed-step.js';
|
|
118
136
|
|
|
119
137
|
const log = debug('sereus:cadre:node');
|
|
120
138
|
const timing = debug('sereus:cadre:timing');
|
|
@@ -149,6 +167,30 @@ function relayStrandAddrPeer(relay: CircuitRelayTarget): StrandAddrPeer {
|
|
|
149
167
|
return { peerId: relay.relayPeerId, addrs: [multiaddr(relay.relayAddr)] };
|
|
150
168
|
}
|
|
151
169
|
|
|
170
|
+
/**
|
|
171
|
+
* `primary` followed by the entries of `extra` it does not already contain, de-duplicated
|
|
172
|
+
* and order-preserving.
|
|
173
|
+
*
|
|
174
|
+
* Used wherever a strand's discovery seed unions two sources of address strings — the
|
|
175
|
+
* freshly-RPC'd sibling answers and the cross-party addresses learned at formation. The
|
|
176
|
+
* caller passes the FRESHER source as `primary`, so a stale entry can only ever be
|
|
177
|
+
* appended, never promoted ahead of a current one, and each source keeps whatever
|
|
178
|
+
* signaling-first ordering it arrived with.
|
|
179
|
+
*/
|
|
180
|
+
function unionAddrs(primary: readonly string[], extra: readonly string[]): string[] {
|
|
181
|
+
if (extra.length === 0) {
|
|
182
|
+
return [...primary];
|
|
183
|
+
}
|
|
184
|
+
const out = [...primary];
|
|
185
|
+
const seen = new Set(primary);
|
|
186
|
+
for (const addr of extra) {
|
|
187
|
+
if (seen.has(addr)) continue;
|
|
188
|
+
seen.add(addr);
|
|
189
|
+
out.push(addr);
|
|
190
|
+
}
|
|
191
|
+
return out;
|
|
192
|
+
}
|
|
193
|
+
|
|
152
194
|
/**
|
|
153
195
|
* Build the signing callback the `ControlDatabase` writers expect: they hand it the
|
|
154
196
|
* canonical row-bound message BYTES (see `buildAuthorizationMessage`), and it ed25519-signs
|
|
@@ -172,6 +214,60 @@ function requireNonBlank(value: string, label: string): string {
|
|
|
172
214
|
return trimmed;
|
|
173
215
|
}
|
|
174
216
|
|
|
217
|
+
/**
|
|
218
|
+
* Is the live `Strand` row the one a publish of `desired` would have produced?
|
|
219
|
+
*
|
|
220
|
+
* "Identical content" means `(Type, MemberPrivateKey)` and deliberately NOT
|
|
221
|
+
* `FounderOwnerKey`: that column is provenance (WHICH machine published the row), not
|
|
222
|
+
* content. Two machines of one party racing to found the same id must keep resolving as
|
|
223
|
+
* "the winner's row stands" — the loser adopts the row and, since the winner's key is on
|
|
224
|
+
* it, correctly comes up as a joiner (see `launchStrand`'s founder derivation). Comparing
|
|
225
|
+
* it here would turn that benign race into a hard throw. `Id` is the lookup key and
|
|
226
|
+
* `StampId` is a single-use nonce, so the two compared columns are all the content there
|
|
227
|
+
* is. A live row matching on both is, by construction, the state a repeat publish would
|
|
228
|
+
* have reached, whichever branch of `Strand.AuthorizedInsert` seated it (owner-signed, or
|
|
229
|
+
* the unsigned consent branch an invite redemption uses).
|
|
230
|
+
*
|
|
231
|
+
* Returns the mismatching column names, empty when the row matches. The KEY's value is
|
|
232
|
+
* deliberately never returned or logged — it is the closed strand's read-gating secret.
|
|
233
|
+
*/
|
|
234
|
+
function strandRowMismatches(live: StrandRow, desired: StrandRow): string[] {
|
|
235
|
+
const mismatches: string[] = [];
|
|
236
|
+
if (live.Type !== desired.Type) {
|
|
237
|
+
mismatches.push(`Type is '${live.Type}', not the requested '${desired.Type}'`);
|
|
238
|
+
}
|
|
239
|
+
if ((live.MemberPrivateKey ?? null) !== (desired.MemberPrivateKey ?? null)) {
|
|
240
|
+
mismatches.push('MemberPrivateKey differs from the one supplied');
|
|
241
|
+
}
|
|
242
|
+
return mismatches;
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* The live row when it matches `desired` ({@link strandRowMismatches}); otherwise throw
|
|
247
|
+
* naming the columns that differ.
|
|
248
|
+
*
|
|
249
|
+
* A mismatch must stay a hard error: accepting one would let a retry reopen a strand the
|
|
250
|
+
* party closed, or swap the key that gates its reads — so the two cases the raw
|
|
251
|
+
* `UNIQUE constraint failed: Strand.Id` could not distinguish ("already done, carry on"
|
|
252
|
+
* vs. "genuine conflict, stop") are separated here rather than left to the caller.
|
|
253
|
+
*
|
|
254
|
+
* @param situation - How the row came to be there, as a verb phrase completing
|
|
255
|
+
* `Strand <id> …`, e.g. `'is already published'`.
|
|
256
|
+
*/
|
|
257
|
+
function requireMatchingStrandRow(live: StrandRow, desired: StrandRow, situation: string): StrandRow {
|
|
258
|
+
const mismatches = strandRowMismatches(live, desired);
|
|
259
|
+
if (mismatches.length > 0) {
|
|
260
|
+
throw new Error(
|
|
261
|
+
`Strand ${desired.Id} ${situation} with DIFFERENT content, so this publish is a ` +
|
|
262
|
+
`conflict, not a repeat: ${mismatches.join('; ')}. Reconcile deliberately — attach ` +
|
|
263
|
+
'the existing strand with addStrand/foundStrand to keep it, or unpublishStrand first ' +
|
|
264
|
+
'to re-seat it (destructive for a closed strand: its MemberPrivateKey is stored ' +
|
|
265
|
+
'nowhere else).'
|
|
266
|
+
);
|
|
267
|
+
}
|
|
268
|
+
return live;
|
|
269
|
+
}
|
|
270
|
+
|
|
175
271
|
/**
|
|
176
272
|
* CadreNode is the main entry point for a cadre member.
|
|
177
273
|
* It manages:
|
|
@@ -203,6 +299,14 @@ export class CadreNode implements SAppIdLookup {
|
|
|
203
299
|
private trustedOwnerStore: TrustedOwnerStore | null = null;
|
|
204
300
|
private controlNode: Libp2p | null = null;
|
|
205
301
|
private controlDatabase: ControlDatabase | null = null;
|
|
302
|
+
/**
|
|
303
|
+
* The CONTROL network's peer-join block catch-up — same module as the
|
|
304
|
+
* per-strand ones, but membership-gated (see {@link startControlBackfill}).
|
|
305
|
+
* Created in {@link start} after the control node and database are up, stopped
|
|
306
|
+
* and dropped in {@link cleanup} before either is torn down, so a stop()→
|
|
307
|
+
* start() cycle rebuilds it with a fresh caught-up-peer memo.
|
|
308
|
+
*/
|
|
309
|
+
private controlBackfill: PeerJoinBackfill | null = null;
|
|
206
310
|
private strandWatcher: StrandWatcher | null = null;
|
|
207
311
|
private strandManager: StrandInstanceManager;
|
|
208
312
|
private hibernationManager: HibernationManager;
|
|
@@ -228,10 +332,11 @@ export class CadreNode implements SAppIdLookup {
|
|
|
228
332
|
private _running = false;
|
|
229
333
|
/**
|
|
230
334
|
* The control database's own raw storage (cache-wrapped), resolved once per
|
|
231
|
-
* `start()` and released in {@link cleanup}. Owning it is what keeps
|
|
232
|
-
*
|
|
233
|
-
* `
|
|
234
|
-
*
|
|
335
|
+
* `start()` and released in {@link cleanup}. Owning it is what keeps the
|
|
336
|
+
* provider call for the control scope — `controlStorageScope(partyId)`, see
|
|
337
|
+
* `storage-scope.ts` — a once-per-runtime call: a `stop()` then `start()` cycle
|
|
338
|
+
* on this object re-resolves against a live cache, and never orphans the
|
|
339
|
+
* previous wrapper's registration in the shared cache pool.
|
|
235
340
|
*/
|
|
236
341
|
private controlStorage: IRawStorage | null = null;
|
|
237
342
|
/**
|
|
@@ -291,6 +396,15 @@ export class CadreNode implements SAppIdLookup {
|
|
|
291
396
|
/** Map of strandId -> sAppConfig for sAppId filtering and management */
|
|
292
397
|
private sAppConfigs: Map<string, SAppConfig> = new Map();
|
|
293
398
|
|
|
399
|
+
/**
|
|
400
|
+
* Strands the control network advertises that no local sAppConfig claims — the
|
|
401
|
+
* backlog behind `strand:discovered`. Maintained in {@link handleStrandAdded}
|
|
402
|
+
* (added), {@link addStrand} (claimed), {@link detachStrand} (stopped or its
|
|
403
|
+
* control row vanished) and {@link cleanup}. Read through
|
|
404
|
+
* {@link getDiscoveredStrands}.
|
|
405
|
+
*/
|
|
406
|
+
private discoveredStrands: Map<string, StrandRow> = new Map();
|
|
407
|
+
|
|
294
408
|
/**
|
|
295
409
|
* Most-recently pushed invite addresses (see {@link setInviteAddresses}).
|
|
296
410
|
* When non-null these take priority over `libp2pNode.getMultiaddrs()` when
|
|
@@ -397,40 +511,155 @@ export class CadreNode implements SAppIdLookup {
|
|
|
397
511
|
private readonly strandPeerAddrRefreshAt = new Map<string, number>();
|
|
398
512
|
|
|
399
513
|
/**
|
|
400
|
-
*
|
|
401
|
-
*
|
|
402
|
-
*
|
|
403
|
-
*
|
|
404
|
-
*
|
|
405
|
-
*
|
|
514
|
+
* CROSS-PARTY strand-network addresses learned at formation, keyed by strandId — the
|
|
515
|
+
* responder's live strand addrs, carried back on the formation result
|
|
516
|
+
* (`FormationResultMessage.strandAddrs`) and recorded by {@link formStrand}.
|
|
517
|
+
*
|
|
518
|
+
* This is the ONLY cross-party discovery input there is. The strand-addr RPC that
|
|
519
|
+
* resolves a strand's addresses is membership-gated and answers own-party siblings
|
|
520
|
+
* only, so without this map a joiner's cohort seed for a two-party strand is empty and
|
|
521
|
+
* the mesh never forms.
|
|
522
|
+
*
|
|
523
|
+
* Read by {@link resolveCohortSeed} (launch + hibernation resume) and re-merged by
|
|
524
|
+
* every {@link refreshStrandPeerAddrs} pass — the re-merge is what keeps the entries
|
|
525
|
+
* alive past the peerStore's one-hour address expiry, since nothing can re-resolve
|
|
526
|
+
* them.
|
|
527
|
+
*
|
|
528
|
+
* IN-MEMORY and ONE-SHOT, deliberately (see `docs/strands.md`): the addresses die with
|
|
529
|
+
* the process, and a restarted initiator with no sibling running the strand is back to
|
|
530
|
+
* an empty seed until the durability work lands
|
|
531
|
+
* (`backlog/feat-cross-party-strand-addr-durability`). They are also never refreshed,
|
|
532
|
+
* so a responder whose relay reservation rotates before the initiator dials leaves a
|
|
533
|
+
* dead entry behind; recovery today is a fresh invitation.
|
|
534
|
+
*
|
|
535
|
+
* NOTE: entries are kept for the node's lifetime, including across a
|
|
536
|
+
* {@link stopStrand} — a stopped strand may be rediscovered by the watcher and must
|
|
537
|
+
* still get its seed. Each strand's list is capped at `MAX_STRAND_ADDRS`, but the
|
|
538
|
+
* number of KEYS is bounded only by the strands this node has ever formed as an
|
|
539
|
+
* initiator (not by time); if a node ever forms strands at scale, evict on
|
|
540
|
+
* `unpublishStrand` or cap the map.
|
|
541
|
+
*/
|
|
542
|
+
private readonly crossPartyStrandAddrs = new Map<string, string[]>();
|
|
543
|
+
|
|
544
|
+
/**
|
|
545
|
+
* PENDING strand membership invitations learned at formation, keyed by strandId — the
|
|
546
|
+
* single-use `Strand.Invite` credential a closed-strand formation result carried back
|
|
547
|
+
* ({@link FormStrandResult.membershipInvite}), waiting for this node's strand bring-up
|
|
548
|
+
* to redeem (`consumeInvite` seats the `Strand.Member` row under this party's own key
|
|
549
|
+
* — the `strand-node-binds-member-peer` half of the party-identity chain).
|
|
550
|
+
*
|
|
551
|
+
* IN-MEMORY, deliberately, like {@link crossPartyStrandAddrs}: a restarted joiner
|
|
552
|
+
* re-forms from scratch anyway (durability is
|
|
553
|
+
* `backlog/feat-cross-party-strand-addr-durability`'s problem), and re-forming issues
|
|
554
|
+
* a fresh invitation. A re-formation against the same strand REPLACES the entry — the
|
|
555
|
+
* fresh invitation supersedes one that may have expired. The party key the invitation
|
|
556
|
+
* admits, by contrast, IS persisted (`StrandPartyKey`, seated by
|
|
557
|
+
* {@link adoptFormationMembershipInvite} before the entry lands here), so a lost
|
|
558
|
+
* invitation never orphans an identity.
|
|
559
|
+
*
|
|
560
|
+
* INVALIDATION is owned by the strand's bring-up membership reconciler
|
|
561
|
+
* (`strand-membership-reconciler.ts`, wired via `StartStrandConfig.pendingMembershipInvite`
|
|
562
|
+
* in {@link launchStrand}): it deletes the entry once the invitation is redeemed, burned
|
|
563
|
+
* against an already-seated member, or found dead (expired / cancelled / consumed
|
|
564
|
+
* elsewhere / the strand sealed). Until the strand is actually launched here the entry
|
|
565
|
+
* just waits.
|
|
566
|
+
*
|
|
567
|
+
* NOTE: entries live for the node's lifetime (one small pair per formed closed
|
|
568
|
+
* strand), the same unbounded-keys tripwire {@link crossPartyStrandAddrs} documents —
|
|
569
|
+
* if a node ever forms strands at scale, evict alongside that map.
|
|
570
|
+
*/
|
|
571
|
+
private readonly pendingMembershipInvites = new Map<string, StrandMembershipInvite>();
|
|
572
|
+
|
|
573
|
+
/**
|
|
574
|
+
* Founder launches refused because the strand was founded before the per-party
|
|
575
|
+
* identity split (`PreSplitStrandIdentityError`), keyed by strandId. Read by
|
|
576
|
+
* {@link issueStrandMembershipInvite}, which rethrows the recorded error so a bound
|
|
577
|
+
* redemption against the strand is rejected as "must be recreated" — not told to retry
|
|
578
|
+
* because the runtime is not live (a refused fresh launch leaves none) or failed by the
|
|
579
|
+
* `Strand.Invite` gate (a refused in-place founding leaves a joiner instance up whose
|
|
580
|
+
* party key is no manager).
|
|
581
|
+
*
|
|
582
|
+
* Written and cleared by {@link launchStrand}; also cleared by {@link detachStrand},
|
|
583
|
+
* {@link unpublishStrand} and {@link cleanup}, so a recreated id starts clean.
|
|
584
|
+
* In-memory: after a restart the next founder launch of the strand records it again;
|
|
585
|
+
* until then issuance still reads the fingerprint off the live rows whenever a runtime
|
|
586
|
+
* is up — only a strand with no runtime at all answers retryably meanwhile.
|
|
587
|
+
*/
|
|
588
|
+
private readonly strandLaunchRefusals = new Map<string, PreSplitStrandIdentityError>();
|
|
589
|
+
|
|
590
|
+
/**
|
|
591
|
+
* Dial targets learned out of band, keyed by peer id, valued by the multiaddr
|
|
592
|
+
* strings this node was handed (parsed lazily, at dial time). Two writers:
|
|
593
|
+
* {@link recordSeedBootstrapPeers} retains the owner-flagged peers of every seed
|
|
594
|
+
* this node applies, and {@link addDrone} retains the addresses of every node
|
|
595
|
+
* this node adds. Two readers: {@link dialColdStartBootstrap} dials every entry
|
|
596
|
+
* while the control database still has no siblings, and
|
|
597
|
+
* {@link resolveControlDialAddrs} falls back to a sibling's entry when neither
|
|
598
|
+
* its signed record nor the address book yields an address.
|
|
406
599
|
*
|
|
407
600
|
* A STORE rather than a plain map (see `bootstrap-peer-store.ts`), because the
|
|
408
601
|
* targets must outlive the process: a node seeded into a party it could not
|
|
409
602
|
* reach has nothing else on disk naming that party's addresses (`applySeed`
|
|
410
603
|
* writes no control row, and `CadrePeer` fills in only after a connection
|
|
411
604
|
* succeeds), so an in-memory-only set left it stranded permanently across a
|
|
412
|
-
* restart.
|
|
605
|
+
* restart. An owner that added a node it cannot be dialed by is in the same
|
|
606
|
+
* position: the added node's row stays unsigned, so unresolvable, until that
|
|
607
|
+
* node self-publishes over a connection only the owner can open. Constructed (or
|
|
608
|
+
* adopted from `config.bootstrapPeers.store`) by
|
|
413
609
|
* {@link initializeBootstrapPeerStore} during {@link start}; deliberately NOT
|
|
414
610
|
* cleared by {@link cleanup}, so it survives a stop()→start() cycle of the same
|
|
415
611
|
* node instance — same lifecycle as {@link trustedOwnerStore}. Durability
|
|
416
|
-
* depends on the injected backend
|
|
417
|
-
* RN/browser stay ephemeral for now.
|
|
612
|
+
* depends on the injected backend; the default is in-memory.
|
|
418
613
|
*
|
|
419
614
|
* Deliberately NOT the libp2p peer store, which {@link peerStoreAddrs} already
|
|
420
615
|
* reads for the steady-state path. The peer store is shared with everything
|
|
421
616
|
* libp2p discovers, so "dial every entry" would grow into dialing arbitrary
|
|
422
617
|
* discovered peers as the node lives longer; this store holds exactly the peers
|
|
423
|
-
* an owner-signed, trust-anchored seed nominated as owners
|
|
424
|
-
*
|
|
425
|
-
*
|
|
618
|
+
* an owner-signed, trust-anchored seed nominated as owners, and the nodes this
|
|
619
|
+
* node chose to add. `applySeed` also merges a seed's addresses into the peer
|
|
620
|
+
* store, where they age out; an added node's addresses are never merged there,
|
|
621
|
+
* because only verified addresses go into the address book (see
|
|
622
|
+
* {@link warmSiblingAddrBook}).
|
|
426
623
|
*
|
|
427
|
-
* A later
|
|
624
|
+
* A later record OVERWRITES an entry rather than merging, so a re-seed after an
|
|
428
625
|
* owner's address changes replaces the stale address instead of accumulating
|
|
429
|
-
* dead ones
|
|
430
|
-
* the
|
|
626
|
+
* dead ones, and {@link warmSiblingAddrBook} replaces an entry with the addresses
|
|
627
|
+
* the sibling's signed record resolves to when they differ. Only this node's own
|
|
628
|
+
* {@link removePeer} evicts an entry: otherwise they are the node's only way back
|
|
629
|
+
* to those peers if it is ever stranded again.
|
|
431
630
|
*/
|
|
432
631
|
private bootstrapPeerStore: BootstrapPeerStore | null = null;
|
|
433
632
|
|
|
633
|
+
/**
|
|
634
|
+
* Node-local record of the party's enrolled-machine count (see
|
|
635
|
+
* `enrolled-machine-store.ts`), kept so the CONTROL node can declare a
|
|
636
|
+
* block-repair yardstick at bring-up. Constructed (or adopted from
|
|
637
|
+
* `config.enrolledMachines.store`) by {@link initializeEnrolledMachineStore};
|
|
638
|
+
* written by {@link refreshAuthorizedControlPeers}. Like
|
|
639
|
+
* {@link bootstrapPeerStore} it is deliberately NOT cleared by {@link cleanup},
|
|
640
|
+
* so it survives a stop()→start() cycle of the same node instance.
|
|
641
|
+
*/
|
|
642
|
+
private enrolledMachineStore: EnrolledMachineStore | null = null;
|
|
643
|
+
|
|
644
|
+
/**
|
|
645
|
+
* The enrolled-machine count this node's CONTROL libp2p node was (or will be)
|
|
646
|
+
* built with — read out of {@link enrolledMachineStore} during {@link start},
|
|
647
|
+
* before {@link createControlNode}, and consumed by
|
|
648
|
+
* {@link buildControlNodeOptions}.
|
|
649
|
+
*
|
|
650
|
+
* A captured FIELD rather than a live `store.count()` read, deliberately: the
|
|
651
|
+
* store moves as membership changes, but Optimystic froze the policy when the
|
|
652
|
+
* node was built, so this field is the honest answer to "what did this node
|
|
653
|
+
* actually declare?" and does not drift away from the running node. It is
|
|
654
|
+
* re-read on each {@link start}, which is what makes a stop()→start() cycle pick
|
|
655
|
+
* up a count recorded during the previous run.
|
|
656
|
+
*
|
|
657
|
+
* `undefined` — a brand-new node, an unreadable slot, or the ephemeral default
|
|
658
|
+
* store — means "this node does not know", which `controlClusterPolicy` answers
|
|
659
|
+
* with the frozen base policy itself.
|
|
660
|
+
*/
|
|
661
|
+
private declaredEnrolledMachines: number | undefined;
|
|
662
|
+
|
|
434
663
|
/** Initial self-registration timer (see {@link scheduleSelfRegistration}). */
|
|
435
664
|
private selfRegistrationTimer: ReturnType<typeof setTimeout> | null = null;
|
|
436
665
|
/** TTL heartbeat that re-publishes the self record before it goes stale. */
|
|
@@ -450,7 +679,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
450
679
|
* together; collapsing concurrent passes into one in-flight run prevents two
|
|
451
680
|
* passes from double-dialing the same siblings (mirrors {@link registerSelfInFlight}).
|
|
452
681
|
*/
|
|
453
|
-
private reconcileControlCohortInFlight: Promise<
|
|
682
|
+
private reconcileControlCohortInFlight: Promise<ControlCohortReconcileResult> | null = null;
|
|
454
683
|
/**
|
|
455
684
|
* Single-flight guard for {@link registerSelf}. Concurrent callers (the explicit
|
|
456
685
|
* CLI `--owner` publish, the 1s startup timer, the TTL heartbeat, and the
|
|
@@ -458,6 +687,19 @@ export class CadreNode implements SAppIdLookup {
|
|
|
458
687
|
* both read "no row yet" and race a duplicate INSERT (a `CadrePeer` PK conflict).
|
|
459
688
|
*/
|
|
460
689
|
private registerSelfInFlight: Promise<SelfRegistrationOutcome> | null = null;
|
|
690
|
+
/**
|
|
691
|
+
* When this node last PUBLISHED its own `CadrePeer` address record (inserted or
|
|
692
|
+
* refreshed), or null if it never has in this session. Stamped by
|
|
693
|
+
* {@link noteSelfRecordPublished} on every successful publish, wherever it was driven
|
|
694
|
+
* from — the boot pass, the heartbeat, an address change, a drain or an explicit call.
|
|
695
|
+
*
|
|
696
|
+
* Read by {@link escalateIfSelfRecordStale} for the one consequence worth an operator's
|
|
697
|
+
* attention: once this gap exceeds {@link DEFAULT_PEER_RECORD_MAX_AGE_MS}, every other
|
|
698
|
+
* machine in the party is already discarding this node's address as stale.
|
|
699
|
+
*/
|
|
700
|
+
private lastSelfRecordPublishAt: number | null = null;
|
|
701
|
+
/** Say-once latch for {@link escalateIfSelfRecordStale}; re-armed by the next publish. */
|
|
702
|
+
private selfRecordStaleWarned = false;
|
|
461
703
|
|
|
462
704
|
// ── Write-while-alone re-replication (control-write-ensure-replicated) ──────
|
|
463
705
|
/**
|
|
@@ -493,6 +735,14 @@ export class CadreNode implements SAppIdLookup {
|
|
|
493
735
|
* exactly a "removals from before this lifetime" case).
|
|
494
736
|
*/
|
|
495
737
|
private reissuedHeldRevocations = false;
|
|
738
|
+
/**
|
|
739
|
+
* Whether this process has seen the singleton `Revocation` ledger marker filed — its own
|
|
740
|
+
* `'opened'` or an `'already-open'` answer, in {@link openRevocationLedgerIfDue}. A
|
|
741
|
+
* record of work done, not a cached authorization answer: the marker can never be
|
|
742
|
+
* deleted (`NoDelete`), so this can only go stale if the database itself is replaced,
|
|
743
|
+
* which is why stop() clears it with the other per-process replication state.
|
|
744
|
+
*/
|
|
745
|
+
private revocationLedgerOpened = false;
|
|
496
746
|
/** This node's own `CadrePeer` self-write committed local-only (re-touched on growth). */
|
|
497
747
|
private pendingSelfPeerWrite = false;
|
|
498
748
|
/** This node's own `DeviceToken` self-write committed local-only (re-touched on growth). */
|
|
@@ -647,6 +897,26 @@ export class CadreNode implements SAppIdLookup {
|
|
|
647
897
|
return this.strandManager.getInstances();
|
|
648
898
|
}
|
|
649
899
|
|
|
900
|
+
/**
|
|
901
|
+
* Strands the control network advertises that no local sAppConfig claims —
|
|
902
|
+
* the backlog behind `strand:discovered`.
|
|
903
|
+
*
|
|
904
|
+
* The event fires once per strand, and it can fire before the app has
|
|
905
|
+
* subscribed (the watcher's first poll runs inside `start()`). So an app that
|
|
906
|
+
* auto-joins discovered strands must subscribe FIRST and then drain this map,
|
|
907
|
+
* not rely on the event alone. Entries leave the map when the strand is
|
|
908
|
+
* claimed ({@link addStrand}) or its control row disappears.
|
|
909
|
+
*
|
|
910
|
+
* Bounded by the number of strands this party has that this node does not run
|
|
911
|
+
* — the control database holds only this party's rows — so there is no cap and
|
|
912
|
+
* no eviction policy to reason about.
|
|
913
|
+
*
|
|
914
|
+
* Returns a snapshot; mutating it does not affect the node.
|
|
915
|
+
*/
|
|
916
|
+
getDiscoveredStrands(): Map<string, StrandRow> {
|
|
917
|
+
return new Map(this.discoveredStrands);
|
|
918
|
+
}
|
|
919
|
+
|
|
650
920
|
/**
|
|
651
921
|
* Get a specific strand instance
|
|
652
922
|
*/
|
|
@@ -702,6 +972,14 @@ export class CadreNode implements SAppIdLookup {
|
|
|
702
972
|
// loaded before the first reconcile pass could consult them.
|
|
703
973
|
this.initializeBootstrapPeerStore();
|
|
704
974
|
|
|
975
|
+
// Read the party's enrolled-machine count out of its node-local record and
|
|
976
|
+
// capture it for buildControlNodeOptions below. This MUST precede
|
|
977
|
+
// createControlNode: Optimystic freezes the cluster policy when the node is
|
|
978
|
+
// built, and the ControlDatabase that could answer the question live does not
|
|
979
|
+
// exist until after that. Remembering the number across the restart is the
|
|
980
|
+
// only way to declare it at all — see `enrolled-machine-store.ts`.
|
|
981
|
+
this.initializeEnrolledMachineStore();
|
|
982
|
+
|
|
705
983
|
// Arm the connection gate's BRING-UP QUIET PERIOD before the libp2p node
|
|
706
984
|
// exists, so no connection can form ahead of it: `@libp2p/bootstrap` emits
|
|
707
985
|
// its discovery events ~1 s after `libp2p.start()` and the connection
|
|
@@ -769,6 +1047,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
769
1047
|
// Committed-delete seam → write-while-alone revocation queue. Same wiring
|
|
770
1048
|
// window argument as the membership listener above.
|
|
771
1049
|
this.controlDatabase.setGuardedDeleteListener((revocation) => this.noteGuardedDelete(revocation));
|
|
1050
|
+
// Abandoned-write seam. Wired here, before the first control write can run, so a
|
|
1051
|
+
// write lost during the rest of bring-up is reported like any other.
|
|
1052
|
+
this.controlDatabase.setControlWriteAbandonedListener(
|
|
1053
|
+
(abandonment) => this.noteControlWriteAbandoned(abandonment));
|
|
772
1054
|
|
|
773
1055
|
// Create strand queryable using the control database
|
|
774
1056
|
const queryable = this.createStrandQueryable();
|
|
@@ -835,6 +1117,13 @@ export class CadreNode implements SAppIdLookup {
|
|
|
835
1117
|
// write-while-alone re-replication drain must fire on the first 0→≥1 edge.
|
|
836
1118
|
this.wireControlConnectionListeners();
|
|
837
1119
|
|
|
1120
|
+
// Arm the control network's peer-join block catch-up now that the control
|
|
1121
|
+
// DB (which the push-time membership gate reads) is up. Armed even when
|
|
1122
|
+
// this node is alone — like the strand one, it only does work when the
|
|
1123
|
+
// control libp2p node reports a peer connection, and arming it at start is
|
|
1124
|
+
// what closes the "genesised alone, headers never replicate" hole.
|
|
1125
|
+
this.startControlBackfill();
|
|
1126
|
+
|
|
838
1127
|
// Seed the per-stream gate's materialized authorized set now that the
|
|
839
1128
|
// control DB is up (non-blocking; the helper never rejects). Until it
|
|
840
1129
|
// lands the snapshot is empty, which the stream gate treats as cold
|
|
@@ -846,7 +1135,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
846
1135
|
// scheduleSelfRegistration, so this node's first published `CadrePeer` row
|
|
847
1136
|
// already carries the `/p2p-circuit` address the reservation earns it.
|
|
848
1137
|
// Throws on a relay that will not have us — start() is fail-fast for a node
|
|
849
|
-
// whose operator named a relay.
|
|
1138
|
+
// whose operator named a relay, unless network.requireRelay is false.
|
|
850
1139
|
await this.driveControlRelayReservation();
|
|
851
1140
|
|
|
852
1141
|
// Schedule self-registration in background
|
|
@@ -1049,10 +1338,54 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1049
1338
|
}
|
|
1050
1339
|
|
|
1051
1340
|
/**
|
|
1052
|
-
*
|
|
1053
|
-
*
|
|
1054
|
-
*
|
|
1055
|
-
*
|
|
1341
|
+
* Construct (or adopt) the node-local enrolled-machine store and capture the
|
|
1342
|
+
* count this run will declare into {@link declaredEnrolledMachines}.
|
|
1343
|
+
*
|
|
1344
|
+
* Synchronous for the same reason {@link initializeBootstrapPeerStore} is: an
|
|
1345
|
+
* injected store has already loaded its persisted count by the time it is handed
|
|
1346
|
+
* in (its `open` is the async part), and the in-memory default has nothing to
|
|
1347
|
+
* load. Must run BEFORE {@link createControlNode} — that is the whole point of
|
|
1348
|
+
* the record; see `enrolled-machine-store.ts`.
|
|
1349
|
+
*
|
|
1350
|
+
* The store instance is kept across stop()→start() (so a count recorded during
|
|
1351
|
+
* the previous run is not lost with the ephemeral default), but the DECLARED
|
|
1352
|
+
* count is re-read every time — which is how a restart applies a number the
|
|
1353
|
+
* previous run learned.
|
|
1354
|
+
*
|
|
1355
|
+
* A store scoped to a different party is a configuration error, fail closed. It
|
|
1356
|
+
* is only a repair hint, but a hint sized by a foreign party's membership is a
|
|
1357
|
+
* number nobody chose, and the mismatch always means a miswired embedder.
|
|
1358
|
+
*/
|
|
1359
|
+
private initializeEnrolledMachineStore(): void {
|
|
1360
|
+
const partyId = this.config.controlNetwork.partyId;
|
|
1361
|
+
const store = this.enrolledMachineStore
|
|
1362
|
+
?? this.config.enrolledMachines?.store
|
|
1363
|
+
?? new MemoryEnrolledMachineStore(partyId);
|
|
1364
|
+
if (store.partyId !== partyId) {
|
|
1365
|
+
throw new Error(
|
|
1366
|
+
`CadreNodeConfig: enrolledMachines.store is scoped to party ${store.partyId}, ` +
|
|
1367
|
+
`but this node serves party ${partyId} — refusing to size a repair yardstick from a foreign party`
|
|
1368
|
+
);
|
|
1369
|
+
}
|
|
1370
|
+
this.enrolledMachineStore = store;
|
|
1371
|
+
this.declaredEnrolledMachines = store.count();
|
|
1372
|
+
log('control repair yardstick will be declared from %o enrolled machine(s)', this.declaredEnrolledMachines);
|
|
1373
|
+
}
|
|
1374
|
+
|
|
1375
|
+
/**
|
|
1376
|
+
* The node-local enrolled-machine store (null before {@link start}) — what this
|
|
1377
|
+
* node last knew about its party's size. Exposed for diagnostics and for a host
|
|
1378
|
+
* that wants to show which repair yardstick the next launch will declare.
|
|
1379
|
+
*/
|
|
1380
|
+
getEnrolledMachineStore(): EnrolledMachineStore | null {
|
|
1381
|
+
return this.enrolledMachineStore;
|
|
1382
|
+
}
|
|
1383
|
+
|
|
1384
|
+
/**
|
|
1385
|
+
* The node-local bootstrap-peer store (null before {@link start}) — the dial
|
|
1386
|
+
* targets learned out of band that {@link reconcileControlCohort} falls back to.
|
|
1387
|
+
* Exposed for diagnostics and for a host that wants to show "what would this
|
|
1388
|
+
* node dial if it is stranded?".
|
|
1056
1389
|
*/
|
|
1057
1390
|
getBootstrapPeerStore(): BootstrapPeerStore | null {
|
|
1058
1391
|
return this.bootstrapPeerStore;
|
|
@@ -1098,6 +1431,93 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1098
1431
|
return await createLibp2pNode(this.buildControlNodeOptions());
|
|
1099
1432
|
}
|
|
1100
1433
|
|
|
1434
|
+
/**
|
|
1435
|
+
* The control libp2p node's Optimystic network name — ONE binding for every
|
|
1436
|
+
* derivation from it (`db-p2p` namespaces all of the node's protocol ids as
|
|
1437
|
+
* `/optimystic/<networkName>/...`), so the node options and the block-transfer
|
|
1438
|
+
* protocol prefix the control backfill dials can never drift apart.
|
|
1439
|
+
*/
|
|
1440
|
+
private controlNetworkName(): string {
|
|
1441
|
+
return `control-${this.config.controlNetwork.partyId}`;
|
|
1442
|
+
}
|
|
1443
|
+
|
|
1444
|
+
/**
|
|
1445
|
+
* Arm the CONTROL network's peer-join block catch-up: push every block in the
|
|
1446
|
+
* control database's own raw store to each newly connected AUTHORIZED member
|
|
1447
|
+
* this runtime has not yet caught up. This is what physically replicates
|
|
1448
|
+
* control blocks committed while the writer was alone — the named
|
|
1449
|
+
* collection-header blocks written once at genesis above all, whose revision
|
|
1450
|
+
* never moves again, so no later commit ever carries them to a member that
|
|
1451
|
+
* joined after them. Without it, such a member that restarts offline reads
|
|
1452
|
+
* the affected control tables as EMPTY, silently (`isMember()` answers false
|
|
1453
|
+
* for peers it knew about before the restart).
|
|
1454
|
+
*
|
|
1455
|
+
* Unlike the per-strand instances (see `strand-instance-manager.ts`), pushes
|
|
1456
|
+
* are gated on {@link isAuthorizedMember}, judged at push time: the control
|
|
1457
|
+
* network's inbound connection gate deliberately admits non-members in
|
|
1458
|
+
* several states (seed delivery to an un-enrolled node, an open enrollment
|
|
1459
|
+
* window, an outstanding invitation, configured bootstrap/relay peers), and
|
|
1460
|
+
* pushing the whole control store to such a peer would hand a stranger the
|
|
1461
|
+
* party's entire membership, addresses and strand list. The receiving side
|
|
1462
|
+
* needs no work of its own: `createLibp2pNode` registers the block-transfer
|
|
1463
|
+
* handler on every node it builds, control node included, and this node's
|
|
1464
|
+
* per-stream gate (`authorizeInboundControlStream`) covers the inbound
|
|
1465
|
+
* direction. A denied peer is retried on its next `connection:open`, and —
|
|
1466
|
+
* because the production join order is connect-then-authorize — on every
|
|
1467
|
+
* committed membership change via {@link refreshAuthorizedControlPeers}'s
|
|
1468
|
+
* `scheduleConnectedPeers()` call.
|
|
1469
|
+
*
|
|
1470
|
+
* No-ops (logged) when the embedder configured no control storage or the
|
|
1471
|
+
* node exposes no key network — the backfill would have nothing to read or
|
|
1472
|
+
* no way to dial.
|
|
1473
|
+
*/
|
|
1474
|
+
private startControlBackfill(): void {
|
|
1475
|
+
if (this.config.controlBackfill?.enabled === false || !this.controlNode) {
|
|
1476
|
+
return;
|
|
1477
|
+
}
|
|
1478
|
+
// Through the resolver, not the {@link controlStorage} memo field: the resolver is
|
|
1479
|
+
// idempotent (it returns the same wrapped store `buildControlNodeOptions` already
|
|
1480
|
+
// resolved), so the catch-up cannot be silently disarmed by a future reordering that
|
|
1481
|
+
// moves this call ahead of the one that populated the field.
|
|
1482
|
+
const storage = this.resolveControlStorage();
|
|
1483
|
+
const keyNetwork = (this.controlNode as Libp2pNodeWithRepo).keyNetwork;
|
|
1484
|
+
if (!storage || !keyNetwork) {
|
|
1485
|
+
log('Control peer-join block catch-up is inert: %s',
|
|
1486
|
+
!storage ? 'no control storage configured' : 'control node exposes no keyNetwork');
|
|
1487
|
+
return;
|
|
1488
|
+
}
|
|
1489
|
+
const networkName = this.controlNetworkName();
|
|
1490
|
+
this.controlBackfill = new PeerJoinBackfill({
|
|
1491
|
+
label: networkName,
|
|
1492
|
+
libp2p: this.controlNode,
|
|
1493
|
+
peerNetwork: keyNetwork,
|
|
1494
|
+
storage,
|
|
1495
|
+
// The same prefix the receiver registered its block-transfer handler
|
|
1496
|
+
// under — derived from the same networkName binding the node options
|
|
1497
|
+
// used, never re-spelled here.
|
|
1498
|
+
protocolPrefix: `/optimystic/${networkName}`,
|
|
1499
|
+
authorizePeer: (peerId) => this.isAuthorizedMember(peerId)
|
|
1500
|
+
}, {
|
|
1501
|
+
// A shorter settle window than the strand default (1000 ms): the control
|
|
1502
|
+
// store is small (a party's membership — dozens of blocks), a re-push is
|
|
1503
|
+
// idempotent, and the window a joiner is connected before going offline
|
|
1504
|
+
// can be short (converge one row, then stop) — a slow debounce is the
|
|
1505
|
+
// difference between that joiner holding the collection headers and
|
|
1506
|
+
// reading its own membership as empty after an offline restart.
|
|
1507
|
+
//
|
|
1508
|
+
// NOTE: 250 is reasoned, not swept — the delete-while-alone scenario's
|
|
1509
|
+
// phases 1-3 finish in about a second, so the 1000 ms strand default left
|
|
1510
|
+
// the push racing the joiner's stop, and 250 gives margin. Nobody measured
|
|
1511
|
+
// the shape of the curve either side of it. If the control gates start
|
|
1512
|
+
// flaking on a joiner that stopped before being caught up, lower this
|
|
1513
|
+
// before looking anywhere else; if catch-up pushes start showing up as
|
|
1514
|
+
// connection-churn noise, raise it.
|
|
1515
|
+
debounceMs: 250,
|
|
1516
|
+
...this.config.controlBackfill
|
|
1517
|
+
});
|
|
1518
|
+
this.controlBackfill.start();
|
|
1519
|
+
}
|
|
1520
|
+
|
|
1101
1521
|
/**
|
|
1102
1522
|
* The control database's cache-wrapped raw storage, resolved from
|
|
1103
1523
|
* `config.storage.provider` on first use and then held in {@link controlStorage}
|
|
@@ -1124,8 +1544,12 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1124
1544
|
if (!provider) {
|
|
1125
1545
|
return undefined;
|
|
1126
1546
|
}
|
|
1127
|
-
|
|
1128
|
-
|
|
1547
|
+
// Party-scoped, not the bare literal 'control': the control database holds THIS
|
|
1548
|
+
// party's records, so two parties on one device must not land in one store.
|
|
1549
|
+
// Same key as the cache label, which also makes the shared pool's `stats()` readable.
|
|
1550
|
+
const scope = controlStorageScope(this.config.controlNetwork.partyId);
|
|
1551
|
+
const resolved = typeof provider === 'function' ? provider(scope) : provider;
|
|
1552
|
+
this.controlStorage = wrapStorageWithCache(resolved, scope);
|
|
1129
1553
|
return this.controlStorage;
|
|
1130
1554
|
}
|
|
1131
1555
|
|
|
@@ -1140,9 +1564,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1140
1564
|
* operator whose relay slot is decorative may genuinely want only the announced
|
|
1141
1565
|
* address, so this reports and proceeds.
|
|
1142
1566
|
*
|
|
1143
|
-
* Keyed off
|
|
1144
|
-
*
|
|
1145
|
-
* longer route — is caught too.
|
|
1567
|
+
* Keyed off `relayAddrs` AND `listenAddrs`, so a hand-written `/p2p-circuit` entry
|
|
1568
|
+
* in `listenAddrs` — a reservation by the longer route — is caught too.
|
|
1146
1569
|
*
|
|
1147
1570
|
* NOTE: the only direct `console.*` in this library — a boot-time operator warning, not a
|
|
1148
1571
|
* diagnostic trace (those use `debug`, which an operator never sees without `DEBUG=`). If a
|
|
@@ -1154,9 +1577,13 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1154
1577
|
if (!replacesAdvertisedAddrs(network)) {
|
|
1155
1578
|
return;
|
|
1156
1579
|
}
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1580
|
+
// Read off the raw config, not the resolved listen set: the question is only
|
|
1581
|
+
// "does this config name a relay", and the resolution throws on a hand-written
|
|
1582
|
+
// `<relay>/p2p-circuit` listen entry — which still warrants this warning, and
|
|
1583
|
+
// gets its own refusal a few lines later in `buildControlNodeOptions`.
|
|
1584
|
+
const namesRelay = (network?.relayAddrs?.length ?? 0) > 0
|
|
1585
|
+
|| (network?.listenAddrs ?? []).some((addr) => addr.includes('/p2p-circuit'));
|
|
1586
|
+
if (!namesRelay) {
|
|
1160
1587
|
return;
|
|
1161
1588
|
}
|
|
1162
1589
|
console.warn(
|
|
@@ -1179,11 +1606,16 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1179
1606
|
private buildControlNodeOptions(): Parameters<typeof createLibp2pNode>[0] {
|
|
1180
1607
|
const { controlNetwork, network, profile } = this.config;
|
|
1181
1608
|
const identityKey = this.identityKey;
|
|
1182
|
-
// `
|
|
1183
|
-
//
|
|
1184
|
-
//
|
|
1185
|
-
//
|
|
1186
|
-
const listenAddrs = resolveListenAddrs(network
|
|
1609
|
+
// `network.relayAddrs` contributes the bare `/p2p-circuit` SEARCH entry, which
|
|
1610
|
+
// opens no connection — the reservation is driven explicitly at the END of
|
|
1611
|
+
// `start()`, so the control database is built while this node holds zero
|
|
1612
|
+
// control connections. See `relay-addrs.ts` and `start()`.
|
|
1613
|
+
const listenAddrs = resolveListenAddrs(network);
|
|
1614
|
+
// The transports those listen entries imply. A `/ws` entry switches db-p2p's
|
|
1615
|
+
// WebSocket transport on (without it the entry binds nothing and libp2p reports
|
|
1616
|
+
// nothing); anything outside {tcp, ws/wss, p2p-circuit} throws here rather than
|
|
1617
|
+
// being silently dropped at bring-up. See `relay-addrs.ts`.
|
|
1618
|
+
const transportOptions = resolveTransportOptions(network, listenAddrs);
|
|
1187
1619
|
|
|
1188
1620
|
const enableRelay = this.relayServerEnabled();
|
|
1189
1621
|
|
|
@@ -1194,7 +1626,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1194
1626
|
const nodeOptions: Parameters<typeof createLibp2pNode>[0] = {
|
|
1195
1627
|
port: 0,
|
|
1196
1628
|
bootstrapNodes: controlNetwork.bootstrapNodes,
|
|
1197
|
-
networkName:
|
|
1629
|
+
networkName: this.controlNetworkName(),
|
|
1198
1630
|
storage: controlStorageProvider,
|
|
1199
1631
|
fretProfile: profile === 'storage' ? 'core' : 'edge',
|
|
1200
1632
|
relay: enableRelay,
|
|
@@ -1205,10 +1637,22 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1205
1637
|
// CONTROL_CLUSTER_POLICY: only the admission gate defaults to 2, while the read-repair
|
|
1206
1638
|
// corroboration floor would otherwise fall back to this clusterSize of 16.
|
|
1207
1639
|
clusterSize: CONTROL_REPLICATION_BREADTH,
|
|
1208
|
-
|
|
1640
|
+
// The base control policy with the block-repair corroboration yardstick declared
|
|
1641
|
+
// from the machines this party had enrolled at this node's last look
|
|
1642
|
+
// ({@link declaredEnrolledMachines}, read in start() before this runs). Handed
|
|
1643
|
+
// `undefined` — a brand-new node, an unreadable record, or an embedder that
|
|
1644
|
+
// injected no store — this returns the frozen CONTROL_CLUSTER_POLICY object
|
|
1645
|
+
// itself, so the unknown case is provably byte-for-byte the old behaviour.
|
|
1646
|
+
// The yardstick moves ALONE: `assumedClusterSize` stays pinned at 2, because a
|
|
1647
|
+
// party of phones cannot promise three quarters of its machines are awake.
|
|
1648
|
+
clusterPolicy: controlClusterPolicy(this.declaredEnrolledMachines),
|
|
1209
1649
|
arachnode: { enableRingZulu: profile === 'storage' },
|
|
1210
1650
|
...(identityKey && { privateKey: identityKey }),
|
|
1211
1651
|
...(network?.transports && { transports: network.transports }),
|
|
1652
|
+
// `{ wsPort }` when a listen entry names WebSocket, otherwise `{}` — and always
|
|
1653
|
+
// `{}` when `network.transports` is set, since the embedder owns transport policy
|
|
1654
|
+
// then. Spread NEXT to `transports` because the two answer the same question.
|
|
1655
|
+
...transportOptions,
|
|
1212
1656
|
// Configured `listenAddrs`, plus the bare `/p2p-circuit` search listener when
|
|
1213
1657
|
// `network.relayAddrs` is set — that listener registers the pending reservation
|
|
1214
1658
|
// `driveControlRelayReservation` fills after bring-up.
|
|
@@ -1554,6 +1998,68 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1554
1998
|
const members = await this.listAuthorizedMembers(false);
|
|
1555
1999
|
this.authorizedControlPeers = new Set(members.map((m) => m.peerId));
|
|
1556
2000
|
log('refreshAuthorizedControlPeers(%s): %d authorized peer(s)', reason, this.authorizedControlPeers.size);
|
|
2001
|
+
// Remember the party's size for the NEXT launch's control-node repair yardstick
|
|
2002
|
+
// (see `enrolled-machine-store.ts`): this is the only place the count is known,
|
|
2003
|
+
// and the control node that needs it was built long before this ran. Recorded
|
|
2004
|
+
// from the snapshot just materialized — no second membership query.
|
|
2005
|
+
//
|
|
2006
|
+
// CONTROL-NETWORK ONLY, and that scope is load-bearing. Every enrolled machine
|
|
2007
|
+
// runs the control node by construction, so here the party's machine count IS
|
|
2008
|
+
// the count of machines serving the network — the quantity
|
|
2009
|
+
// `resolveRepairYardstick` asks for. It is NOT that quantity for a strand, which
|
|
2010
|
+
// launches only on machines whose embedder registered its sApp config (see
|
|
2011
|
+
// {@link addStrand}), so this number must never be routed to a strand node:
|
|
2012
|
+
// over-declaring pins Optimystic's repair corroboration floor at two peers and a
|
|
2013
|
+
// strand that can field only one can then never repair. `launchStrand` therefore
|
|
2014
|
+
// declares nothing; see the NOTE there.
|
|
2015
|
+
//
|
|
2016
|
+
// `+ 1` for this node, which is not in its own authorized set. An EMPTY snapshot
|
|
2017
|
+
// is recorded as 1 rather than skipped. That is deliberate, and the reason
|
|
2018
|
+
// is a party that genuinely SHRANK: if empty meant "record nothing", a party
|
|
2019
|
+
// whose other machines were all revoked would keep declaring its old, larger
|
|
2020
|
+
// number forever, and over-declaring is the unsafe direction — at a yardstick of
|
|
2021
|
+
// 3 or more Optimystic pins the repair corroboration floor at two peers, so a
|
|
2022
|
+
// cohort that can only field one peer could never repair at all. Recording 1
|
|
2023
|
+
// lets the number come back down; the floor then turns it into 2, which is
|
|
2024
|
+
// exactly what declaring nothing resolves to.
|
|
2025
|
+
//
|
|
2026
|
+
// NOTE: the cost of that choice is that a TRANSIENT empty read overwrites a good
|
|
2027
|
+
// remembered count with 1 — a membership snapshot is legitimately empty early in
|
|
2028
|
+
// a run on a node whose rows have not replicated yet or whose trusted-owner
|
|
2029
|
+
// anchor is still unseeded (`listAuthorizedMembers` authorizes no one without an
|
|
2030
|
+
// anchor). Bounded and safe: the next refresh once rows land re-records the real
|
|
2031
|
+
// count, and the only node that loses anything is one stopped inside that window,
|
|
2032
|
+
// which then declares 2 — today's behaviour — on its next launch. If a node ever
|
|
2033
|
+
// needs to declare the right number on its FIRST post-restart launch, the fix is
|
|
2034
|
+
// to distinguish "no rows yet" from "no members" at this site, not to skip empty.
|
|
2035
|
+
//
|
|
2036
|
+
// A removed peer stops counting at REVOCATION, not at reap: `queryCadrePeers`
|
|
2037
|
+
// drops rows whose `StampId` is retired, and `listAuthorizedMembers` reads
|
|
2038
|
+
// through it. That is why there is no reap hook writing this record.
|
|
2039
|
+
//
|
|
2040
|
+
// `void`: the store never rejects and logs its own persist failures, and this
|
|
2041
|
+
// method's contract is never-rejects. A failed write leaves the in-memory count
|
|
2042
|
+
// correct for this session and re-lands on the next refresh.
|
|
2043
|
+
void this.enrolledMachineStore?.record(this.authorizedControlPeers.size + 1);
|
|
2044
|
+
// Membership moved: give the control backfill another look at every
|
|
2045
|
+
// connected peer. The production join order is connect-then-authorize, so
|
|
2046
|
+
// a joiner's first catch-up pass was denied at the gate while its
|
|
2047
|
+
// connection stays up — without this re-arm it would wait for a
|
|
2048
|
+
// reconnect. Cheap and idempotent: caught-up peers are skipped before any
|
|
2049
|
+
// timer is set, a peer whose pass is in flight is deferred to the end of
|
|
2050
|
+
// that pass (the pass may already be past its gate check — this very
|
|
2051
|
+
// commit is what would authorize it), and the gate re-judges at push time.
|
|
2052
|
+
//
|
|
2053
|
+
// NOTE: every refresh re-arms every connected peer that is not yet caught
|
|
2054
|
+
// up, and each re-armed pass costs one `CadrePeer` query at the gate. A
|
|
2055
|
+
// connection that is admitted but never authorized — a configured
|
|
2056
|
+
// bootstrap or relay peer, the steady state for a NAT'd node — is
|
|
2057
|
+
// therefore re-judged on every membership write and every timed reconcile
|
|
2058
|
+
// tick, forever. Negligible at a cadre's handful of connections; if a node
|
|
2059
|
+
// ever holds many such connections, or refreshes get frequent, skip
|
|
2060
|
+
// re-arming peers whose last pass was denied under the SAME membership
|
|
2061
|
+
// snapshot instead of re-arming unconditionally.
|
|
2062
|
+
this.controlBackfill?.scheduleConnectedPeers();
|
|
1557
2063
|
} catch (error) {
|
|
1558
2064
|
log('refreshAuthorizedControlPeers(%s) failed — keeping previous snapshot: %o', reason, error);
|
|
1559
2065
|
}
|
|
@@ -1701,6 +2207,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1701
2207
|
const record = this.signSelfRecord(peerId, signingKey, addrs, null);
|
|
1702
2208
|
if (await this.seedBootstrapService.insertSelfPeerRecord(record)) {
|
|
1703
2209
|
if (this.committedAlone()) this.pendingSelfPeerWrite = true;
|
|
2210
|
+
this.noteSelfRecordPublished();
|
|
1704
2211
|
log('registerSelf: inserted own CadrePeer record (owner-signed, updatedAt=%d, %d addrs, sig=%s…)', record.updatedAt, addrs.length, record.sig.slice(0, 16));
|
|
1705
2212
|
return 'inserted';
|
|
1706
2213
|
}
|
|
@@ -1738,10 +2245,89 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1738
2245
|
const record = this.signSelfRecord(peerId, signingKey, addrs, current);
|
|
1739
2246
|
await this.controlDatabase.updateSelfPeerRecord(record);
|
|
1740
2247
|
if (this.committedAlone()) this.pendingSelfPeerWrite = true;
|
|
2248
|
+
this.noteSelfRecordPublished();
|
|
1741
2249
|
log('registerSelf: refreshed own CadrePeer record (updatedAt=%d, %d addrs, sig=%s…)', record.updatedAt, addrs.length, record.sig.slice(0, 16));
|
|
1742
2250
|
return 'refreshed';
|
|
1743
2251
|
}
|
|
1744
2252
|
|
|
2253
|
+
/**
|
|
2254
|
+
* Record that this node's own address record just landed, and re-arm the stale-record
|
|
2255
|
+
* warning. Called from BOTH publishing paths of {@link publishSelfRecord} and nowhere
|
|
2256
|
+
* else — a `skipped` publish wrote nothing, so it must not refresh the stamp.
|
|
2257
|
+
*/
|
|
2258
|
+
private noteSelfRecordPublished(): void {
|
|
2259
|
+
this.lastSelfRecordPublishAt = Date.now();
|
|
2260
|
+
this.selfRecordStaleWarned = false;
|
|
2261
|
+
}
|
|
2262
|
+
|
|
2263
|
+
/**
|
|
2264
|
+
* Report an abandoned control write to the embedding app.
|
|
2265
|
+
*
|
|
2266
|
+
* Every control write funnels through `ControlDatabase.lockedWithRetry`, which now tells
|
|
2267
|
+
* this node when it gives one up. Foreground writes ALSO reject to their caller; the
|
|
2268
|
+
* background ones ({@link startRecordRefresh}'s republish and the two replication
|
|
2269
|
+
* drains) are fired unawaited with a `debug`-only catch, so this event is the only thing
|
|
2270
|
+
* that reaches an app at all.
|
|
2271
|
+
*
|
|
2272
|
+
* Reporting only — the one operator-visible escalation lives on the republish path
|
|
2273
|
+
* ({@link escalateIfSelfRecordStale}), where the consequence is measurable.
|
|
2274
|
+
*/
|
|
2275
|
+
private noteControlWriteAbandoned(abandonment: ControlRetryAbandonment): void {
|
|
2276
|
+
log('Control write abandoned [%s] after %d/%d attempt(s) in %dms (%s): %o',
|
|
2277
|
+
abandonment.label ?? 'unlabelled', abandonment.attemptsMade, abandonment.attemptsAllowed,
|
|
2278
|
+
abandonment.elapsedMs, abandonment.reason, abandonment.error);
|
|
2279
|
+
this.emit('control:write-abandoned', abandonment);
|
|
2280
|
+
}
|
|
2281
|
+
|
|
2282
|
+
/**
|
|
2283
|
+
* Escalate ONE failed self-address republish to the operator — once — when the node has
|
|
2284
|
+
* not published its own record for longer than a record stays fresh.
|
|
2285
|
+
*
|
|
2286
|
+
* The bar is the CONSEQUENCE, not a failure count: a resolver discards a `CadrePeer`
|
|
2287
|
+
* record older than {@link DEFAULT_PEER_RECORD_MAX_AGE_MS} (15 minutes) and the heartbeat
|
|
2288
|
+
* re-stamps at half that, so by the time this gap opens every other machine in the party
|
|
2289
|
+
* has already stopped accepting this node's address. A count would fire far too early —
|
|
2290
|
+
* one failed heartbeat is already half the budget, and a single miss costs nothing.
|
|
2291
|
+
*
|
|
2292
|
+
* Say-once, re-armed by the next successful publish ({@link noteSelfRecordPublished}), so
|
|
2293
|
+
* a node that stays broken warns once rather than every 7.5 minutes.
|
|
2294
|
+
*
|
|
2295
|
+
* A node that has never published in this session is skipped: it has no record out there
|
|
2296
|
+
* to go stale, and the reason it has none (not a member yet, no signing key, revoked) is
|
|
2297
|
+
* already logged by `registerSelf` itself.
|
|
2298
|
+
*
|
|
2299
|
+
* Only a FAILED republish is checked. A heartbeat whose publish reports `skipped` neither
|
|
2300
|
+
* stamps nor warns, so a node that stops being able to publish without erroring — revoked,
|
|
2301
|
+
* or its row removed — goes quiet here. That is the intended reading: a revoked node has
|
|
2302
|
+
* nothing to publish and its unreachability is the point, not a fault to report.
|
|
2303
|
+
*
|
|
2304
|
+
* NOTE: the second `console.*` in this library, and the reason
|
|
2305
|
+
* {@link warnIfAnnounceAddrsDiscardRelay}'s note says a third should not simply be added:
|
|
2306
|
+
* both are operator warnings about a configuration/health condition an embedder cannot
|
|
2307
|
+
* see otherwise, and both have an event beside them (`control:write-abandoned` carries
|
|
2308
|
+
* the underlying write failure). A third such condition should surface through
|
|
2309
|
+
* {@link CadreNodeEvents} alone unless it likewise degrades the whole party in silence.
|
|
2310
|
+
*/
|
|
2311
|
+
private escalateIfSelfRecordStale(reason: string, error: unknown): void {
|
|
2312
|
+
const publishedAt = this.lastSelfRecordPublishAt;
|
|
2313
|
+
if (publishedAt === null || this.selfRecordStaleWarned) {
|
|
2314
|
+
return;
|
|
2315
|
+
}
|
|
2316
|
+
const staleForMs = Date.now() - publishedAt;
|
|
2317
|
+
if (staleForMs < DEFAULT_PEER_RECORD_MAX_AGE_MS) {
|
|
2318
|
+
return;
|
|
2319
|
+
}
|
|
2320
|
+
this.selfRecordStaleWarned = true;
|
|
2321
|
+
console.warn(
|
|
2322
|
+
`[sereus] this machine has not published its own address record for ${Math.round(staleForMs / 60_000)} `
|
|
2323
|
+
+ `minutes. The latest attempt (${reason}) failed with: `
|
|
2324
|
+
+ `${error instanceof Error ? error.message : String(error)}. `
|
|
2325
|
+
+ 'Other machines in the party discard an address record older than '
|
|
2326
|
+
+ `${Math.round(DEFAULT_PEER_RECORD_MAX_AGE_MS / 60_000)} minutes, so any of them not already `
|
|
2327
|
+
+ 'connected to this one can no longer reach it. Publishing keeps retrying; this is reported once.'
|
|
2328
|
+
);
|
|
2329
|
+
}
|
|
2330
|
+
|
|
1745
2331
|
/**
|
|
1746
2332
|
* Sign this node's address record for publication, stamped strictly later than
|
|
1747
2333
|
* `existing` (the row it is about to replace, or null for a first insert) so a
|
|
@@ -1787,6 +2373,30 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1787
2373
|
return null;
|
|
1788
2374
|
}
|
|
1789
2375
|
|
|
2376
|
+
/**
|
|
2377
|
+
* Does the `Strand` row name THIS machine as its founder — i.e. is its
|
|
2378
|
+
* `FounderOwnerKey` this node's own owner key (the key behind its PeerId)?
|
|
2379
|
+
* A null/absent column (a consent-seated strand, or a hand-built row) and a
|
|
2380
|
+
* node with no owner key both derive `false`: without a positive match this
|
|
2381
|
+
* machine must attach, never bootstrap. Pure key derivation, no I/O — cheap
|
|
2382
|
+
* enough for {@link launchStrand}'s tracked-instance early return.
|
|
2383
|
+
*
|
|
2384
|
+
* NOTE: reads "one owner key per machine" — the reference model, where the owner
|
|
2385
|
+
* key IS the key behind the PeerId. Two machines running the SAME identity key
|
|
2386
|
+
* would both derive `true` and each bootstrap on its own replica, which is the
|
|
2387
|
+
* double-`Header` hazard this derivation exists to avoid. Unreachable today: that
|
|
2388
|
+
* configuration also gives both machines one PeerId, which already breaks control
|
|
2389
|
+
* networking well before any strand launches. Revisit if machines ever share an
|
|
2390
|
+
* owner key while holding distinct transport identities — the derivation would
|
|
2391
|
+
* then need a per-machine discriminator (the `CadrePeer` PeerId) on the row.
|
|
2392
|
+
*/
|
|
2393
|
+
private isSelfFoundedRow(strand: StrandRow): boolean {
|
|
2394
|
+
if (strand.FounderOwnerKey == null) {
|
|
2395
|
+
return false;
|
|
2396
|
+
}
|
|
2397
|
+
return strand.FounderOwnerKey === this.getSelfSigningKey()?.publicKeyB64;
|
|
2398
|
+
}
|
|
2399
|
+
|
|
1790
2400
|
/**
|
|
1791
2401
|
* The owner keypair (base64url Ed25519) derived from this node's resolved
|
|
1792
2402
|
* identity key. In the single-key reference model the owner signing key is
|
|
@@ -1839,7 +2449,12 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1839
2449
|
}
|
|
1840
2450
|
|
|
1841
2451
|
const republish = (reason: string) => {
|
|
1842
|
-
void this.registerSelf().catch((error) =>
|
|
2452
|
+
void this.registerSelf().catch((error) => {
|
|
2453
|
+
log('Record refresh (%s) failed: %o', reason, error);
|
|
2454
|
+
// The only failure in this file that degrades the whole party on its own, so it is
|
|
2455
|
+
// the only one that escalates past `debug`. See escalateIfSelfRecordStale.
|
|
2456
|
+
this.escalateIfSelfRecordStale(reason, error);
|
|
2457
|
+
});
|
|
1843
2458
|
};
|
|
1844
2459
|
const reconcile = (reason: string) => {
|
|
1845
2460
|
void this.reconcileControlCohort().catch((error) =>
|
|
@@ -1860,7 +2475,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1860
2475
|
// `CadrePeer` row, so its boot-time publish does nothing, and the two triggers
|
|
1861
2476
|
// wired here are the only ones after that: `self:peer:update` needs an ADDRESS
|
|
1862
2477
|
// CHANGE, and a relay-only node's addresses stop changing the moment its
|
|
1863
|
-
// reservation lands — inside `start()`, before this listener exists
|
|
2478
|
+
// reservation lands — inside `start()`, before this listener exists (on the
|
|
2479
|
+
// `network.requireRelay: false` posture the reservation may instead land LATER,
|
|
2480
|
+
// from the retry supervisor, and then this listener is what publishes the
|
|
2481
|
+
// `/p2p-circuit` address the boot-time publish could not carry). Harmless
|
|
1864
2482
|
// where a late-authorized node's addresses keep churning (NAT/relay rotation
|
|
1865
2483
|
// fire the event anyway), and invisible while parties authorize members before
|
|
1866
2484
|
// they boot. If late enrollment becomes the normal path — an invited phone is
|
|
@@ -1927,10 +2545,16 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1927
2545
|
this.hasControlConnection = false;
|
|
1928
2546
|
this.reconstructedLocalOnlyWrites = false;
|
|
1929
2547
|
this.reissuedHeldRevocations = false;
|
|
2548
|
+
this.revocationLedgerOpened = false;
|
|
1930
2549
|
this.pendingPeerWrites.clear();
|
|
1931
2550
|
this.pendingRevocations.clear();
|
|
1932
2551
|
this.pendingSelfPeerWrite = false;
|
|
1933
2552
|
this.pendingSelfDeviceWrite = false;
|
|
2553
|
+
// Self-record publication history is per-session too: a restarted node re-publishes
|
|
2554
|
+
// from scratch, and carrying the previous lifetime's stamp forward would either
|
|
2555
|
+
// suppress the stale-record warning or fire it on a node that just came up.
|
|
2556
|
+
this.lastSelfRecordPublishAt = null;
|
|
2557
|
+
this.selfRecordStaleWarned = false;
|
|
1934
2558
|
}
|
|
1935
2559
|
|
|
1936
2560
|
/**
|
|
@@ -2028,7 +2652,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2028
2652
|
* Applied by ALL THREE sources of control-dial candidates, so no call site
|
|
2029
2653
|
* invents its own rule and none can produce a mixed list: the signed record
|
|
2030
2654
|
* ({@link resolvePeerAddrs}), the libp2p address book
|
|
2031
|
-
* ({@link peerStoreAddrs}), and a
|
|
2655
|
+
* ({@link peerStoreAddrs}), and a retained out-of-band dial target
|
|
2032
2656
|
* ({@link bootstrapDialAddrs}).
|
|
2033
2657
|
*
|
|
2034
2658
|
* An address naming a DIFFERENT trailing peer id is dropped: it does not reach
|
|
@@ -2055,20 +2679,25 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2055
2679
|
* One address bound to `peerId`, or `null` when it cannot be — logged either
|
|
2056
2680
|
* way, never thrown, so every caller's list-shaping stays total.
|
|
2057
2681
|
*
|
|
2682
|
+
* Used both to normalize addresses this node DIALS (a `CadrePeer` row, a
|
|
2683
|
+
* retained bootstrap addr) and to normalize the ones it ANNOUNCES
|
|
2684
|
+
* ({@link getStrandMultiaddrs}); the rule is the same either way — an address
|
|
2685
|
+
* that does not reach `peerId` is not an address for `peerId`.
|
|
2686
|
+
*
|
|
2058
2687
|
* `withTrailingPeerId` encapsulates `/p2p/<peerId>`, which throws on a peer id
|
|
2059
2688
|
* that does not parse. Unreachable for a `CadrePeer` row (the binding gate
|
|
2060
|
-
* above parsed it already), reachable for a
|
|
2061
|
-
* ({@link bootstrapDialAddrs}).
|
|
2689
|
+
* above parsed it already) and for our own node's id, reachable for a
|
|
2690
|
+
* retained one ({@link bootstrapDialAddrs}).
|
|
2062
2691
|
*/
|
|
2063
2692
|
private bindAddrToPeer(addr: Multiaddr, peerId: string): Multiaddr | null {
|
|
2064
2693
|
try {
|
|
2065
2694
|
const bound = withTrailingPeerId(addr, peerId);
|
|
2066
2695
|
if (!bound) {
|
|
2067
|
-
log('
|
|
2696
|
+
log('bindAddrToPeer: dropping addr %s — it names a peer other than %s', addr.toString(), peerId);
|
|
2068
2697
|
}
|
|
2069
2698
|
return bound;
|
|
2070
2699
|
} catch (error) {
|
|
2071
|
-
log('
|
|
2700
|
+
log('bindAddrToPeer: cannot bind %s to %s: %o', addr.toString(), peerId, error);
|
|
2072
2701
|
return null;
|
|
2073
2702
|
}
|
|
2074
2703
|
}
|
|
@@ -2109,29 +2738,35 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2109
2738
|
* `self:peer:update` (all wired in {@link startRecordRefresh}).
|
|
2110
2739
|
*
|
|
2111
2740
|
* Concurrent triggers collapse into a single in-flight pass
|
|
2112
|
-
* (see {@link reconcileControlCohortInFlight}) so two passes never double-dial
|
|
2741
|
+
* (see {@link reconcileControlCohortInFlight}) so two passes never double-dial;
|
|
2742
|
+
* a call that joins an in-flight pass resolves to THAT pass's result.
|
|
2113
2743
|
* Best-effort throughout: a failure to resolve/dial any one sibling is logged
|
|
2114
2744
|
* and the pass continues; the whole pass is a no-op when the node is alone.
|
|
2745
|
+
*
|
|
2746
|
+
* Resolves to the peers the pass dialled ({@link ControlCohortReconcileResult}),
|
|
2747
|
+
* so a caller can tell a link this pass opened from one something else opened —
|
|
2748
|
+
* the pass skips a peer that is already connected. The timer and event triggers
|
|
2749
|
+
* discard it.
|
|
2115
2750
|
*/
|
|
2116
|
-
async reconcileControlCohort(): Promise<
|
|
2751
|
+
async reconcileControlCohort(): Promise<ControlCohortReconcileResult> {
|
|
2117
2752
|
if (this.reconcileControlCohortInFlight) {
|
|
2118
2753
|
return this.reconcileControlCohortInFlight;
|
|
2119
2754
|
}
|
|
2120
2755
|
const op = this.runReconcileControlCohort();
|
|
2121
2756
|
this.reconcileControlCohortInFlight = op;
|
|
2122
2757
|
try {
|
|
2123
|
-
await op;
|
|
2758
|
+
return await op;
|
|
2124
2759
|
} finally {
|
|
2125
2760
|
this.reconcileControlCohortInFlight = null;
|
|
2126
2761
|
}
|
|
2127
2762
|
}
|
|
2128
2763
|
|
|
2129
2764
|
/** Body of {@link reconcileControlCohort}; serialised by its single-flight guard. */
|
|
2130
|
-
private async runReconcileControlCohort(): Promise<
|
|
2765
|
+
private async runReconcileControlCohort(): Promise<ControlCohortReconcileResult> {
|
|
2131
2766
|
// Shutdown / not-yet-started guard (mirrors publishSelfRecord). A pass that
|
|
2132
2767
|
// fires after stop() began must early-return rather than touch a torn-down node.
|
|
2133
2768
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
2134
|
-
return;
|
|
2769
|
+
return { dialed: [] };
|
|
2135
2770
|
}
|
|
2136
2771
|
// Ride this pass's cadence to refresh the per-stream gate's materialized
|
|
2137
2772
|
// authorized set — membership changes that ARRIVED BY REPLICATION (rather
|
|
@@ -2141,11 +2776,15 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2141
2776
|
// CadrePeer query (two reads per pass), plus a third from
|
|
2142
2777
|
// `refreshStrandPeerAddrs` on the passes where a strand is due AND this node
|
|
2143
2778
|
// holds a control connection (one read for the whole pass, not one per
|
|
2144
|
-
// strand)
|
|
2145
|
-
//
|
|
2779
|
+
// strand). Each also reads Revocation first. Before the Revocation ledger
|
|
2780
|
+
// marker exists (filed below, once connected) that block is missing, so every
|
|
2781
|
+
// one of those reads consults the cohort about it; once the marker exists every
|
|
2782
|
+
// block they touch is held and none of them does (both states pinned in
|
|
2783
|
+
// control-founding-consult-budget.spec.ts). If those reads ever get costly,
|
|
2784
|
+
// share one row-set across all three.
|
|
2146
2785
|
await this.refreshMembershipGate('reconcile');
|
|
2147
2786
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
2148
|
-
return;
|
|
2787
|
+
return { dialed: [] };
|
|
2149
2788
|
}
|
|
2150
2789
|
// Refresh relay delegate grants BEFORE the sibling enumeration below: a
|
|
2151
2790
|
// solo cadre with a party relay has no siblings to dial but must still keep
|
|
@@ -2153,7 +2792,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2153
2792
|
// the reservation and faces the connection gate again).
|
|
2154
2793
|
await this.refreshDelegateGrants();
|
|
2155
2794
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
2156
|
-
return;
|
|
2795
|
+
return { dialed: [] };
|
|
2157
2796
|
}
|
|
2158
2797
|
// Then re-warm each running strand's own address book from its siblings, on
|
|
2159
2798
|
// its own (much longer) throttle. Same reasoning as warmSiblingAddrBook
|
|
@@ -2161,7 +2800,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2161
2800
|
// layer under cadre-core dials strand peers by bare peer id.
|
|
2162
2801
|
await this.refreshStrandPeerAddrs();
|
|
2163
2802
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
2164
|
-
return;
|
|
2803
|
+
return { dialed: [] };
|
|
2165
2804
|
}
|
|
2166
2805
|
const selfPeerId = this.controlNode.peerId.toString();
|
|
2167
2806
|
|
|
@@ -2203,7 +2842,28 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2203
2842
|
log('reconcileControlCohort: reap pass failed (continuing): %o', error);
|
|
2204
2843
|
}
|
|
2205
2844
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
2206
|
-
return;
|
|
2845
|
+
return { dialed: [] };
|
|
2846
|
+
}
|
|
2847
|
+
|
|
2848
|
+
// File the Revocation ledger marker once, so that table stops being a never-written
|
|
2849
|
+
// block, which the storage layer re-checks with the cohort on every read (and every
|
|
2850
|
+
// membership lookup and guarded insert reads it). Connected-only for the reap's
|
|
2851
|
+
// reason above: a marker committed alone is local-only, and one filed by a
|
|
2852
|
+
// disconnected owner while another machine creates the same collection is exactly
|
|
2853
|
+
// that fork. Owner-only, and at most once per process — see openRevocationLedgerIfDue.
|
|
2854
|
+
//
|
|
2855
|
+
// NOTE: accepted tradeoff — a solo founder does not file the marker until it holds a
|
|
2856
|
+
// control connection (normally its first sibling; a relay or bootstrap connection also
|
|
2857
|
+
// counts — see getControlConnectionCount — and is the reap's same residual), so until
|
|
2858
|
+
// then it keeps paying one local findCluster per read of
|
|
2859
|
+
// the missing Revocation block (0.009 ms each, measured upstream; no network work on
|
|
2860
|
+
// a cohort of one). Fork safety weighed over that and kept; revisit if findCluster
|
|
2861
|
+
// ever shows up as material in a device profile, or if a solo-founding marker can be
|
|
2862
|
+
// made fork-safe (e.g. filed inside the genesis transaction, before any other machine
|
|
2863
|
+
// can hold the party's collections).
|
|
2864
|
+
await this.openRevocationLedgerIfDue();
|
|
2865
|
+
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
2866
|
+
return { dialed: [] };
|
|
2207
2867
|
}
|
|
2208
2868
|
}
|
|
2209
2869
|
|
|
@@ -2218,12 +2878,11 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2218
2878
|
// because filling this table needs a connection and getting a connection
|
|
2219
2879
|
// needs this table. Fall back to the seed's bootstrap addresses; a solo
|
|
2220
2880
|
// cadre has none and this is a no-op. Must not throw or busy-loop.
|
|
2221
|
-
await this.dialColdStartBootstrap();
|
|
2222
|
-
return;
|
|
2881
|
+
return { dialed: await this.dialColdStartBootstrap() };
|
|
2223
2882
|
}
|
|
2224
2883
|
// Re-guard after the await: a stop() may have raced the membership read.
|
|
2225
2884
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
2226
|
-
return;
|
|
2885
|
+
return { dialed: [] };
|
|
2227
2886
|
}
|
|
2228
2887
|
|
|
2229
2888
|
// 2. Classify backbone (owner) members and select a bounded dial set.
|
|
@@ -2234,7 +2893,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2234
2893
|
// the anchor if owner status here ever gates something trusted.
|
|
2235
2894
|
const ownerKeys = await this.controlDatabase.getOwnerKeys();
|
|
2236
2895
|
if (!this._running || !this.controlNode) {
|
|
2237
|
-
return;
|
|
2896
|
+
return { dialed: [] };
|
|
2238
2897
|
}
|
|
2239
2898
|
const targetDegree = this.config.network?.controlCohort?.targetDegree
|
|
2240
2899
|
?? DEFAULT_CONTROL_COHORT_TARGET_DEGREE;
|
|
@@ -2249,7 +2908,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2249
2908
|
// keeping what resolved for the dial loop below.
|
|
2250
2909
|
const resolved = await this.warmSiblingAddrBook(siblings);
|
|
2251
2910
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
2252
|
-
return;
|
|
2911
|
+
return { dialed: [] };
|
|
2253
2912
|
}
|
|
2254
2913
|
|
|
2255
2914
|
// 4. Skip already-connected peers (no re-dial / churn for live connections).
|
|
@@ -2257,20 +2916,46 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2257
2916
|
|
|
2258
2917
|
// 5. Dial each selected, not-yet-connected sibling, best-effort, from the
|
|
2259
2918
|
// addresses step 3 already resolved for it.
|
|
2260
|
-
|
|
2919
|
+
const dialed: string[] = [];
|
|
2261
2920
|
for (const sibling of dials) {
|
|
2262
2921
|
if (!this._running || !this.controlNode) {
|
|
2263
|
-
return;
|
|
2922
|
+
return { dialed };
|
|
2264
2923
|
}
|
|
2265
2924
|
if (connected.has(sibling.peerId)) {
|
|
2266
2925
|
continue;
|
|
2267
2926
|
}
|
|
2268
2927
|
if (await this.dialControlSibling(sibling, resolved.get(sibling.peerId) ?? [])) {
|
|
2269
|
-
dialed
|
|
2928
|
+
dialed.push(sibling.peerId);
|
|
2270
2929
|
}
|
|
2271
2930
|
}
|
|
2272
2931
|
log('reconcileControlCohort: pass complete (siblings=%d, selected=%d, dialed=%d)',
|
|
2273
|
-
siblings.length, dials.length, dialed);
|
|
2932
|
+
siblings.length, dials.length, dialed.length);
|
|
2933
|
+
return { dialed };
|
|
2934
|
+
}
|
|
2935
|
+
|
|
2936
|
+
/**
|
|
2937
|
+
* The owner-only half of the reconcile pass's ledger-marker step: file the singleton
|
|
2938
|
+
* `Revocation` marker ({@link SeedBootstrapService.openRevocationLedger}) unless this
|
|
2939
|
+
* process has already seen it filed. The connectivity gate is the caller's
|
|
2940
|
+
* ({@link runReconcileControlCohort}); why the marker exists is on
|
|
2941
|
+
* {@link ControlDatabase.openRevocationLedger}.
|
|
2942
|
+
*
|
|
2943
|
+
* Sets {@link revocationLedgerOpened} on `'opened'` or `'already-open'`. A node that
|
|
2944
|
+
* cannot sign as an owner does nothing: it picks the owner's marker up the first time it
|
|
2945
|
+
* reads the table, like any other replicated row. Best-effort like every step of the
|
|
2946
|
+
* pass — a failure is logged and leaves the flag clear, so the next connected pass retries.
|
|
2947
|
+
*/
|
|
2948
|
+
private async openRevocationLedgerIfDue(): Promise<void> {
|
|
2949
|
+
if (this.revocationLedgerOpened || !this.seedBootstrapService?.canAuthorize()) {
|
|
2950
|
+
return;
|
|
2951
|
+
}
|
|
2952
|
+
try {
|
|
2953
|
+
const outcome = await this.seedBootstrapService.openRevocationLedger();
|
|
2954
|
+
this.revocationLedgerOpened = true;
|
|
2955
|
+
log('reconcileControlCohort: revocation ledger marker %s', outcome);
|
|
2956
|
+
} catch (error) {
|
|
2957
|
+
log('reconcileControlCohort: filing the revocation ledger marker failed (retrying next pass): %o', error);
|
|
2958
|
+
}
|
|
2274
2959
|
}
|
|
2275
2960
|
|
|
2276
2961
|
/**
|
|
@@ -2292,6 +2977,9 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2292
2977
|
* resolves to nothing (revoked, stale, untrusted) is not written at all, so its
|
|
2293
2978
|
* existing entry ages out on its own.
|
|
2294
2979
|
*
|
|
2980
|
+
* The same resolution also keeps a sibling's retained out-of-band dial target
|
|
2981
|
+
* current ({@link refreshDialHint}).
|
|
2982
|
+
*
|
|
2295
2983
|
* NOTE: this resolves EVERY sibling serially before the dial loop below runs,
|
|
2296
2984
|
* so it costs one record query per sibling per reconcile pass (~15s) and each
|
|
2297
2985
|
* one delays the pass's first dial. A cadre is a handful of devices, so today
|
|
@@ -2301,6 +2989,9 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2301
2989
|
private async warmSiblingAddrBook(siblings: CohortPeerRow[]): Promise<Map<string, Multiaddr[]>> {
|
|
2302
2990
|
const resolved = new Map<string, Multiaddr[]>();
|
|
2303
2991
|
const counts: Record<MergeAddrsResult, number> = { merged: 0, restamped: 0, skipped: 0, failed: 0 };
|
|
2992
|
+
// One copy of the store's map for the whole loop; a refresh replaces an entry
|
|
2993
|
+
// rather than mutating it, so the copy stays a consistent "before" view.
|
|
2994
|
+
const hints = this.bootstrapPeerStore?.all();
|
|
2304
2995
|
for (const sibling of siblings) {
|
|
2305
2996
|
if (!this._running || !this.controlNode) {
|
|
2306
2997
|
break;
|
|
@@ -2313,6 +3004,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2313
3004
|
if (!this._running || !controlNode) {
|
|
2314
3005
|
break;
|
|
2315
3006
|
}
|
|
3007
|
+
this.refreshDialHint(sibling.peerId, addrs, hints?.get(sibling.peerId)?.addrs);
|
|
2316
3008
|
// `mergePeerAddrs` owns the whole best-effort contract — including parsing
|
|
2317
3009
|
// a malformed `CadrePeer.PeerId` — so one bad row cannot abort the pass.
|
|
2318
3010
|
counts[await mergePeerAddrs(controlNode, sibling.peerId, addrs)]++;
|
|
@@ -2322,6 +3014,41 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2322
3014
|
return resolved;
|
|
2323
3015
|
}
|
|
2324
3016
|
|
|
3017
|
+
/**
|
|
3018
|
+
* Replace a sibling's retained out-of-band dial target (see
|
|
3019
|
+
* {@link bootstrapPeerStore}) with the addresses its signed record just resolved
|
|
3020
|
+
* to, when the two differ — so an address change this node saw while the record
|
|
3021
|
+
* was fresh (a new port, a new LAN address) is what it dials after a relaunch
|
|
3022
|
+
* that outlives the record's freshness window.
|
|
3023
|
+
*
|
|
3024
|
+
* Only a sibling that already HAS an entry (`retained`) is refreshed: the store
|
|
3025
|
+
* holds peers learned out of band and must not grow into a copy of every
|
|
3026
|
+
* sibling's addresses. An empty resolution leaves the entry alone — a stale or
|
|
3027
|
+
* not-yet-signed record is exactly when the entry is needed. The comparison is
|
|
3028
|
+
* order-insensitive and runs on the retained addresses after binding them to the
|
|
3029
|
+
* peer id, so an entry recorded without `/p2p/` suffixes is not rewritten merely
|
|
3030
|
+
* for lacking them.
|
|
3031
|
+
*
|
|
3032
|
+
* NOTE: replaces rather than merges, so the entry becomes exactly what the
|
|
3033
|
+
* sibling announces. If a node's signed record ever lists fewer addresses this
|
|
3034
|
+
* node can reach than it was added with (say an announce override naming only a
|
|
3035
|
+
* public address, for a node added by its LAN address), a relaunch dials the
|
|
3036
|
+
* worse set; merge the two lists here if that shows up.
|
|
3037
|
+
*/
|
|
3038
|
+
private refreshDialHint(peerId: string, resolved: Multiaddr[], retained: string[] | undefined): void {
|
|
3039
|
+
if (!retained || resolved.length === 0) {
|
|
3040
|
+
return;
|
|
3041
|
+
}
|
|
3042
|
+
// Both lists are de-duplicated by `normalizeDialAddrs`, so equal size plus
|
|
3043
|
+
// containment is set equality.
|
|
3044
|
+
const current = resolved.map((addr) => addr.toString());
|
|
3045
|
+
const previous = new Set(this.bootstrapDialAddrs(peerId, retained).map((addr) => addr.toString()));
|
|
3046
|
+
if (current.length === previous.size && current.every((addr) => previous.has(addr))) {
|
|
3047
|
+
return;
|
|
3048
|
+
}
|
|
3049
|
+
this.retainDialTarget(peerId, current, 'refreshDialHint');
|
|
3050
|
+
}
|
|
3051
|
+
|
|
2325
3052
|
/**
|
|
2326
3053
|
* {@link resolvePeerAddrs} for one sibling, best-effort: a control-DB read
|
|
2327
3054
|
* failure yields `[]` (and the cold-start fallback then gets its turn) rather
|
|
@@ -2337,26 +3064,32 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2337
3064
|
}
|
|
2338
3065
|
|
|
2339
3066
|
/**
|
|
2340
|
-
*
|
|
3067
|
+
* Time limits for dialing ONE control-network peer from its candidate
|
|
3068
|
+
* addresses ({@link dialPeerAddrs}): the reconcile pass's steady-state sibling
|
|
2341
3069
|
* ({@link dialControlSibling}) and cold-start bootstrap peer
|
|
2342
|
-
* ({@link dialBootstrapPeer})
|
|
2343
|
-
*
|
|
3070
|
+
* ({@link dialBootstrapPeer}) dials, and — handed to every
|
|
3071
|
+
* {@link SeedBootstrapService} this node builds — `applySeed`'s owner dials and
|
|
3072
|
+
* `dialInvite`.
|
|
2344
3073
|
*
|
|
2345
|
-
* See {@link DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS} for why
|
|
2346
|
-
*
|
|
3074
|
+
* See {@link DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS} for why a peer's dial is
|
|
3075
|
+
* bounded as a whole, and {@link DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS}
|
|
3076
|
+
* for why each address is bounded as well.
|
|
2347
3077
|
*/
|
|
2348
|
-
private
|
|
2349
|
-
|
|
2350
|
-
|
|
3078
|
+
private controlDialBudget(): PeerDialBudget {
|
|
3079
|
+
const cohort = this.config.network?.controlCohort;
|
|
3080
|
+
return {
|
|
3081
|
+
perAddressMs: cohort?.perAddressDialTimeoutMs ?? DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS,
|
|
3082
|
+
totalMs: cohort?.dialTimeoutMs ?? DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS,
|
|
3083
|
+
};
|
|
2351
3084
|
}
|
|
2352
3085
|
|
|
2353
3086
|
/**
|
|
2354
3087
|
* Dial one sibling from its already-resolved addresses, best-effort. Returns
|
|
2355
|
-
* whether
|
|
3088
|
+
* whether the dial resolved (false when no address resolves or the dial fails).
|
|
2356
3089
|
*
|
|
2357
3090
|
* A per-peer failure (NAT, offline, relay down, connection-gater denial, or
|
|
2358
|
-
* the {@link
|
|
2359
|
-
*
|
|
3091
|
+
* the {@link controlDialBudget} expiring) is logged and swallowed so one
|
|
3092
|
+
* unreachable sibling never aborts the pass — exactly like
|
|
2360
3093
|
* {@link SeedBootstrapService.applySeed}'s owner-dial loop. A failed dial is
|
|
2361
3094
|
* simply retried on the next pass.
|
|
2362
3095
|
*/
|
|
@@ -2372,10 +3105,11 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2372
3105
|
}
|
|
2373
3106
|
try {
|
|
2374
3107
|
log('reconcileControlCohort: dialing sibling %s (%d addr(s))', sibling.peerId, addrs.length);
|
|
2375
|
-
await
|
|
2376
|
-
|
|
2377
|
-
|
|
2378
|
-
(
|
|
3108
|
+
await dialPeerAddrs(
|
|
3109
|
+
controlNode,
|
|
3110
|
+
addrs,
|
|
3111
|
+
this.controlDialBudget(),
|
|
3112
|
+
`reconcileControlCohort dial of sibling ${sibling.peerId}`
|
|
2379
3113
|
);
|
|
2380
3114
|
return true;
|
|
2381
3115
|
} catch (error) {
|
|
@@ -2385,20 +3119,66 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2385
3119
|
}
|
|
2386
3120
|
|
|
2387
3121
|
/**
|
|
2388
|
-
* A sibling's control-network dial addresses for the reconcile pass
|
|
3122
|
+
* A sibling's control-network dial addresses for the reconcile pass, from the
|
|
3123
|
+
* first of three sources that yields any:
|
|
3124
|
+
*
|
|
3125
|
+
* 1. the signed, fresh, trust-gated control addresses `resolved` for it by
|
|
3126
|
+
* {@link warmSiblingAddrBook} — passed in rather than re-resolved, so the
|
|
3127
|
+
* pass makes one record query per sibling;
|
|
3128
|
+
* 2. the libp2p address book ({@link peerStoreAddrs}) — the entries `applySeed`
|
|
3129
|
+
* and identify put there, until they age out;
|
|
3130
|
+
* 3. the sibling's retained out-of-band dial target ({@link retainedDialAddrs}).
|
|
3131
|
+
* For a node that cannot listen, this is the only source for a sibling it
|
|
3132
|
+
* added but has not yet connected to: the added node's row stays unsigned
|
|
3133
|
+
* until it self-publishes, which needs the connection this dial opens. It is
|
|
3134
|
+
* also what a relaunch that outlived the record's freshness window uses.
|
|
3135
|
+
*
|
|
3136
|
+
* Returns `[]` (never throws) when none yields an address — that sibling is
|
|
3137
|
+
* skipped this pass.
|
|
2389
3138
|
*
|
|
2390
|
-
*
|
|
2391
|
-
* `
|
|
2392
|
-
*
|
|
2393
|
-
*
|
|
2394
|
-
*
|
|
2395
|
-
*
|
|
3139
|
+
* The list may name transports this node cannot dial (a lent node reports TCP
|
|
3140
|
+
* and `/ws` addresses to a phone that dials WebSockets only). That needs no
|
|
3141
|
+
* filtering here: libp2p's dial queue (`calculateMultiaddrs`, libp2p 3.1.3)
|
|
3142
|
+
* rejects an address no transport can dial before touching the network, so
|
|
3143
|
+
* such an address costs a log line.
|
|
3144
|
+
*
|
|
3145
|
+
* The list is NOT handed to one `dial()`. libp2p tries a multi-address dial's
|
|
3146
|
+
* addresses one after another under a single deadline, with no limit per
|
|
3147
|
+
* address, and sorts loopback addresses last — so one or two addresses that
|
|
3148
|
+
* never answer used up the whole deadline before the address that worked was
|
|
3149
|
+
* tried. {@link dialPeerAddrs} dials each address on its own time limit instead.
|
|
2396
3150
|
*/
|
|
2397
3151
|
private async resolveControlDialAddrs(peerId: string, resolved: Multiaddr[]): Promise<Multiaddr[]> {
|
|
2398
3152
|
if (resolved.length > 0) {
|
|
2399
3153
|
return resolved;
|
|
2400
3154
|
}
|
|
2401
|
-
|
|
3155
|
+
const booked = await this.peerStoreAddrs(peerId);
|
|
3156
|
+
if (booked.length > 0) {
|
|
3157
|
+
return booked;
|
|
3158
|
+
}
|
|
3159
|
+
return this.retainedDialAddrs(peerId);
|
|
3160
|
+
}
|
|
3161
|
+
|
|
3162
|
+
/**
|
|
3163
|
+
* A sibling's retained out-of-band dial target (see {@link bootstrapPeerStore}),
|
|
3164
|
+
* bound to its peer id through {@link bootstrapDialAddrs}; `[]` when it has none.
|
|
3165
|
+
*
|
|
3166
|
+
* These addresses are never merged into the libp2p address book: they are
|
|
3167
|
+
* unverified, and the address book takes only verified ones (see
|
|
3168
|
+
* `mergePeerAddrs`). Layers that dial by bare peer id use the connection this
|
|
3169
|
+
* dial opens, and after it drops the next reconcile pass dials again.
|
|
3170
|
+
*
|
|
3171
|
+
* NOTE: an entry is consulted here only for a current sibling, so a peer revoked
|
|
3172
|
+
* by another owner is not dialed from it, even though the entry stays (only this
|
|
3173
|
+
* node's own {@link removePeer} forgets one). The cold-start branch
|
|
3174
|
+
* ({@link dialColdStartBootstrap}) still dials every entry while there are no
|
|
3175
|
+
* siblings at all; if entries for revoked peers ever cause dial churn there,
|
|
3176
|
+
* prune entries whose peer has a retired `CadrePeer` row during the membership
|
|
3177
|
+
* refresh.
|
|
3178
|
+
*/
|
|
3179
|
+
private retainedDialAddrs(peerId: string): Multiaddr[] {
|
|
3180
|
+
const entry = this.bootstrapPeerStore?.all().get(peerId);
|
|
3181
|
+
return entry ? this.bootstrapDialAddrs(peerId, entry.addrs) : [];
|
|
2402
3182
|
}
|
|
2403
3183
|
|
|
2404
3184
|
/**
|
|
@@ -2450,39 +3230,57 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2450
3230
|
* signature-checked against a trust-anchored signer before this runs, and the
|
|
2451
3231
|
* flag only *selects a dial target*; a dial grants no authority.
|
|
2452
3232
|
*
|
|
2453
|
-
*
|
|
2454
|
-
* that applies a seed minted after it joined finds itself in the owner list;
|
|
2455
|
-
* retaining that would make the cold-start pass dial this node forever (the
|
|
2456
|
-
* steady-state pass filters self out of its sibling list for the same reason).
|
|
3233
|
+
* Addressless peers and self are skipped by {@link retainDialTarget}.
|
|
2457
3234
|
*/
|
|
2458
3235
|
private recordSeedBootstrapPeers(seed: ControlNetworkSeed): void {
|
|
3236
|
+
for (const peer of seed.peers) {
|
|
3237
|
+
if (peer.isOwner) {
|
|
3238
|
+
this.retainDialTarget(peer.peerId, peer.multiaddrs, 'recordSeedBootstrapPeers');
|
|
3239
|
+
}
|
|
3240
|
+
}
|
|
3241
|
+
}
|
|
3242
|
+
|
|
3243
|
+
/**
|
|
3244
|
+
* Retain (or replace) one peer's out-of-band dial target in
|
|
3245
|
+
* {@link bootstrapPeerStore}. Shared by every writer: seed intake, {@link addDrone}
|
|
3246
|
+
* and {@link refreshDialHint}.
|
|
3247
|
+
*
|
|
3248
|
+
* Fire-and-log: the entry is visible synchronously by the store's contract, and
|
|
3249
|
+
* the promise tracks durability only. A persist failure costs restart survival,
|
|
3250
|
+
* never this session's dial set — the same trade
|
|
3251
|
+
* `SeedBootstrapService.anchorAcceptedSigner` makes — so it is logged rather than
|
|
3252
|
+
* failing a seed or an add that has already been accepted.
|
|
3253
|
+
*
|
|
3254
|
+
* Two peers are never retained. One with no address: there is nothing to dial.
|
|
3255
|
+
* And self: `createSeed` projects EVERY `CadrePeer` row, so an owner that applies
|
|
3256
|
+
* a seed minted after it joined finds itself in the owner list, and `addDrone`
|
|
3257
|
+
* could be handed this node's own id. Retaining self would make the cold-start
|
|
3258
|
+
* pass dial this node forever (the steady-state pass filters self out of its
|
|
3259
|
+
* sibling list for the same reason).
|
|
3260
|
+
*/
|
|
3261
|
+
private retainDialTarget(peerId: string, addrs: readonly string[], caller: string): void {
|
|
2459
3262
|
const store = this.bootstrapPeerStore;
|
|
2460
3263
|
if (!store) {
|
|
2461
3264
|
// Unreachable in production: start() builds the store before any network
|
|
2462
|
-
// bring-up, and
|
|
2463
|
-
// SeedBootstrapService rejects
|
|
2464
|
-
|
|
3265
|
+
// bring-up, and every caller needs a started node (an uninitialized
|
|
3266
|
+
// SeedBootstrapService rejects a seed or an add, and the reconcile pass
|
|
3267
|
+
// needs a running one).
|
|
3268
|
+
log('%s: no bootstrap-peer store yet; not retaining %s', caller, peerId);
|
|
2465
3269
|
return;
|
|
2466
3270
|
}
|
|
2467
|
-
|
|
2468
|
-
|
|
2469
|
-
if (!peer.isOwner || peer.multiaddrs.length === 0 || peer.peerId === selfPeerId) {
|
|
2470
|
-
continue;
|
|
2471
|
-
}
|
|
2472
|
-
// Sync-visible by contract; the promise tracks durability only. A persist
|
|
2473
|
-
// failure costs restart survival, never this session's retry set — the same
|
|
2474
|
-
// trade `SeedBootstrapService.anchorAcceptedSigner` makes — so log and carry
|
|
2475
|
-
// on rather than failing the seed that was already accepted.
|
|
2476
|
-
void store.record(peer.peerId, peer.multiaddrs).catch((error: unknown) => {
|
|
2477
|
-
log('recordSeedBootstrapPeers: persisting bootstrap peer %s failed (retained in memory): %o',
|
|
2478
|
-
peer.peerId, error);
|
|
2479
|
-
});
|
|
3271
|
+
if (addrs.length === 0 || peerId === this.controlNode?.peerId.toString()) {
|
|
3272
|
+
return;
|
|
2480
3273
|
}
|
|
3274
|
+
void store.record(peerId, addrs).catch((error: unknown) => {
|
|
3275
|
+
log('%s: persisting dial target %s failed (retained in memory): %o', caller, peerId, error);
|
|
3276
|
+
});
|
|
2481
3277
|
}
|
|
2482
3278
|
|
|
2483
3279
|
/**
|
|
2484
|
-
* Cold-start branch of the reconcile pass: re-dial
|
|
2485
|
-
*
|
|
3280
|
+
* Cold-start branch of the reconcile pass: re-dial every retained out-of-band
|
|
3281
|
+
* dial target (see {@link bootstrapPeerStore}) — in practice the owner peers of
|
|
3282
|
+
* the seeds this node applied — while the control database still has no
|
|
3283
|
+
* siblings to dial.
|
|
2486
3284
|
*
|
|
2487
3285
|
* `SeedBootstrapService.applySeed` dials those owners exactly ONCE, best-effort.
|
|
2488
3286
|
* When that single dial fails — owner momentarily down, relay reservation not
|
|
@@ -2503,20 +3301,20 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2503
3301
|
* Best-effort per peer, exactly like {@link dialControlSibling}: one dead
|
|
2504
3302
|
* address never aborts the pass.
|
|
2505
3303
|
*/
|
|
2506
|
-
private async dialColdStartBootstrap(): Promise<
|
|
3304
|
+
private async dialColdStartBootstrap(): Promise<string[]> {
|
|
2507
3305
|
const controlNode = this.controlNode;
|
|
2508
3306
|
// Snapshot up front: the store hands back a copy, so a seed applied mid-pass
|
|
2509
3307
|
// cannot mutate what we are iterating.
|
|
2510
3308
|
const targets = this.bootstrapPeerStore?.all();
|
|
2511
3309
|
if (!controlNode || !targets || targets.size === 0) {
|
|
2512
|
-
return;
|
|
3310
|
+
return [];
|
|
2513
3311
|
}
|
|
2514
3312
|
// Same skip rule as step 3 of the steady-state pass: no churn on live links.
|
|
2515
3313
|
const connected = new Set(controlNode.getConnections().map((c) => c.remotePeer.toString()));
|
|
2516
|
-
|
|
3314
|
+
const dialed: string[] = [];
|
|
2517
3315
|
for (const [peerId, entry] of targets) {
|
|
2518
3316
|
if (!this._running || !this.controlNode) {
|
|
2519
|
-
return;
|
|
3317
|
+
return dialed;
|
|
2520
3318
|
}
|
|
2521
3319
|
// NOTE: a connection this node still holds in `status: 'open'` after the remote
|
|
2522
3320
|
// has already aborted it counts as connected here, so the cold-start retry stays
|
|
@@ -2529,21 +3327,23 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2529
3327
|
continue;
|
|
2530
3328
|
}
|
|
2531
3329
|
if (await this.dialBootstrapPeer(peerId, entry.addrs)) {
|
|
2532
|
-
dialed
|
|
3330
|
+
dialed.push(peerId);
|
|
2533
3331
|
}
|
|
2534
3332
|
}
|
|
2535
3333
|
log('reconcileControlCohort: cold-start pass complete (bootstrap=%d, connected=%d, dialed=%d)',
|
|
2536
|
-
targets.size, connected.size, dialed);
|
|
3334
|
+
targets.size, connected.size, dialed.length);
|
|
3335
|
+
return dialed;
|
|
2537
3336
|
}
|
|
2538
3337
|
|
|
2539
3338
|
/**
|
|
2540
|
-
* Bind a
|
|
2541
|
-
* authenticates the peer it is aiming at rather than
|
|
3339
|
+
* Bind a retained dial target's addresses (see {@link bootstrapPeerStore}) to
|
|
3340
|
+
* its peer id, so the dial authenticates the peer it is aiming at rather than
|
|
3341
|
+
* trusting whoever answers.
|
|
2542
3342
|
*
|
|
2543
3343
|
* The same {@link normalizeDialAddrs} rule the resolved and address-book paths
|
|
2544
3344
|
* use, on the third and last source of control-dial candidates: an address that
|
|
2545
3345
|
* already terminates in `/p2p/<id>` must carry THIS peer's id or it is dropped
|
|
2546
|
-
* (
|
|
3346
|
+
* (an entry that disagrees with itself is not a dial target); one that does not —
|
|
2547
3347
|
* a bare listen addr, or a relay hop with the destination missing — gets the id
|
|
2548
3348
|
* encapsulated; an unencapsulatable one is dropped rather than dialed bare.
|
|
2549
3349
|
*/
|
|
@@ -2564,10 +3364,11 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2564
3364
|
}
|
|
2565
3365
|
try {
|
|
2566
3366
|
log('reconcileControlCohort(cold-start): dialing bootstrap peer %s (%d addr(s))', peerId, parsed.length);
|
|
2567
|
-
await
|
|
2568
|
-
|
|
2569
|
-
|
|
2570
|
-
(
|
|
3367
|
+
await dialPeerAddrs(
|
|
3368
|
+
controlNode,
|
|
3369
|
+
parsed,
|
|
3370
|
+
this.controlDialBudget(),
|
|
3371
|
+
`reconcileControlCohort cold-start dial of bootstrap peer ${peerId}`
|
|
2571
3372
|
);
|
|
2572
3373
|
return true;
|
|
2573
3374
|
} catch (error) {
|
|
@@ -3276,6 +4077,13 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3276
4077
|
// (register a config + addStrand); the strand-agnostic seam keeps this
|
|
3277
4078
|
// class free of any app's join policy. A self-configured strand (config
|
|
3278
4079
|
// already present) keeps auto-starting below, unchanged.
|
|
4080
|
+
//
|
|
4081
|
+
// Recorded BEFORE the emit so a handler that synchronously drains
|
|
4082
|
+
// `getDiscoveredStrands()` sees this strand too. The watcher never offers
|
|
4083
|
+
// the same strand twice (its `knownStrands` retains the id for the life of
|
|
4084
|
+
// the process), so this map — not the event — is what a late subscriber
|
|
4085
|
+
// reads. See the `strand:discovered` doc in types.ts.
|
|
4086
|
+
this.discoveredStrands.set(strand.Id, strand);
|
|
3279
4087
|
this.emit('strand:discovered', { strandId: strand.Id, strand });
|
|
3280
4088
|
return;
|
|
3281
4089
|
}
|
|
@@ -3396,8 +4204,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3396
4204
|
// Stop all strand instances
|
|
3397
4205
|
await this.strandManager.stopAll();
|
|
3398
4206
|
|
|
3399
|
-
// Clear sApp configs
|
|
4207
|
+
// Clear sApp configs, recorded launch refusals and the unclaimed-strand backlog
|
|
3400
4208
|
this.sAppConfigs.clear();
|
|
4209
|
+
this.strandLaunchRefusals.clear();
|
|
4210
|
+
this.discoveredStrands.clear();
|
|
3401
4211
|
|
|
3402
4212
|
// Drop delegate-admission state: the grants are scoped to the session that
|
|
3403
4213
|
// recorded them, so a stop()/start() cycle on this object must not keep
|
|
@@ -3406,12 +4216,22 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3406
4216
|
this.delegateAdmission.clear();
|
|
3407
4217
|
this.delegateAnnounceAt.clear();
|
|
3408
4218
|
|
|
4219
|
+
// Control backfill first — before the database closes and the control node
|
|
4220
|
+
// stops — so no NEW catch-up push is issued against a torn-down transport.
|
|
4221
|
+
// A push already in flight is not awaited; it fails into the module's own
|
|
4222
|
+
// per-chunk catch. (Mirrors releaseRuntime in strand-instance-manager.ts.)
|
|
4223
|
+
if (this.controlBackfill) {
|
|
4224
|
+
this.controlBackfill.stop();
|
|
4225
|
+
this.controlBackfill = null;
|
|
4226
|
+
}
|
|
4227
|
+
|
|
3409
4228
|
// Close control database (this also shuts down the collection factory).
|
|
3410
4229
|
// Detach the membership hub first: nothing this teardown does should drive a
|
|
3411
4230
|
// gate refresh, and a notification arriving after this point is a no-op.
|
|
3412
4231
|
if (this.controlDatabase) {
|
|
3413
4232
|
this.controlDatabase.setMembershipChangeListener(null);
|
|
3414
4233
|
this.controlDatabase.setGuardedDeleteListener(null);
|
|
4234
|
+
this.controlDatabase.setControlWriteAbandonedListener(null);
|
|
3415
4235
|
await this.controlDatabase.close();
|
|
3416
4236
|
this.controlDatabase = null;
|
|
3417
4237
|
}
|
|
@@ -3444,6 +4264,22 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3444
4264
|
this._running = false;
|
|
3445
4265
|
}
|
|
3446
4266
|
|
|
4267
|
+
/**
|
|
4268
|
+
* A gated joiner's database was just published (`StartStrandConfig.onWritable`): the
|
|
4269
|
+
* instance is `'active'` now, so re-arm its idle timer as any activity would. Nothing
|
|
4270
|
+
* records activity on a gated joiner, so its idle timer may already have fired — the
|
|
4271
|
+
* instance read `'idle'` with a hibernate timer pending — and without this the flip
|
|
4272
|
+
* back to `'active'` would leave that timer to quiesce a strand the app just started
|
|
4273
|
+
* using. Emits `strand:writable` after the timers are settled.
|
|
4274
|
+
*/
|
|
4275
|
+
private handleStrandWritable(strandId: string): void {
|
|
4276
|
+
const instance = this.strandManager.getInstance(strandId);
|
|
4277
|
+
if (instance) {
|
|
4278
|
+
this.hibernationManager.recordActivity(instance);
|
|
4279
|
+
}
|
|
4280
|
+
this.emit('strand:writable', { strandId });
|
|
4281
|
+
}
|
|
4282
|
+
|
|
3447
4283
|
// Hibernation callbacks
|
|
3448
4284
|
private async handleStrandIdle(strandId: string): Promise<void> {
|
|
3449
4285
|
const instance = this.strandManager.getInstance(strandId);
|
|
@@ -3498,9 +4334,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3498
4334
|
return;
|
|
3499
4335
|
}
|
|
3500
4336
|
|
|
3501
|
-
// Still live (idle wake, or defensive double-wake): no rebuild needed.
|
|
4337
|
+
// Still live (idle wake, or defensive double-wake): no rebuild needed. A joiner
|
|
4338
|
+
// still behind its first-sync gate wakes back to `'syncing'`, not `'active'`.
|
|
3502
4339
|
if (instance.libp2pNode || instance.database) {
|
|
3503
|
-
instance.status =
|
|
4340
|
+
instance.status = liveStrandStatus(instance);
|
|
3504
4341
|
instance.lastActivity = new Date();
|
|
3505
4342
|
log('Strand %s woke (already live)', strandId);
|
|
3506
4343
|
this.emit('strand:waking', { strandId });
|
|
@@ -3519,8 +4356,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3519
4356
|
/**
|
|
3520
4357
|
* Rebuild a quiesced strand's runtime, re-resolving the volatile cohort input
|
|
3521
4358
|
* first: the discovery seed may have grown since the strand last ran. Shared by
|
|
3522
|
-
* the wake (`handleStrandWake`) and check-in (`handleStrandCheckIn`) paths so
|
|
3523
|
-
*
|
|
4359
|
+
* the wake (`handleStrandWake`) and check-in (`handleStrandCheckIn`) paths so both
|
|
4360
|
+
* apply the same fresh resolution. `resumeStrand` is idempotent
|
|
3524
4361
|
* (returns the live instance unchanged) as a backstop against double-resume.
|
|
3525
4362
|
*/
|
|
3526
4363
|
private async resumeStrandRuntime(strandId: string): Promise<void> {
|
|
@@ -3531,6 +4368,9 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3531
4368
|
? peerIdFromPrivateKey(await strandTransportKey(this.identityKey, strandId)).toString()
|
|
3532
4369
|
: undefined;
|
|
3533
4370
|
const bootstrapNodes = await this.resolveCohortSeed(strandId, delegatePeerId);
|
|
4371
|
+
// NOTE: no `servingMachines` override — see `launchStrand` for why a strand node
|
|
4372
|
+
// declares no repair yardstick, and `StartStrandConfig.servingMachines` for the
|
|
4373
|
+
// count that would legitimately go here once one exists.
|
|
3534
4374
|
const instance = await this.strandManager.resumeStrand(strandId, { bootstrapNodes });
|
|
3535
4375
|
// Same reason as the launch path: `bootstrapNodes` only reaches the address
|
|
3536
4376
|
// book through @libp2p/bootstrap discovery, so merge it directly as well.
|
|
@@ -3622,7 +4462,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3622
4462
|
|
|
3623
4463
|
const sawActivity = instance.lastActivity !== activityMark;
|
|
3624
4464
|
if (sawActivity) {
|
|
3625
|
-
instance.status =
|
|
4465
|
+
instance.status = liveStrandStatus(instance);
|
|
3626
4466
|
this.emit('strand:waking', { strandId });
|
|
3627
4467
|
log('Wake window: strand %s saw activity during the window; staying active', strandId);
|
|
3628
4468
|
return true;
|
|
@@ -3672,29 +4512,90 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3672
4512
|
* Add a strand with its sApp configuration.
|
|
3673
4513
|
* The hosting application must provide the sApp schema when creating a strand.
|
|
3674
4514
|
*
|
|
4515
|
+
* This is the ATTACH half only — it starts the local instance and never publishes the
|
|
4516
|
+
* `Strand` row. A joiner (the row arrived over the control network) wants exactly this; a
|
|
4517
|
+
* FOUNDER wants {@link foundStrand}, which publishes and attaches in one resumable call.
|
|
4518
|
+
* With no explicit `founder` flag, founder-ness is derived from the row's
|
|
4519
|
+
* `FounderOwnerKey` (see {@link StrandConfig.founder}) — so attaching a row THIS machine
|
|
4520
|
+
* published (e.g. re-attaching its own orphan after a restart) founds it, and attaching
|
|
4521
|
+
* anyone else's row joins, without the caller needing to know which it is.
|
|
4522
|
+
*
|
|
3675
4523
|
* A rejected call leaves nothing running but DOES leave the sApp config
|
|
3676
4524
|
* registered, deliberately: both an explicit retry and the {@link StrandWatcher}'s
|
|
3677
|
-
* automatic relaunch need it.
|
|
3678
|
-
*
|
|
3679
|
-
*
|
|
3680
|
-
*
|
|
3681
|
-
*
|
|
4525
|
+
* automatic relaunch need it. That background relaunch covers only a strand the
|
|
4526
|
+
* WATCHER launched: it forgets a row whose `onStrandAdded` threw and re-attempts it
|
|
4527
|
+
* on a later poll (each failure re-emitting `strand:error`) until it succeeds. A
|
|
4528
|
+
* strand claimed here after it was DISCOVERED gets no such retry — the watcher
|
|
4529
|
+
* already recorded it in its `knownStrands` when it offered it as
|
|
4530
|
+
* `strand:discovered`, and no later poll re-offers a strand it knows. This call has
|
|
4531
|
+
* also already dropped it from {@link getDiscoveredStrands}, so a failed claim
|
|
4532
|
+
* leaves the strand dormant until the caller retries or the node restarts (see
|
|
4533
|
+
* `backlog/bug-discovered-strand-lost-when-claim-fails`).
|
|
4534
|
+
* {@link detachStrand} (reached via {@link stopStrand}) is what abandons a strand
|
|
4535
|
+
* for good.
|
|
3682
4536
|
*/
|
|
3683
4537
|
async addStrand(config: StrandConfig): Promise<StrandInstance> {
|
|
3684
4538
|
if (!this._running) {
|
|
3685
4539
|
throw new Error('CadreNode not running');
|
|
3686
4540
|
}
|
|
3687
4541
|
|
|
3688
|
-
const { strandRow, sAppConfig, founder } = config;
|
|
4542
|
+
const { strandRow, sAppConfig, founder, partyMemberPrivateKey } = config;
|
|
4543
|
+
if (partyMemberPrivateKey !== undefined && strandRow.Type !== 'c') {
|
|
4544
|
+
// Only a closed strand seats a Member/Manager, so an open launch would drop this
|
|
4545
|
+
// key silently. Refuse instead: the caller has confused the party identity key
|
|
4546
|
+
// with something an open strand uses, and a silent drop hides that.
|
|
4547
|
+
throw new Error(
|
|
4548
|
+
`addStrand(${strandRow.Id}): partyMemberPrivateKey belongs to a CLOSED strand ` +
|
|
4549
|
+
`(Type 'c'), but this row is Type '${strandRow.Type}' — an open strand seats no ` +
|
|
4550
|
+
'Member/Manager, so the key would be ignored.'
|
|
4551
|
+
);
|
|
4552
|
+
}
|
|
3689
4553
|
|
|
3690
|
-
// Store sApp config for this strand
|
|
4554
|
+
// Store sApp config for this strand. The strand is claimed now, so it leaves the
|
|
4555
|
+
// unclaimed backlog — a later `getDiscoveredStrands()` drain must not re-offer it.
|
|
3691
4556
|
this.sAppConfigs.set(strandRow.Id, sAppConfig);
|
|
4557
|
+
this.discoveredStrands.delete(strandRow.Id);
|
|
3692
4558
|
log('Registered sAppConfig for strand %s (sApp: %s, founder: %s)',
|
|
3693
|
-
strandRow.Id, sAppConfig.id, founder ??
|
|
4559
|
+
strandRow.Id, sAppConfig.id, founder ?? 'derived');
|
|
4560
|
+
|
|
4561
|
+
// An unset `founder` is DERIVED from the row inside launchStrand (this node founds
|
|
4562
|
+
// iff the row's FounderOwnerKey is its own owner key); an explicit flag wins — the
|
|
4563
|
+
// formation/responder flows pass one deliberately, since their consent-seated rows
|
|
4564
|
+
// carry a null column. See StrandConfig.founder. Same rule for the explicit
|
|
4565
|
+
// partyMemberPrivateKey: it wins over the StrandPartyKey control-row read.
|
|
4566
|
+
const instance = await this.launchStrand(strandRow, sAppConfig, founder, partyMemberPrivateKey);
|
|
4567
|
+
|
|
4568
|
+
// The first-sync write gate: a JOINING machine's database is withheld until it has
|
|
4569
|
+
// received the strand's Header from another member, because a write before that
|
|
4570
|
+
// forks every table it touches (see strand-first-sync-gate.ts). Waiting here is what
|
|
4571
|
+
// lets an app write straight after `addStrand` resolves — the reference chat apps
|
|
4572
|
+
// do — so the default is to wait, bounded by `strandFirstSync.timeoutMs`. A timeout
|
|
4573
|
+
// rejects with the retryable `StrandAwaitingFirstSyncError` and leaves the launch
|
|
4574
|
+
// up: this same call, made again once a member is reachable, completes the attach
|
|
4575
|
+
// (the tracked-instance path above returns the still-syncing instance, and this
|
|
4576
|
+
// wait picks it up). Founders and machines that already hold the Header resolve at
|
|
4577
|
+
// once. Only a LIVE gated instance is waited on — a hibernating one is returned as
|
|
4578
|
+
// it always was (no database either; a wake publishes it).
|
|
4579
|
+
if (config.awaitFirstSync !== false && this.strandManager.isAwaitingFirstSync(strandRow.Id)) {
|
|
4580
|
+
await this.strandManager.whenWritable(strandRow.Id, { timeoutMs: this.config.strandFirstSync?.timeoutMs });
|
|
4581
|
+
}
|
|
4582
|
+
return instance;
|
|
4583
|
+
}
|
|
3694
4584
|
|
|
3695
|
-
|
|
3696
|
-
|
|
3697
|
-
|
|
4585
|
+
/**
|
|
4586
|
+
* Resolve once a launched strand is writable — its database published to the app and
|
|
4587
|
+
* its status `'active'` — or reject with the retryable `StrandAwaitingFirstSyncError`
|
|
4588
|
+
* after `timeoutMs` (default `strandFirstSync.timeoutMs`, else
|
|
4589
|
+
* `DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS`). The promise form of the `strand:writable`
|
|
4590
|
+
* event, for a caller that attached without waiting ({@link StrandConfig.awaitFirstSync}
|
|
4591
|
+
* `false`) or that holds a strand the watcher launched. Resolves immediately for a
|
|
4592
|
+
* strand that is already writable; rejects immediately for one this node does not run.
|
|
4593
|
+
* A hibernating strand is not woken — wake it first if that is what is wanted.
|
|
4594
|
+
*/
|
|
4595
|
+
async whenStrandWritable(strandId: string, options?: { timeoutMs?: number }): Promise<StrandInstance> {
|
|
4596
|
+
return await this.strandManager.whenWritable(strandId, {
|
|
4597
|
+
timeoutMs: options?.timeoutMs ?? this.config.strandFirstSync?.timeoutMs
|
|
4598
|
+
});
|
|
3698
4599
|
}
|
|
3699
4600
|
|
|
3700
4601
|
/**
|
|
@@ -3702,40 +4603,231 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3702
4603
|
* owner identity, so other cadre members discover it via control-network
|
|
3703
4604
|
* sync (their {@link StrandWatcher} fires `strand:discovered`).
|
|
3704
4605
|
*
|
|
3705
|
-
*
|
|
3706
|
-
*
|
|
3707
|
-
*
|
|
3708
|
-
*
|
|
3709
|
-
* `addStrand` (the row already exists).
|
|
4606
|
+
* **Founding a strand is TWO steps** — this one (publish the row cadre-wide) and
|
|
4607
|
+
* {@link addStrand} (start the local instance, `founder: true` to write the strand's
|
|
4608
|
+
* `Header`/founding membership). `addStrand` deliberately omits the insert: it only
|
|
4609
|
+
* starts the LOCAL instance, whereas publishing makes the strand visible cadre-wide. A
|
|
4610
|
+
* discovering peer only does `addStrand` (the row already exists). Callers founding a
|
|
4611
|
+
* strand should use {@link foundStrand}, which performs both steps and is safe to
|
|
4612
|
+
* re-run — hand-rolling the pair is what left strands half-founded when an app was
|
|
4613
|
+
* killed between them.
|
|
4614
|
+
*
|
|
4615
|
+
* **Idempotent for identical content.** A live row whose `(Type, MemberPrivateKey)` match
|
|
4616
|
+
* the arguments is the state this call would have produced, so the call logs and returns
|
|
4617
|
+
* that row instead of writing — a publish interrupted after it committed can be repeated.
|
|
4618
|
+
* DIFFERENT content on the same id throws, naming the columns that differ, rather than
|
|
4619
|
+
* surfacing the raw `UNIQUE constraint failed: Strand.Id`: silently accepting it would
|
|
4620
|
+
* let a retry reopen a closed strand or swap the key gating its reads. The read and the
|
|
4621
|
+
* insert are not atomic, so a concurrent founder (two machines of one party, same id) is
|
|
4622
|
+
* caught on the insert's uniqueness rejection and resolved the same way — re-read, no-op
|
|
4623
|
+
* on a match, rethrow otherwise. Never an overwrite either way.
|
|
4624
|
+
*
|
|
4625
|
+
* A tombstoned strand is NOT resurrected here: {@link unpublishStrand} deletes the row, so
|
|
4626
|
+
* the idempotent branch is unreachable and the ordinary publish path re-seats it, exactly
|
|
4627
|
+
* as that method documents.
|
|
4628
|
+
*
|
|
4629
|
+
* **Already stuck on `UNIQUE constraint failed: Strand.Id`?** The row is already published
|
|
4630
|
+
* — attach, do NOT republish: `addStrand` (or {@link foundStrand}) with the id and, for a
|
|
4631
|
+
* closed strand, the `MemberPrivateKey` read back from the row. `unpublishStrand` + a
|
|
4632
|
+
* fresh publish also clears it for an OPEN strand, but is destructive for a closed one
|
|
4633
|
+
* (the key exists nowhere else). No app-data wipe is needed for either.
|
|
3710
4634
|
*
|
|
3711
4635
|
* The insert is signed with the ed25519 key behind this node's PeerId — which
|
|
3712
4636
|
* {@link ed25519KeyPairFromLibp2p} also exposes as the node's owner keypair,
|
|
3713
|
-
* so peer identity and owner key are one and the same. That key
|
|
3714
|
-
*
|
|
4637
|
+
* so peer identity and owner key are one and the same. That key is also persisted on the
|
|
4638
|
+
* row as `FounderOwnerKey` (the schema pins the column to the verified signer), which is
|
|
4639
|
+
* what later lets any launch derive "this machine is the founder" from the row alone.
|
|
4640
|
+
* The key must be enrolled in `OwnerKey` (e.g. via {@link ControlDatabase.ensureOwnerKey}
|
|
3715
4641
|
* at genesis) or the schema's `Strand.AuthorizedInsert` constraint rejects the write.
|
|
3716
4642
|
* Failing loudly here is intentional: a silently-unpublished strand would run
|
|
3717
4643
|
* as a local-only island that no peer could ever discover or join.
|
|
3718
4644
|
*
|
|
4645
|
+
* A CLOSED strand's publish also seats this party's own membership identity key —
|
|
4646
|
+
* the `StrandPartyKey` row {@link ensureStrandPartyKey} documents — which is what the
|
|
4647
|
+
* founder bootstrap derives `Member.Key`/`Manager.MemberKey` from. The
|
|
4648
|
+
* `memberPrivateKey` ARGUMENT stays the strand-wide read secret formation hands to
|
|
4649
|
+
* joiners; it derives nobody's identity.
|
|
4650
|
+
*
|
|
3719
4651
|
* @param strandId - Unique strand identifier (typically the same id passed to
|
|
3720
4652
|
* {@link addStrand}).
|
|
3721
4653
|
* @param type - `'o'` for open (default) or `'c'` for closed.
|
|
3722
|
-
* @param memberPrivateKey - Optional membership key for a closed strand.
|
|
3723
|
-
* @
|
|
3724
|
-
*
|
|
3725
|
-
|
|
3726
|
-
|
|
4654
|
+
* @param memberPrivateKey - Optional shared membership (read) key for a closed strand.
|
|
4655
|
+
* @returns The live `Strand` row — the one just inserted, or the matching one already
|
|
4656
|
+
* there. A closed strand's caller should carry THIS row's `MemberPrivateKey` forward:
|
|
4657
|
+
* on a repeat it is the stored key, not the argument.
|
|
4658
|
+
* @throws if the node is not started, exposes no owner signing key, the id is blank, a
|
|
4659
|
+
* row with the same id holds different content, or the control DB rejects the
|
|
4660
|
+
* (unauthorized) insert.
|
|
4661
|
+
*/
|
|
4662
|
+
async publishStrand(strandId: string, type: 'o' | 'c' = 'o', memberPrivateKey?: string): Promise<StrandRow> {
|
|
3727
4663
|
const signingKey = this.requireOwnerSigningKey(`publish strand ${strandId}`);
|
|
3728
4664
|
// Trim/reject here so the id that lands matches the one unpublishStrand looks up: it
|
|
3729
4665
|
// trims too, and an untrimmed row would be unreachable by the same string.
|
|
3730
4666
|
const trimmed = requireNonBlank(strandId, 'strand id');
|
|
3731
|
-
|
|
3732
|
-
|
|
3733
|
-
|
|
3734
|
-
|
|
3735
|
-
|
|
3736
|
-
memberPrivateKey
|
|
3737
|
-
|
|
4667
|
+
// FounderOwnerKey records THIS machine as the row's publisher (the schema pins it to
|
|
4668
|
+
// the signing owner), which is what later lets a relaunch derive founder-ness.
|
|
4669
|
+
const desired: StrandRow = {
|
|
4670
|
+
Id: trimmed,
|
|
4671
|
+
Type: type,
|
|
4672
|
+
MemberPrivateKey: memberPrivateKey ?? null,
|
|
4673
|
+
FounderOwnerKey: signingKey.publicKeyB64,
|
|
4674
|
+
};
|
|
4675
|
+
|
|
4676
|
+
// Read first rather than leaning on the insert's collision for BOTH readings. The
|
|
4677
|
+
// ordinary resume (an app killed after its own publish committed) then needs no error
|
|
4678
|
+
// matching at all: the collision path can only recognise itself by the engine's message
|
|
4679
|
+
// TEXT (see isStrandIdConflict), and an upstream reword there must degrade the RARE race,
|
|
4680
|
+
// not the common resume. It also spends no stamp or signature on a known no-op.
|
|
4681
|
+
const existing = await this.controlDatabase!.queryStrand(trimmed);
|
|
4682
|
+
if (existing) {
|
|
4683
|
+
// NOTE: reads local converged state, and `queryStrand` does not filter rows whose
|
|
4684
|
+
// StampId a Revocation has retired. So on a sibling that still physically holds a row
|
|
4685
|
+
// deleted elsewhere while ALONE (the delete-while-alone gap in docs/architecture.md),
|
|
4686
|
+
// an owner-signed republish now no-ops onto that doomed row instead of re-seating the
|
|
4687
|
+
// id under a fresh stamp. Harmless while that gap is itself unresolved — both
|
|
4688
|
+
// behaviours end at the same unconverged state — but if delete-while-alone ever gains
|
|
4689
|
+
// real replay, filter retired stamps here (queryStrandStampId + the Revocation read)
|
|
4690
|
+
// so a re-seat is not mistaken for a repeat.
|
|
4691
|
+
const matched = requireMatchingStrandRow(existing, desired, 'is already published');
|
|
4692
|
+
log('publishStrand(%s): already published (type %s) with identical content — no-op', trimmed, type);
|
|
4693
|
+
// A closed strand THIS machine published must hold a party key (its own membership
|
|
4694
|
+
// identity — see ensureStrandPartyKey). Insert-if-absent, so the ordinary repeat
|
|
4695
|
+
// is a read-only no-op and a publish interrupted between the row insert and the
|
|
4696
|
+
// mint heals here. A row another machine published is that machine's to mint for.
|
|
4697
|
+
if (type === 'c' && this.isSelfFoundedRow(matched)) {
|
|
4698
|
+
await this.ensureStrandPartyKey(trimmed);
|
|
4699
|
+
}
|
|
4700
|
+
return matched;
|
|
4701
|
+
}
|
|
4702
|
+
|
|
4703
|
+
try {
|
|
4704
|
+
await this.controlDatabase!.insertStrand(
|
|
4705
|
+
trimmed,
|
|
4706
|
+
type,
|
|
4707
|
+
signingKey.publicKeyB64,
|
|
4708
|
+
signMessageWith(signingKey.privateKeyB64),
|
|
4709
|
+
memberPrivateKey
|
|
4710
|
+
);
|
|
4711
|
+
} catch (error) {
|
|
4712
|
+
// Only the uniqueness collision is a candidate for the idempotent reading; every
|
|
4713
|
+
// other rejection (unauthorized signer, retired stamp) must keep surfacing.
|
|
4714
|
+
if (!isStrandIdConflict(error)) {
|
|
4715
|
+
throw error;
|
|
4716
|
+
}
|
|
4717
|
+
const landed = await this.controlDatabase!.queryStrand(trimmed);
|
|
4718
|
+
if (!landed) {
|
|
4719
|
+
// Collided, yet no row is readable: not the race this branch handles (a row
|
|
4720
|
+
// deleted between the rejection and the re-read, or a stale local view). Surface
|
|
4721
|
+
// the original rather than guessing.
|
|
4722
|
+
throw error;
|
|
4723
|
+
}
|
|
4724
|
+
log('publishStrand(%s): lost a concurrent founding race; re-read the landed row', trimmed);
|
|
4725
|
+
return requireMatchingStrandRow(landed, desired, 'landed concurrently from another founder');
|
|
4726
|
+
}
|
|
3738
4727
|
log('Published strand %s (type %s) to control DB under owner %s', trimmed, type, signingKey.publicKeyB64);
|
|
4728
|
+
// A closed strand's publish also mints this party's own membership identity key
|
|
4729
|
+
// (StrandPartyKey) — the key the founder bootstrap derives Member/Manager from,
|
|
4730
|
+
// deliberately distinct from the shared memberPrivateKey argument above. A failure
|
|
4731
|
+
// here surfaces (the strand row has already committed, so a retried publish takes
|
|
4732
|
+
// the idempotent branch and heals the mint; so does the next founder launch).
|
|
4733
|
+
if (type === 'c') {
|
|
4734
|
+
await this.ensureStrandPartyKey(trimmed);
|
|
4735
|
+
}
|
|
4736
|
+
return desired;
|
|
4737
|
+
}
|
|
4738
|
+
|
|
4739
|
+
/**
|
|
4740
|
+
* Found a strand — publish its row cadre-wide AND start the local instance as its
|
|
4741
|
+
* founder — in one resumable call. The single entry point for creating a strand; a
|
|
4742
|
+
* caller joining one someone else founded uses {@link addStrand} alone.
|
|
4743
|
+
*
|
|
4744
|
+
* Safe to re-run from any point of interruption, which hand-rolling
|
|
4745
|
+
* {@link publishStrand} + {@link addStrand} is not. Either half can already have
|
|
4746
|
+
* happened:
|
|
4747
|
+
*
|
|
4748
|
+
* - **row already published** → the stored row is adopted rather than re-published, so a
|
|
4749
|
+
* run killed after the insert committed no longer dies on
|
|
4750
|
+
* `UNIQUE constraint failed: Strand.Id`. For a closed strand the row's STORED
|
|
4751
|
+
* `MemberPrivateKey` wins over `config.memberPrivateKey`: a caller that mints a key per
|
|
4752
|
+
* attempt (as the reference apps do) would otherwise present a key that does not match
|
|
4753
|
+
* the membership already seated in the strand.
|
|
4754
|
+
* - **instance already running** → the tracked instance is returned, and — because the
|
|
4755
|
+
* `Strand` row records its publishing machine (`FounderOwnerKey`) — a founder request
|
|
4756
|
+
* against an instance that was first launched as a joiner now runs the (idempotent,
|
|
4757
|
+
* insert-if-absent) founder bootstrap on it rather than silently skipping it
|
|
4758
|
+
* ({@link StrandInstanceManager.foundExistingStrand}). So a strand that something else
|
|
4759
|
+
* attached first — the reference RN app's `strand:discovered` handler after a restart,
|
|
4760
|
+
* or this node's own {@link StrandWatcher} poll winning `launchStrand`'s
|
|
4761
|
+
* `resolveCohortSeed` window — still ends up with its `Strand.Header` written.
|
|
4762
|
+
*
|
|
4763
|
+
* Founder-ness is resolved FROM THE ROW, not assumed: this machine founds iff the
|
|
4764
|
+
* resolved row's `FounderOwnerKey` is its own owner key. A fresh publish records this
|
|
4765
|
+
* machine, so the common path founds; a machine that lost a concurrent founding race to
|
|
4766
|
+
* a sibling adopts the sibling's row and ATTACHES instead — deliberately, since two
|
|
4767
|
+
* machines bootstrapping the same strand on separate replicas is the double-`Header`
|
|
4768
|
+
* hazard. The returned {@link FoundStrandResult.founded} says which happened.
|
|
4769
|
+
*
|
|
4770
|
+
* `Type` is compared, not adopted: a stored row of the other type means the caller and
|
|
4771
|
+
* the control plane disagree about what this strand IS, so it throws.
|
|
4772
|
+
*
|
|
4773
|
+
* @returns The instance AND the row the strand actually runs under — read the membership
|
|
4774
|
+
* key from the returned row, not from a freshly minted one ({@link FoundStrandResult}).
|
|
4775
|
+
* @throws if the node is not started, the id is blank, a published row of the same id has
|
|
4776
|
+
* a different `Type`, or either half rejects.
|
|
4777
|
+
*/
|
|
4778
|
+
async foundStrand(config: FoundStrandConfig): Promise<FoundStrandResult> {
|
|
4779
|
+
const { strandId, type = 'o', memberPrivateKey, sAppConfig, awaitFirstSync } = config;
|
|
4780
|
+
if (!this._running || !this.controlDatabase) {
|
|
4781
|
+
throw new Error(`CadreNode must be started before attempting to found strand ${strandId}`);
|
|
4782
|
+
}
|
|
4783
|
+
const controlDatabase = this.controlDatabase;
|
|
4784
|
+
const trimmed = requireNonBlank(strandId, 'strand id');
|
|
4785
|
+
const timed = <T>(step: string, op: () => Promise<T>) => timedStep('foundStrand', trimmed, step, op);
|
|
4786
|
+
const tTotal = performance.now();
|
|
4787
|
+
const published = await timed('queryStrand', () => controlDatabase.queryStrand(trimmed));
|
|
4788
|
+
const strandRow = published
|
|
4789
|
+
? this.adoptPublishedStrand(published, type, memberPrivateKey)
|
|
4790
|
+
: await timed('publishStrand', () => this.publishStrand(trimmed, type, memberPrivateKey));
|
|
4791
|
+
// Derived, not hardcoded `true`: adopting a row another machine published means that
|
|
4792
|
+
// machine runs the bootstrap — this one must attach or it would write a second Header.
|
|
4793
|
+
const founded = this.isSelfFoundedRow(strandRow);
|
|
4794
|
+
if (!founded) {
|
|
4795
|
+
log('foundStrand(%s): the stored row was published by a different machine ' +
|
|
4796
|
+
'(FounderOwnerKey is not this node\'s owner key) — attaching as a joiner; ' +
|
|
4797
|
+
'the founder bootstrap runs on the publishing machine', trimmed);
|
|
4798
|
+
}
|
|
4799
|
+
const instance = await timed('addStrand',
|
|
4800
|
+
() => this.addStrand({ strandRow, sAppConfig, founder: founded, awaitFirstSync }));
|
|
4801
|
+
timing(`[foundStrand:${trimmed}] total: ${Math.round(performance.now() - tTotal)}ms`);
|
|
4802
|
+
return { instance, strandRow, founded };
|
|
4803
|
+
}
|
|
4804
|
+
|
|
4805
|
+
/**
|
|
4806
|
+
* Resume onto an already-published `Strand` row: keep its stored content, reject a type
|
|
4807
|
+
* disagreement. The stored `MemberPrivateKey` is authoritative — see {@link foundStrand}
|
|
4808
|
+
* for why a caller-minted key must lose here (and why that is NOT the same rule as
|
|
4809
|
+
* {@link publishStrand}'s, which throws on a key mismatch because it is being asked to
|
|
4810
|
+
* WRITE that key, not to resume onto what already exists).
|
|
4811
|
+
*/
|
|
4812
|
+
private adoptPublishedStrand(
|
|
4813
|
+
published: StrandRow,
|
|
4814
|
+
type: 'o' | 'c',
|
|
4815
|
+
memberPrivateKey?: string
|
|
4816
|
+
): StrandRow {
|
|
4817
|
+
if (published.Type !== type) {
|
|
4818
|
+
throw new Error(
|
|
4819
|
+
`Cannot found strand ${published.Id} as type '${type}': it is already published as ` +
|
|
4820
|
+
`type '${published.Type}'. Reconcile deliberately rather than re-founding — a ` +
|
|
4821
|
+
'published open strand cannot be closed, nor a closed one opened, by re-publishing.'
|
|
4822
|
+
);
|
|
4823
|
+
}
|
|
4824
|
+
if (memberPrivateKey !== undefined && memberPrivateKey !== published.MemberPrivateKey) {
|
|
4825
|
+
log('foundStrand(%s): already published — resuming on the STORED MemberPrivateKey; ' +
|
|
4826
|
+
'the supplied key is discarded (read it back from the returned instance)', published.Id);
|
|
4827
|
+
} else {
|
|
4828
|
+
log('foundStrand(%s): already published (type %s) — adopting the stored row', published.Id, type);
|
|
4829
|
+
}
|
|
4830
|
+
return published;
|
|
3739
4831
|
}
|
|
3740
4832
|
|
|
3741
4833
|
/**
|
|
@@ -3751,10 +4843,12 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3751
4843
|
* is an owner-signed control-plane write like every other, not "destroy the network".
|
|
3752
4844
|
*
|
|
3753
4845
|
* Irreversible for a closed strand (`Type='c'`): the row carries `MemberPrivateKey`,
|
|
3754
|
-
* this party's
|
|
3755
|
-
*
|
|
3756
|
-
*
|
|
3757
|
-
*
|
|
4846
|
+
* this party's read secret for that network, and the strand's `StrandPartyKey` row —
|
|
4847
|
+
* this party's own membership identity, removed (and tombstoned) in the SAME
|
|
4848
|
+
* transaction — is stored nowhere else either. With both gone the party can never
|
|
4849
|
+
* again sign as its seated member/manager there, and a re-published row mints a fresh
|
|
4850
|
+
* identity that does not match the membership already written into the strand's RBAC
|
|
4851
|
+
* layer. The strand id itself is NOT
|
|
3758
4852
|
* blacklisted: a fresh owner-signed {@link publishStrand} re-seats it under a new
|
|
3759
4853
|
* stamp — only the unsigned consent re-seat is permanently foreclosed (the removal's
|
|
3760
4854
|
* `Revocation` tombstone names the id, which the consent branch of
|
|
@@ -3809,6 +4903,26 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3809
4903
|
'row deletion itself cannot be replayed, so other nodes may keep running the strand ' +
|
|
3810
4904
|
'until the collection converges.', trimmed);
|
|
3811
4905
|
}
|
|
4906
|
+
// A refused pre-split launch left no tracked instance, so the stop below never
|
|
4907
|
+
// reaches detachStrand for it — clear the refusal here so a re-founded id starts clean.
|
|
4908
|
+
this.strandLaunchRefusals.delete(trimmed);
|
|
4909
|
+
// Clear THIS machine's own MemberPeer binding while the strand runtime — and the
|
|
4910
|
+
// retained party key — is still live: the delete above already destroyed the
|
|
4911
|
+
// party's StrandPartyKey row, so once this process forgets the key nothing can
|
|
4912
|
+
// ever sign the removal again. Runs AFTER the control delete so a rejected
|
|
4913
|
+
// unpublish (unauthorized signer) never strips a binding the party still needs;
|
|
4914
|
+
// best-effort by contract (never throws) — a hibernating or unreachable strand
|
|
4915
|
+
// keeps the stale binding, which grants nothing today. Sibling machines observing
|
|
4916
|
+
// this removal via their watchers do NOT clear their own bindings (see the NOTE on
|
|
4917
|
+
// StrandInstanceManager.clearOwnMemberPeerBinding).
|
|
4918
|
+
//
|
|
4919
|
+
// NOTE: this makes unpublishStrand await a STRAND-NETWORK write it never used to —
|
|
4920
|
+
// however long that write takes to reject or commit is now added to unpublish's
|
|
4921
|
+
// wall-clock. Fine while strand writes fail fast on an unreachable quorum; if
|
|
4922
|
+
// unpublish ever starts hanging on an isolated strand, give this call its own
|
|
4923
|
+
// deadline rather than dropping it (a binding cleared late is still worth more than
|
|
4924
|
+
// one never cleared).
|
|
4925
|
+
await this.strandManager.clearOwnMemberPeerBinding(trimmed);
|
|
3812
4926
|
// Converge locally now rather than waiting up to a poll interval. The watcher fires
|
|
3813
4927
|
// onStrandRemoved for a strand it tracked; the explicit stop below covers a node whose
|
|
3814
4928
|
// strandFilter never admitted this strand (the watcher never knew it, so it will never
|
|
@@ -3821,6 +4935,105 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3821
4935
|
log('Unpublished strand %s from control DB under owner %s', trimmed, signingKey.publicKeyB64);
|
|
3822
4936
|
}
|
|
3823
4937
|
|
|
4938
|
+
/**
|
|
4939
|
+
* Seat this party's own strand membership identity key — the `StrandPartyKey` row —
|
|
4940
|
+
* for `strandId`, minting a fresh one when none is given, and adopting the stored one
|
|
4941
|
+
* when a row already exists. Insert-if-absent and stable thereafter: the founding
|
|
4942
|
+
* `Member.Key` must not change across restarts, or the bootstrap's insert-if-absent
|
|
4943
|
+
* guards stop matching.
|
|
4944
|
+
*
|
|
4945
|
+
* This is the identity half of the closed-strand key split: the row's `PrivateKey` —
|
|
4946
|
+
* NOT the strand row's shared `MemberPrivateKey`, which formation hands to every
|
|
4947
|
+
* joining party — is what the founder bootstrap derives `Member.Key`/`Manager.MemberKey`
|
|
4948
|
+
* from. It replicates to every machine this party owns (same plaintext-at-rest stance
|
|
4949
|
+
* as `MemberPrivateKey`; docs/strands.md → "Closed-Strand Member Key Handling") and is
|
|
4950
|
+
* never put on the formation wire.
|
|
4951
|
+
*
|
|
4952
|
+
* Called internally by {@link publishStrand} (closed strands mint at publish) and by a
|
|
4953
|
+
* founder launch that finds no row (a publish interrupted before its mint); public so a
|
|
4954
|
+
* test harness — or a joiner flow that persists a formation-issued identity — can seat
|
|
4955
|
+
* a specific key deliberately.
|
|
4956
|
+
*
|
|
4957
|
+
* @param strandId - The strand the key is this party's identity for.
|
|
4958
|
+
* @param partyMemberPrivateKey - Optional specific key (base64 protobuf, as
|
|
4959
|
+
* `generateStrandMemberKey` mints). When a row already holds a DIFFERENT key this
|
|
4960
|
+
* throws — a party has one identity per strand; rotation is a deliberate
|
|
4961
|
+
* remove-then-insert, not a silent swap.
|
|
4962
|
+
* @returns The live party key: the one just seated, or the stored one.
|
|
4963
|
+
*/
|
|
4964
|
+
async ensureStrandPartyKey(strandId: string, partyMemberPrivateKey?: string): Promise<string> {
|
|
4965
|
+
const trimmed = requireNonBlank(strandId, 'strand id');
|
|
4966
|
+
const signingKey = this.requireOwnerSigningKey(`seat a party key for strand ${trimmed}`);
|
|
4967
|
+
const existing = await this.controlDatabase!.queryStrandPartyKey(trimmed);
|
|
4968
|
+
if (existing !== null) {
|
|
4969
|
+
if (partyMemberPrivateKey !== undefined && partyMemberPrivateKey !== existing) {
|
|
4970
|
+
// Neither key is included: the party key is this party's membership identity secret.
|
|
4971
|
+
throw new Error(
|
|
4972
|
+
`Strand ${trimmed} already has a party key and the supplied one differs. The stored ` +
|
|
4973
|
+
'key is the identity the strand membership was seated under; to rotate it, remove ' +
|
|
4974
|
+
'the row deliberately (deleteStrandPartyKey) rather than overwriting it.'
|
|
4975
|
+
);
|
|
4976
|
+
}
|
|
4977
|
+
return existing;
|
|
4978
|
+
}
|
|
4979
|
+
const key = partyMemberPrivateKey ?? await generateStrandMemberKey();
|
|
4980
|
+
try {
|
|
4981
|
+
await this.controlDatabase!.insertStrandPartyKey(
|
|
4982
|
+
trimmed, key, signingKey.publicKeyB64, signMessageWith(signingKey.privateKeyB64));
|
|
4983
|
+
} catch (error) {
|
|
4984
|
+
// The read and the insert are not atomic; a concurrent seat of the same strand's
|
|
4985
|
+
// key can land in between. Adopt the landed row when it satisfies this request —
|
|
4986
|
+
// the same resolution publishStrand applies to its own read-then-insert window —
|
|
4987
|
+
// and rethrow every other rejection (unauthorized signer, retired stamp).
|
|
4988
|
+
const landed = await this.controlDatabase!.queryStrandPartyKey(trimmed);
|
|
4989
|
+
if (landed === null || (partyMemberPrivateKey !== undefined && partyMemberPrivateKey !== landed)) {
|
|
4990
|
+
throw error;
|
|
4991
|
+
}
|
|
4992
|
+
log('ensureStrandPartyKey(%s): lost a concurrent seat race; adopting the landed key', trimmed);
|
|
4993
|
+
return landed;
|
|
4994
|
+
}
|
|
4995
|
+
log('Seated strand party key for %s under owner %s', trimmed, signingKey.publicKeyB64);
|
|
4996
|
+
return key;
|
|
4997
|
+
}
|
|
4998
|
+
|
|
4999
|
+
/**
|
|
5000
|
+
* Resolve the party membership key a closed strand's launch threads into the founder
|
|
5001
|
+
* bootstrap: the explicit attach-time key when given, else the party's persisted
|
|
5002
|
+
* `StrandPartyKey` row, else — on the one machine whose owner key the row names as
|
|
5003
|
+
* founder — a freshly minted-and-persisted key, which seats the identity for a publish
|
|
5004
|
+
* that was interrupted before its mint. Everyone else resolves undefined: non-founding
|
|
5005
|
+
* machines never mint (no mint race between a party's machines), and only a founder
|
|
5006
|
+
* bootstrap needs the key at all.
|
|
5007
|
+
*
|
|
5008
|
+
* Minting does NOT repair a strand founded before the key split: its founding
|
|
5009
|
+
* membership was seated under the shared `MemberPrivateKey`, and nothing re-seats it —
|
|
5010
|
+
* the founder bootstrap refuses such a strand (`PreSplitStrandIdentityError`) instead.
|
|
5011
|
+
*
|
|
5012
|
+
* NOTE: "founding machine" here is the ROW's provenance, not the launch's resolved
|
|
5013
|
+
* `founder` flag — so an explicit `founder: false` over a row this machine published
|
|
5014
|
+
* still mints, an owner-signed write a caller that said "I am not founding" did not ask
|
|
5015
|
+
* for. Harmless (the party needs that identity for the strand either way, and the mint
|
|
5016
|
+
* is insert-if-absent) and unreachable today: the one explicit `founder: false` caller
|
|
5017
|
+
* passes a row with a null `FounderOwnerKey`. If a caller ever pairs `founder: false`
|
|
5018
|
+
* with its own published row, gate the mint on the resolved flag too.
|
|
5019
|
+
*/
|
|
5020
|
+
private async resolveStrandPartyKey(strand: StrandRow, explicitKey?: string): Promise<string | undefined> {
|
|
5021
|
+
if (explicitKey !== undefined) {
|
|
5022
|
+
return explicitKey;
|
|
5023
|
+
}
|
|
5024
|
+
if (!this.controlDatabase) {
|
|
5025
|
+
return undefined;
|
|
5026
|
+
}
|
|
5027
|
+
const stored = await this.controlDatabase.queryStrandPartyKey(strand.Id);
|
|
5028
|
+
if (stored !== null) {
|
|
5029
|
+
return stored;
|
|
5030
|
+
}
|
|
5031
|
+
if (!this.isSelfFoundedRow(strand)) {
|
|
5032
|
+
return undefined;
|
|
5033
|
+
}
|
|
5034
|
+
return await this.ensureStrandPartyKey(strand.Id);
|
|
5035
|
+
}
|
|
5036
|
+
|
|
3824
5037
|
/**
|
|
3825
5038
|
* Publish an owner-signed `FormationInvite` (open-invitation token) to the
|
|
3826
5039
|
* shared control database, so a later {@link formStrand} redemption can be
|
|
@@ -3974,26 +5187,109 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3974
5187
|
* seed, starts the strand, and registers it with the hibernation manager
|
|
3975
5188
|
* before emitting `strand:started`.
|
|
3976
5189
|
*
|
|
3977
|
-
*
|
|
3978
|
-
*
|
|
3979
|
-
*
|
|
3980
|
-
*
|
|
3981
|
-
*
|
|
5190
|
+
* Founder-ness: an explicit `founder` argument wins (the formation/responder flows pass
|
|
5191
|
+
* one deliberately — their consent-seated rows carry a null `FounderOwnerKey`); when
|
|
5192
|
+
* unset it is DERIVED from the row, so the control-discovered path
|
|
5193
|
+
* (`handleStrandAdded`) and a restart's re-attach found this machine's own strands
|
|
5194
|
+
* without the caller having to know. The derivation is pure key comparison — no I/O.
|
|
5195
|
+
*
|
|
5196
|
+
* Idempotent when the strand manager already tracks `strand.Id` — the watcher
|
|
5197
|
+
* rediscovers a row this node already started and calls this again via
|
|
5198
|
+
* `handleStrandAdded`; without the guard that re-entry would resolve a fresh
|
|
3982
5199
|
* (already-connected) cohort seed and re-emit `strand:started` for an
|
|
3983
5200
|
* instance that never stopped. The guard runs before the cohort-seed RPC
|
|
3984
|
-
* fan-out, so a rediscovery costs nothing beyond the map lookup
|
|
5201
|
+
* fan-out, so a rediscovery costs nothing beyond the map lookup plus the (pure)
|
|
5202
|
+
* founder derivation. NOT a silent no-op any more when the launch resolves as a
|
|
5203
|
+
* FOUNDER: the tracked instance may have been launched first as a joiner (an app
|
|
5204
|
+
* attach, or this node's own watcher poll winning the `resolveCohortSeed` window
|
|
5205
|
+
* below), which used to drop the founder request and leave the strand headerless —
|
|
5206
|
+
* now the tracked instance is founded in place
|
|
5207
|
+
* ({@link StrandInstanceManager.foundExistingStrand}), waking it first if quiesced
|
|
5208
|
+
* so the bootstrap actually runs before this resolves.
|
|
5209
|
+
*
|
|
5210
|
+
* A founder launch refused as pre-split (`PreSplitStrandIdentityError`) is recorded in
|
|
5211
|
+
* {@link strandLaunchRefusals} for the formation arm, then rethrown; a founder launch
|
|
5212
|
+
* that succeeds clears the record (it ran the bootstrap's pre-split check and passed).
|
|
3985
5213
|
*/
|
|
3986
5214
|
private async launchStrand(
|
|
3987
5215
|
strand: StrandRow,
|
|
3988
5216
|
sAppConfig: SAppConfig,
|
|
3989
|
-
founder?: boolean
|
|
5217
|
+
founder?: boolean,
|
|
5218
|
+
explicitPartyKey?: string
|
|
5219
|
+
): Promise<StrandInstance> {
|
|
5220
|
+
const resolvedFounder = founder ?? this.isSelfFoundedRow(strand);
|
|
5221
|
+
try {
|
|
5222
|
+
const instance = await this.startOrFoundStrand(strand, sAppConfig, resolvedFounder, explicitPartyKey);
|
|
5223
|
+
if (resolvedFounder) {
|
|
5224
|
+
this.strandLaunchRefusals.delete(strand.Id);
|
|
5225
|
+
}
|
|
5226
|
+
return instance;
|
|
5227
|
+
} catch (error) {
|
|
5228
|
+
if (error instanceof PreSplitStrandIdentityError) {
|
|
5229
|
+
this.strandLaunchRefusals.set(strand.Id, error);
|
|
5230
|
+
}
|
|
5231
|
+
throw error;
|
|
5232
|
+
}
|
|
5233
|
+
}
|
|
5234
|
+
|
|
5235
|
+
/**
|
|
5236
|
+
* The launch itself, for {@link launchStrand} (which documents the behaviour): found a
|
|
5237
|
+
* tracked instance in place, or start a fresh one.
|
|
5238
|
+
*/
|
|
5239
|
+
private async startOrFoundStrand(
|
|
5240
|
+
strand: StrandRow,
|
|
5241
|
+
sAppConfig: SAppConfig,
|
|
5242
|
+
resolvedFounder: boolean,
|
|
5243
|
+
explicitPartyKey: string | undefined
|
|
3990
5244
|
): Promise<StrandInstance> {
|
|
5245
|
+
const timed = <T>(step: string, op: () => Promise<T>) => timedStep('startOrFoundStrand', strand.Id, step, op);
|
|
3991
5246
|
const existing = this.strandManager.getInstance(strand.Id);
|
|
3992
5247
|
if (existing) {
|
|
3993
|
-
|
|
5248
|
+
if (resolvedFounder) {
|
|
5249
|
+
// The resolver runs only when the retained config lacks a party key for a
|
|
5250
|
+
// closed strand (see foundExistingStrand), so the common watcher re-entry
|
|
5251
|
+
// ('already-founder') still costs no control read.
|
|
5252
|
+
const outcome = await timed('foundExistingStrand', () => this.strandManager.foundExistingStrand(strand.Id,
|
|
5253
|
+
() => this.resolveStrandPartyKey(strand, explicitPartyKey)));
|
|
5254
|
+
if (outcome === 'needs-resume') {
|
|
5255
|
+
try {
|
|
5256
|
+
// Quiesced instance: the retained config now founds, but founding promises
|
|
5257
|
+
// the bootstrap has RUN by the time the caller resolves — wake through the
|
|
5258
|
+
// hibernation manager (coalesced with any in-flight wake, timer-aware) so
|
|
5259
|
+
// the rebuild executes it now rather than at some eventual wake.
|
|
5260
|
+
await timed('wakeStrand', () => this.wakeStrand(strand.Id));
|
|
5261
|
+
// The wake's rebuild founds — UNLESS a wake was already in flight when the
|
|
5262
|
+
// config flipped, in which case it had already read the pre-flip config and
|
|
5263
|
+
// rebuilt as a joiner, and `wakeStrand` merely coalesced onto it. Re-run the
|
|
5264
|
+
// (insert-if-absent) bootstrap so founding never resolves headerless.
|
|
5265
|
+
await timed('ensureFounderBootstrap', () => this.strandManager.ensureFounderBootstrap(strand.Id));
|
|
5266
|
+
} catch (error) {
|
|
5267
|
+
// The founding did not happen (e.g. the rebuild refused a pre-split strand and
|
|
5268
|
+
// rolled back, leaving the instance tracked with no runtime): withdraw the flip
|
|
5269
|
+
// so the next attempt re-runs the founding instead of resolving
|
|
5270
|
+
// 'already-founder' over an instance that never founded.
|
|
5271
|
+
this.strandManager.withdrawFounderRequest(strand.Id);
|
|
5272
|
+
throw error;
|
|
5273
|
+
}
|
|
5274
|
+
}
|
|
5275
|
+
log('launchStrand: strand %s already tracked — founder request honored (%s)',
|
|
5276
|
+
strand.Id, outcome);
|
|
5277
|
+
} else {
|
|
5278
|
+
log('launchStrand: strand %s already tracked locally — skipping re-launch', strand.Id);
|
|
5279
|
+
}
|
|
3994
5280
|
return existing;
|
|
3995
5281
|
}
|
|
3996
5282
|
|
|
5283
|
+
// A closed strand's launch carries the party's OWN membership identity key: the
|
|
5284
|
+
// explicit attach-time key, else the control-layer StrandPartyKey row — minted here
|
|
5285
|
+
// when this machine is the row-derived founder and no row exists yet (a publish
|
|
5286
|
+
// interrupted before its mint). A joiner with no persisted key threads undefined;
|
|
5287
|
+
// only a FOUNDER bootstrap needs the key, and that path throws loudly without one
|
|
5288
|
+
// (StrandDatabase).
|
|
5289
|
+
const partyMemberPrivateKey = strand.Type === 'c'
|
|
5290
|
+
? await timed('resolveStrandPartyKey', () => this.resolveStrandPartyKey(strand, explicitPartyKey))
|
|
5291
|
+
: undefined;
|
|
5292
|
+
|
|
3997
5293
|
// Each strand node gets its own transport identity, derived from the cadre
|
|
3998
5294
|
// identity key + strandId (see strand-transport-key.ts). Sharing the
|
|
3999
5295
|
// control node's key here gave every node one peerId, which collides at a
|
|
@@ -4009,8 +5305,9 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4009
5305
|
// identity is ever supported, fall back to `undefined` here — libp2p then
|
|
4010
5306
|
// generates a random per-strand key, which still avoids the collision but
|
|
4011
5307
|
// gives up peerId stability across restarts.
|
|
4012
|
-
const
|
|
4013
|
-
|
|
5308
|
+
const identityKey = this.identityKey;
|
|
5309
|
+
const transportKey = identityKey
|
|
5310
|
+
? await timed('strandTransportKey', () => strandTransportKey(identityKey, strand.Id))
|
|
4014
5311
|
: undefined;
|
|
4015
5312
|
|
|
4016
5313
|
// Derived BEFORE seed resolution so the seed pass doubles as the delegate
|
|
@@ -4018,9 +5315,9 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4018
5315
|
// `libp2p.start()` (the responder records the grant before replying; the
|
|
4019
5316
|
// client awaits the replies).
|
|
4020
5317
|
const delegatePeerId = transportKey ? peerIdFromPrivateKey(transportKey).toString() : undefined;
|
|
4021
|
-
const bootstrapNodes = await this.resolveCohortSeed(strand.Id, delegatePeerId);
|
|
5318
|
+
const bootstrapNodes = await timed('resolveCohortSeed', () => this.resolveCohortSeed(strand.Id, delegatePeerId));
|
|
4022
5319
|
|
|
4023
|
-
const instance = await this.strandManager.startStrand({
|
|
5320
|
+
const instance = await timed('strandManager.startStrand', () => this.strandManager.startStrand({
|
|
4024
5321
|
strandRow: strand,
|
|
4025
5322
|
sAppConfig,
|
|
4026
5323
|
storage: this.config.storage,
|
|
@@ -4031,16 +5328,47 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4031
5328
|
bootstrapNodes,
|
|
4032
5329
|
requireSignedSchemas: this.config.requireSignedSchemas,
|
|
4033
5330
|
clusterSize: this.config.strandClusterSize,
|
|
5331
|
+
// NOTE: deliberately NO `servingMachines`, so the strand node declares no repair
|
|
5332
|
+
// yardstick and runs the frozen STRAND_CLUSTER_POLICY. The only count this node holds
|
|
5333
|
+
// is the party's enrolled machines (`authorizedControlPeers`), and passing that here
|
|
5334
|
+
// is the regression `bug-strand-yardstick-counts-party-machines` removed — a strand
|
|
5335
|
+
// runs on a subset of the party, and over-declaring makes repair impossible rather
|
|
5336
|
+
// than merely weak. The full argument, and the count that will legitimately go here,
|
|
5337
|
+
// are on `StartStrandConfig.servingMachines`.
|
|
4034
5338
|
backfill: this.config.strandBackfill,
|
|
4035
|
-
|
|
4036
|
-
|
|
5339
|
+
revocationEnforcement: this.config.strandRevocationEnforcement,
|
|
5340
|
+
membershipReconciliation: this.config.strandMembershipReconciliation,
|
|
5341
|
+
onSelfRevoked: (revokedStrandId) => this.emit('strand:revoked', { strandId: revokedStrandId }),
|
|
5342
|
+
// The joiner's first-sync write gate (strand-first-sync-gate.ts): a launch that
|
|
5343
|
+
// comes up `'syncing'` announces the moment its database is published.
|
|
5344
|
+
firstSync: this.config.strandFirstSync,
|
|
5345
|
+
onWritable: (writableStrandId) => this.handleStrandWritable(writableStrandId),
|
|
5346
|
+
// Re-announce the delegate to ONE relay before the strand node's reservation
|
|
5347
|
+
// supervisor re-drives it (see announceDelegateToRelay). Retained with the
|
|
5348
|
+
// launch config, so a hibernation wake's rebuilt supervisors get it too.
|
|
5349
|
+
announceDelegateToRelay: (announcedStrandId, relayAddr, announcedDelegatePeerId) =>
|
|
5350
|
+
this.announceDelegateToRelay(announcedStrandId, relayAddr, announcedDelegatePeerId),
|
|
5351
|
+
// The staged formation invitation seam for the bring-up membership
|
|
5352
|
+
// reconciler: read lazily per pass (a re-formation replaces the entry) and
|
|
5353
|
+
// cleared once spent, burned, or dead — see pendingMembershipInvites.
|
|
5354
|
+
pendingMembershipInvite: {
|
|
5355
|
+
get: () => this.pendingMembershipInvites.get(strand.Id),
|
|
5356
|
+
clear: () => { this.pendingMembershipInvites.delete(strand.Id); }
|
|
5357
|
+
},
|
|
5358
|
+
// The RESOLVED flag, never the raw argument — see the doc comment above.
|
|
5359
|
+
founder: resolvedFounder,
|
|
5360
|
+
// Retained with the launch config, so a hibernation wake rebuilds under the
|
|
5361
|
+
// same identity without re-reading the control DB.
|
|
5362
|
+
partyMemberPrivateKey
|
|
5363
|
+
}));
|
|
4037
5364
|
|
|
4038
5365
|
// The seed reached the node as `bootstrapNodes`, which only enters the
|
|
4039
5366
|
// address book via @libp2p/bootstrap discovery. Merge it directly too, so a
|
|
4040
5367
|
// sibling is dialable by bare peer id from the first moment (see
|
|
4041
5368
|
// mergeStrandPeerAddrs; refreshStrandPeerAddrs keeps it warm from here on).
|
|
4042
|
-
|
|
4043
|
-
|
|
5369
|
+
const strandNode = instance.libp2pNode;
|
|
5370
|
+
if (strandNode) {
|
|
5371
|
+
await timed('mergeStrandPeerAddrs', () => this.mergeStrandPeerAddrs(strandNode, bootstrapNodes, strand.Id));
|
|
4044
5372
|
}
|
|
4045
5373
|
|
|
4046
5374
|
this.hibernationManager.trackStrand(instance);
|
|
@@ -4075,6 +5403,21 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4075
5403
|
* for a relay we are not yet connected to.
|
|
4076
5404
|
*/
|
|
4077
5405
|
private async resolveCohortSeed(strandId: string, delegatePeerId?: string): Promise<string[]> {
|
|
5406
|
+
const siblings = await this.resolveSiblingSeed(strandId, delegatePeerId);
|
|
5407
|
+
// Sibling answers FIRST: they were resolved just now, while a contact-map entry is as
|
|
5408
|
+
// old as the formation that produced it. `unionAddrs` appends only what the siblings
|
|
5409
|
+
// did not already name, so a fresher answer is never displaced and each source keeps
|
|
5410
|
+
// its own signaling-first ordering.
|
|
5411
|
+
return unionAddrs(siblings, this.crossPartyStrandAddrs.get(strandId) ?? []);
|
|
5412
|
+
}
|
|
5413
|
+
|
|
5414
|
+
/**
|
|
5415
|
+
* The own-party half of {@link resolveCohortSeed}: strand-network addresses resolved
|
|
5416
|
+
* from CONNECTED cohort siblings over the control-mesh strand-addr RPC, doubling as
|
|
5417
|
+
* the delegate announcement when `delegatePeerId` is given. Empty when the control DB
|
|
5418
|
+
* or node is absent (not yet started / torn down).
|
|
5419
|
+
*/
|
|
5420
|
+
private async resolveSiblingSeed(strandId: string, delegatePeerId?: string): Promise<string[]> {
|
|
4078
5421
|
if (!this.controlDatabase || !this.controlNode) {
|
|
4079
5422
|
return [];
|
|
4080
5423
|
}
|
|
@@ -4177,10 +5520,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4177
5520
|
* Recorded OPTIMISTICALLY at announce time: `collectStrandAddrs` folds
|
|
4178
5521
|
* per-peer failure to `[]` and reports no per-peer success, and threading
|
|
4179
5522
|
* success out would change its API for little gain. A failed INITIAL announce
|
|
4180
|
-
*
|
|
4181
|
-
*
|
|
4182
|
-
*
|
|
4183
|
-
* (15 min), still inside the 30 min TTL.
|
|
5523
|
+
* costs the strand supervisor its first attempt only (the relay denies the
|
|
5524
|
+
* reservation; every re-drive re-announces first through
|
|
5525
|
+
* {@link announceDelegateToRelay}); a failed REFRESH retries within
|
|
5526
|
+
* `DELEGATE_GRANT_TTL_MS / 2` (15 min), still inside the 30 min TTL.
|
|
4184
5527
|
*/
|
|
4185
5528
|
private recordDelegateAnnounces(relayPeerIds: readonly string[], strandId: string, now = Date.now()): void {
|
|
4186
5529
|
for (const relayPeerId of relayPeerIds) {
|
|
@@ -4226,6 +5569,55 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4226
5569
|
));
|
|
4227
5570
|
}
|
|
4228
5571
|
|
|
5572
|
+
/**
|
|
5573
|
+
* Announce ONE strand's delegate peerId to ONE relay, UNTHROTTLED — the
|
|
5574
|
+
* `beforeRedrive` hook of that strand node's per-relay reservation supervisor
|
|
5575
|
+
* (`strand-instance-manager.ts` → `buildStrandRuntime`), run before every
|
|
5576
|
+
* re-drive after the first attempt.
|
|
5577
|
+
*
|
|
5578
|
+
* Why a re-drive must re-announce first: a party control node running the relay
|
|
5579
|
+
* server admits the strand node's derived peerId on an in-memory delegate grant
|
|
5580
|
+
* (`delegate-admission.ts`), and a relay restart drops every grant it held
|
|
5581
|
+
* without telling the announcer. {@link refreshDelegateGrants} would re-announce
|
|
5582
|
+
* on its own at most every `DELEGATE_GRANT_TTL_MS / 2` (15 min) per (relay,
|
|
5583
|
+
* strand) — far slower than the supervisor's backoff — so without this the first
|
|
5584
|
+
* re-drives after a relay restart would be denied at the relay's connection gate.
|
|
5585
|
+
* The throttle map is updated afterwards, so the periodic pass does not announce
|
|
5586
|
+
* again right away.
|
|
5587
|
+
*
|
|
5588
|
+
* Against a dedicated ops relay (no strand-addr RPC) the request fails per-peer
|
|
5589
|
+
* and `collectStrandAddrs` folds it to `[]`: one wasted protocol negotiation per
|
|
5590
|
+
* re-drive attempt, bounded by the supervisor's backoff. Never throws on that
|
|
5591
|
+
* path; a relay addr that names no peer id is logged and skipped (the hook's
|
|
5592
|
+
* caller runs the drive regardless).
|
|
5593
|
+
*
|
|
5594
|
+
* NOTE: against a relay that is DOWN this hook costs up to two strand-addr
|
|
5595
|
+
* timeouts (10 s each: dial by peer id, then by addr) before the 10 s drive even
|
|
5596
|
+
* starts, so one failed re-drive can hold the supervisor `driving` for ~30 s.
|
|
5597
|
+
* Bounded and harmless while the relay is unreachable anyway; if recovery
|
|
5598
|
+
* latency after a relay comes back ever matters, skip the announce when the
|
|
5599
|
+
* control node holds no connection to the relay (the drive's own dial fails
|
|
5600
|
+
* faster) rather than shortening the strand-addr timeout.
|
|
5601
|
+
*/
|
|
5602
|
+
private async announceDelegateToRelay(strandId: string, relayAddr: string, delegatePeerId: string): Promise<void> {
|
|
5603
|
+
const controlNode = this.controlNode;
|
|
5604
|
+
if (!controlNode) {
|
|
5605
|
+
return;
|
|
5606
|
+
}
|
|
5607
|
+
const relayPeerId = trailingPeerId(multiaddr(relayAddr));
|
|
5608
|
+
if (relayPeerId === null) {
|
|
5609
|
+
log('announceDelegateToRelay: relay addr %s names no peer id; strand %s not announced', relayAddr, strandId);
|
|
5610
|
+
return;
|
|
5611
|
+
}
|
|
5612
|
+
await collectStrandAddrs(
|
|
5613
|
+
controlNode,
|
|
5614
|
+
[{ peerId: relayPeerId, addrs: [multiaddr(relayAddr)] }],
|
|
5615
|
+
strandId,
|
|
5616
|
+
{ delegatePeerId }
|
|
5617
|
+
);
|
|
5618
|
+
this.recordDelegateAnnounces([relayPeerId], strandId);
|
|
5619
|
+
}
|
|
5620
|
+
|
|
4229
5621
|
/** One strand's share of {@link refreshDelegateGrants}: announce to the relays whose grant is due. */
|
|
4230
5622
|
private async announceDelegateToDueRelays(
|
|
4231
5623
|
strandId: string,
|
|
@@ -4296,36 +5688,42 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4296
5688
|
if (due.length === 0) {
|
|
4297
5689
|
return;
|
|
4298
5690
|
}
|
|
4299
|
-
// A due strand with nobody to ask leaves its throttle unstamped (see below),
|
|
4300
|
-
// so this pass re-enters on EVERY 15 s reconcile tick for as long as the node
|
|
4301
|
-
// is alone — which is the steady state of a solo node running a strand.
|
|
4302
5691
|
// `connectedSiblingTargets`' membership read is unbounded, and with zero
|
|
4303
|
-
// connections its answer is empty whatever the table holds, so decide it
|
|
4304
|
-
//
|
|
4305
|
-
|
|
4306
|
-
|
|
4307
|
-
|
|
5692
|
+
// connections its answer is empty whatever the table holds, so decide it from the
|
|
5693
|
+
// connection list instead of paying for the read once a tick. Note this only skips
|
|
5694
|
+
// the RPC: a strand carrying cross-party contact addresses still refreshes below,
|
|
5695
|
+
// because those never came from a sibling in the first place.
|
|
5696
|
+
//
|
|
4308
5697
|
// NOTE: one strand-addr RPC per (due strand × connected sibling) per refresh
|
|
4309
5698
|
// interval — each a tiny request/response on an already-open control
|
|
4310
5699
|
// connection. If a node ever runs MANY strands at once, batch the RPC to
|
|
4311
5700
|
// carry several strand ids per request rather than one fan-out per strand.
|
|
4312
|
-
const targets =
|
|
4313
|
-
|
|
4314
|
-
|
|
4315
|
-
|
|
4316
|
-
|
|
4317
|
-
|
|
4318
|
-
//
|
|
4319
|
-
if (
|
|
5701
|
+
const targets = this.controlNode.getConnections().length === 0
|
|
5702
|
+
? []
|
|
5703
|
+
: await this.connectedSiblingTargets().catch((error): StrandAddrPeer[] => {
|
|
5704
|
+
log('refreshStrandPeerAddrs: sibling enumeration failed (skipping pass): %o', error);
|
|
5705
|
+
return [];
|
|
5706
|
+
});
|
|
5707
|
+
// A shutdown landed mid-enumeration.
|
|
5708
|
+
if (!this._running || !this.controlNode) {
|
|
4320
5709
|
return;
|
|
4321
5710
|
}
|
|
4322
5711
|
await Promise.all(due.map((strandId) => this.refreshOneStrandPeerAddrs(strandId, targets, now)));
|
|
4323
5712
|
}
|
|
4324
5713
|
|
|
4325
5714
|
/**
|
|
4326
|
-
* One strand's share of {@link refreshStrandPeerAddrs}: RPC the siblings,
|
|
4327
|
-
*
|
|
4328
|
-
* swallowed so one strand's failure never
|
|
5715
|
+
* One strand's share of {@link refreshStrandPeerAddrs}: RPC the siblings, union their
|
|
5716
|
+
* answers with this strand's cross-party contact addresses, and merge the lot into the
|
|
5717
|
+
* strand's address book. Errors are logged and swallowed so one strand's failure never
|
|
5718
|
+
* costs the others their refresh.
|
|
5719
|
+
*
|
|
5720
|
+
* `targets` may be EMPTY — a solo node, or one whose only control peers are strangers.
|
|
5721
|
+
* The cross-party addresses still have to be re-merged in that case: they were learned
|
|
5722
|
+
* at formation and can never be re-resolved, so this periodic re-merge is the ONLY
|
|
5723
|
+
* thing standing between them and the peerStore's one-hour expiry. A strand with
|
|
5724
|
+
* neither a sibling to ask nor a contact entry does nothing and leaves its throttle
|
|
5725
|
+
* unstamped, so the next reconcile tick retries rather than sitting out the whole
|
|
5726
|
+
* refresh interval having done nothing.
|
|
4329
5727
|
*/
|
|
4330
5728
|
private async refreshOneStrandPeerAddrs(
|
|
4331
5729
|
strandId: string,
|
|
@@ -4337,13 +5735,19 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4337
5735
|
if (!controlNode || !strandNode) {
|
|
4338
5736
|
return;
|
|
4339
5737
|
}
|
|
5738
|
+
const contactAddrs = this.crossPartyStrandAddrs.get(strandId) ?? [];
|
|
5739
|
+
if (targets.length === 0 && contactAddrs.length === 0) {
|
|
5740
|
+
return;
|
|
5741
|
+
}
|
|
4340
5742
|
try {
|
|
4341
5743
|
// The running strand node's own peerId is the delegate to announce — see
|
|
4342
5744
|
// the relationship note on refreshStrandPeerAddrs.
|
|
4343
|
-
const
|
|
4344
|
-
|
|
4345
|
-
|
|
4346
|
-
|
|
5745
|
+
const siblingAddrs = targets.length === 0
|
|
5746
|
+
? []
|
|
5747
|
+
: await collectStrandAddrs(controlNode, [...targets], strandId, {
|
|
5748
|
+
delegatePeerId: strandNode.peerId.toString()
|
|
5749
|
+
});
|
|
5750
|
+
// Stamped on the pass having happened, not on its answer: the fan-out is
|
|
4347
5751
|
// what the throttle exists to bound, and an all-empty round is a normal
|
|
4348
5752
|
// steady state for a strand no connected sibling currently runs.
|
|
4349
5753
|
this.strandPeerAddrRefreshAt.set(strandId, now);
|
|
@@ -4352,7 +5756,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4352
5756
|
if (!this._running || this.strandManager.getInstance(strandId)?.libp2pNode !== strandNode) {
|
|
4353
5757
|
return;
|
|
4354
5758
|
}
|
|
4355
|
-
await this.mergeStrandPeerAddrs(strandNode,
|
|
5759
|
+
await this.mergeStrandPeerAddrs(strandNode, unionAddrs(siblingAddrs, contactAddrs), strandId);
|
|
4356
5760
|
} catch (error) {
|
|
4357
5761
|
log('refreshStrandPeerAddrs: strand %s refresh failed (continuing): %o', strandId, error);
|
|
4358
5762
|
}
|
|
@@ -4376,6 +5780,15 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4376
5780
|
* handshake, and a bad entry costs one failed dial that ages out at the
|
|
4377
5781
|
* peerStore's one-hour expiry. No new gating here — cross-party strand trust is
|
|
4378
5782
|
* `backlog/strand-network-nat-relay-reachability`.
|
|
5783
|
+
*
|
|
5784
|
+
* NOTE: one input does NOT age out — the cross-party addresses a formation
|
|
5785
|
+
* carried back ({@link crossPartyStrandAddrs}). Nothing can re-resolve them, so
|
|
5786
|
+
* {@link refreshStrandPeerAddrs} re-merges the same stored list every pass, which
|
|
5787
|
+
* also means a junk entry from a responder survives for the node's lifetime rather
|
|
5788
|
+
* than an hour. Bounded to {@link MAX_STRAND_ADDRS} per strand, from a party this
|
|
5789
|
+
* node deliberately formed a strand with, and still authority-free — so the
|
|
5790
|
+
* exposure is a handful of failed dials, not a trust hole. Evict on
|
|
5791
|
+
* `unpublishStrand` if that ever stops being an acceptable price.
|
|
4379
5792
|
*/
|
|
4380
5793
|
private async mergeStrandPeerAddrs(strandNode: Libp2p, addrs: string[], strandId: string): Promise<void> {
|
|
4381
5794
|
const counts: Record<MergeAddrsResult, number> = { merged: 0, restamped: 0, skipped: 0, failed: 0 };
|
|
@@ -4419,14 +5832,40 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4419
5832
|
if (!node) {
|
|
4420
5833
|
return [];
|
|
4421
5834
|
}
|
|
4422
|
-
|
|
5835
|
+
// Every entry is bound to THIS strand node's id, because both consumers — the
|
|
5836
|
+
// strand-addr RPC answer and the formation result's `strandAddrs` — are attributed
|
|
5837
|
+
// per peer on arrival (`groupAddrsByPeerId`) and an entry naming no destination is
|
|
5838
|
+
// dropped there. libp2p already appends the id to its announced addresses, so this
|
|
5839
|
+
// is normally a no-op; it is the guard for a bare listen addr and for a relay hop
|
|
5840
|
+
// whose trailing `/p2p/` names the RELAY. An addr terminating in a DIFFERENT id is
|
|
5841
|
+
// dropped rather than announced — it does not reach this node, and announcing it
|
|
5842
|
+
// would file our address under someone else's id in the receiver's book.
|
|
5843
|
+
//
|
|
5844
|
+
// Re-parsed through this package's own `multiaddr` rather than passed straight in:
|
|
5845
|
+
// libp2p hands back its nested `@multiformats/multiaddr` copy, a structurally
|
|
5846
|
+
// different type from the one `withTrailingPeerId` takes (see
|
|
5847
|
+
// `backlog/reference-app-web-libp2p-interface-dedup`). Re-parsing a string libp2p
|
|
5848
|
+
// itself produced cannot fail.
|
|
5849
|
+
const selfId = node.peerId.toString();
|
|
5850
|
+
const addrs: string[] = [];
|
|
5851
|
+
for (const ma of node.getMultiaddrs()) {
|
|
5852
|
+
const bound = this.bindAddrToPeer(multiaddr(ma.toString()), selfId);
|
|
5853
|
+
if (bound !== null) {
|
|
5854
|
+
addrs.push(bound.toString());
|
|
5855
|
+
}
|
|
5856
|
+
}
|
|
5857
|
+
return orderSignalingFirst(addrs);
|
|
4423
5858
|
}
|
|
4424
5859
|
|
|
4425
5860
|
/**
|
|
4426
5861
|
* Stop a strand on THIS node only: untrack it from hibernation, drop its sApp config,
|
|
4427
5862
|
* stop the local instance, and emit `strand:stopped`. The shared `Strand` row is left
|
|
4428
|
-
* intact, so on the next node
|
|
4429
|
-
*
|
|
5863
|
+
* intact, so on the next node RESTART the strand is rediscovered and surfaces as
|
|
5864
|
+
* `strand:discovered` again — not on the next watcher poll, which never re-offers a
|
|
5865
|
+
* strand it has already seen (`StrandWatcher.knownStrands` retains the id for the life
|
|
5866
|
+
* of the process). The stop also drops the strand from {@link getDiscoveredStrands},
|
|
5867
|
+
* so a drain cannot undo a deliberate stop. Party-wide removal is
|
|
5868
|
+
* {@link unpublishStrand}.
|
|
4430
5869
|
*/
|
|
4431
5870
|
async stopStrand(strandId: string): Promise<void> {
|
|
4432
5871
|
if (!this._running) {
|
|
@@ -4438,20 +5877,28 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4438
5877
|
|
|
4439
5878
|
/**
|
|
4440
5879
|
* Local teardown for one strand, shared by the caller-driven {@link stopStrand} and the
|
|
4441
|
-
* watcher-driven `handleStrandRemoved`: untrack hibernation, drop the sApp config
|
|
4442
|
-
* the instance, emit `strand:stopped`. Touches no
|
|
4443
|
-
* removal the node is on is the caller's concern,
|
|
5880
|
+
* watcher-driven `handleStrandRemoved`: untrack hibernation, drop the sApp config and any
|
|
5881
|
+
* recorded launch refusal, stop the instance, emit `strand:stopped`. Touches no
|
|
5882
|
+
* control-plane row — which side of the removal the node is on is the caller's concern,
|
|
5883
|
+
* not this method's.
|
|
4444
5884
|
*
|
|
4445
5885
|
* The stop + emit are skipped when the strand manager holds no instance for `strandId` —
|
|
4446
5886
|
* e.g. a party owner that published a strand's row but never ran it locally, or an
|
|
4447
5887
|
* explicit {@link stopStrand} for an id this node never started. `hibernationManager`
|
|
4448
|
-
* untrack and the `sAppConfigs`
|
|
4449
|
-
* nothing to remove), so a launch
|
|
4450
|
-
* gets its stray
|
|
5888
|
+
* untrack and the `sAppConfigs` / {@link strandLaunchRefusals} / {@link discoveredStrands}
|
|
5889
|
+
* deletes stay unconditional (all no-ops when there is nothing to remove), so a launch
|
|
5890
|
+
* that failed before an instance was ever tracked still gets its stray entries cleared.
|
|
5891
|
+
*
|
|
5892
|
+
* Dropping the {@link discoveredStrands} entry here covers both callers, and both
|
|
5893
|
+
* readings are the intended one: `handleStrandRemoved` arrives because the control row
|
|
5894
|
+
* is gone, and an explicit {@link stopStrand} is a deliberate abandonment — neither
|
|
5895
|
+
* strand may be re-offered to a later `getDiscoveredStrands()` drain.
|
|
4451
5896
|
*/
|
|
4452
5897
|
private async detachStrand(strandId: string): Promise<void> {
|
|
4453
5898
|
this.hibernationManager.untrackStrand(strandId);
|
|
4454
5899
|
this.sAppConfigs.delete(strandId);
|
|
5900
|
+
this.strandLaunchRefusals.delete(strandId);
|
|
5901
|
+
this.discoveredStrands.delete(strandId);
|
|
4455
5902
|
if (!this.strandManager.hasStrand(strandId)) {
|
|
4456
5903
|
log('detachStrand: strand %s not tracked locally — no-op (no stop, no strand:stopped)', strandId);
|
|
4457
5904
|
return;
|
|
@@ -4677,6 +6124,11 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4677
6124
|
* relay indistinguishable from a hung start, and much shorter would fail nodes
|
|
4678
6125
|
* on links that were merely slow. The retries carry on in the background after
|
|
4679
6126
|
* this resolves, exactly as they do for a {@link reserveRelays} caller.
|
|
6127
|
+
*
|
|
6128
|
+
* `network.requireRelay === false` softens only the outcome below: a first
|
|
6129
|
+
* attempt that lands no `/p2p-circuit` address is logged instead of thrown, and
|
|
6130
|
+
* `start()` carries on with the retry supervisor already running in the
|
|
6131
|
+
* background (the same supervisor {@link reserveRelays} always starts).
|
|
4680
6132
|
*/
|
|
4681
6133
|
private async driveControlRelayReservation(): Promise<void> {
|
|
4682
6134
|
const relayAddrs = this.config.network?.relayAddrs ?? [];
|
|
@@ -4687,6 +6139,11 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4687
6139
|
const state = await this.reserveRelays([...relayAddrs]);
|
|
4688
6140
|
timing('[start] reserveRelays: %dms', Math.round(performance.now() - t0));
|
|
4689
6141
|
if (state.status !== 'reserved') {
|
|
6142
|
+
if (this.config.network?.requireRelay === false) {
|
|
6143
|
+
log('Relay reservation did not land on the first attempt (status: %s); ' +
|
|
6144
|
+
'continuing without it because network.requireRelay is false: %o', state.status, state);
|
|
6145
|
+
return;
|
|
6146
|
+
}
|
|
4690
6147
|
throw new RelayReservationFailedError(relayAddrs, state);
|
|
4691
6148
|
}
|
|
4692
6149
|
log('Reserved a relay slot: %o', state.circuitAddrs);
|
|
@@ -4841,6 +6298,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4841
6298
|
partyId: this.config.controlNetwork.partyId,
|
|
4842
6299
|
ownerPrivateKey,
|
|
4843
6300
|
inviteAddressResolver: () => this.resolveInviteAddresses(),
|
|
6301
|
+
dialBudget: this.controlDialBudget(),
|
|
4844
6302
|
trustPolicy: this.config.seedTrustPolicy,
|
|
4845
6303
|
// Seed trust anchors on the node-local store (seeded just above with this
|
|
4846
6304
|
// node's own genesis key), never on the replicated OwnerKey table.
|
|
@@ -5093,6 +6551,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
5093
6551
|
partyId: this.config.controlNetwork.partyId,
|
|
5094
6552
|
// No owner key - this node only receives seeds
|
|
5095
6553
|
inviteAddressResolver: () => this.resolveInviteAddresses(),
|
|
6554
|
+
dialBudget: this.controlDialBudget(),
|
|
5096
6555
|
trustPolicy: this.config.seedTrustPolicy,
|
|
5097
6556
|
// A listener-only node accepts a wire-delivered seed solely against this
|
|
5098
6557
|
// anchor (there is no per-call override on the inbound handler): with no
|
|
@@ -5173,6 +6632,26 @@ export class CadreNode implements SAppIdLookup {
|
|
|
5173
6632
|
}
|
|
5174
6633
|
}
|
|
5175
6634
|
|
|
6635
|
+
/**
|
|
6636
|
+
* Re-materialize a CLOSED strand's revoked-peer deny set now, and hang up any
|
|
6637
|
+
* connected peer it newly covers — the strand-side counterpart to
|
|
6638
|
+
* {@link refreshMembershipGate}.
|
|
6639
|
+
*
|
|
6640
|
+
* Unlike that one, NOTHING calls this automatically: strand membership is
|
|
6641
|
+
* written through the strand's own `Database` handle
|
|
6642
|
+
* (`strand-membership-writer.ts`), which raises no notification this runtime
|
|
6643
|
+
* can hook, and a revocation that arrives by REPLICATION raises none either.
|
|
6644
|
+
* The gate therefore polls (default 30 s). An app that has just called
|
|
6645
|
+
* `revokeMember` or `leaveStrand` should follow the write with this call so
|
|
6646
|
+
* the cut is immediate rather than up to one poll interval late.
|
|
6647
|
+
*
|
|
6648
|
+
* Quiet no-op for a strand that is not running, is quiesced, is open, or has
|
|
6649
|
+
* the gate disabled. Never rejects; resolves once the sweep has finished.
|
|
6650
|
+
*/
|
|
6651
|
+
async refreshRevocationEnforcement(strandId: string): Promise<void> {
|
|
6652
|
+
await this.strandManager.refreshRevocationEnforcement(strandId);
|
|
6653
|
+
}
|
|
6654
|
+
|
|
5176
6655
|
/**
|
|
5177
6656
|
* The single in-flight refresh loop behind {@link refreshMembershipGate}: read
|
|
5178
6657
|
* until the stale flag stays clear, then release the slot.
|
|
@@ -5246,6 +6725,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
5246
6725
|
await this.seedBootstrapService.authorizePeer({ peerId, multiaddrs });
|
|
5247
6726
|
// Queue for re-replication if this committed local-only (no connected cohort).
|
|
5248
6727
|
this.noteControlWrite(peerId, 'authorize');
|
|
6728
|
+
// NOTE: unlike addDrone, `multiaddrs` is not retained as a dial target: the
|
|
6729
|
+
// flows that vouch through here have the new peer reach this node (it applies a
|
|
6730
|
+
// seed naming the owners, or it dialed in with an invite). If one ever vouches a
|
|
6731
|
+
// peer that cannot reach this node, retain its addresses as addDrone does.
|
|
5249
6732
|
}
|
|
5250
6733
|
|
|
5251
6734
|
/**
|
|
@@ -5263,6 +6746,13 @@ export class CadreNode implements SAppIdLookup {
|
|
|
5263
6746
|
await this.seedBootstrapService.removePeer(peerId);
|
|
5264
6747
|
// Track + loudly flag a delete that committed local-only (security-relevant).
|
|
5265
6748
|
this.noteControlWrite(peerId, 'remove');
|
|
6749
|
+
// A removed peer is no longer a dial target, however this node learned its
|
|
6750
|
+
// address. Gone from the store synchronously; a persist failure only lets the
|
|
6751
|
+
// entry reappear after a restart, so it is logged rather than failing a
|
|
6752
|
+
// removal that has already committed.
|
|
6753
|
+
void this.bootstrapPeerStore?.forget(peerId).catch((error: unknown) => {
|
|
6754
|
+
log('removePeer: persisting the removal of dial target %s failed (forgotten in memory): %o', peerId, error);
|
|
6755
|
+
});
|
|
5266
6756
|
}
|
|
5267
6757
|
|
|
5268
6758
|
/**
|
|
@@ -5304,6 +6794,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
5304
6794
|
const tempService = new SeedBootstrapService({
|
|
5305
6795
|
partyId: seed.partyId,
|
|
5306
6796
|
trustPolicy: this.config.seedTrustPolicy,
|
|
6797
|
+
dialBudget: this.controlDialBudget(),
|
|
5307
6798
|
// The anchor is node-scoped, not service-scoped: a throwaway service
|
|
5308
6799
|
// must consult (and persist an accepted signer into) the SAME store the
|
|
5309
6800
|
// persistent one would, or a cold-start enrollment via this path would
|
|
@@ -5393,6 +6884,20 @@ export class CadreNode implements SAppIdLookup {
|
|
|
5393
6884
|
/**
|
|
5394
6885
|
* Add a drone to the cadre (for phone/server adding provider-hosted node).
|
|
5395
6886
|
* Creates authorization and seed for drone initialization.
|
|
6887
|
+
*
|
|
6888
|
+
* Also retains the drone's handed-over addresses as a durable dial target (see
|
|
6889
|
+
* {@link bootstrapPeerStore}). The drone cannot dial an owner that does not
|
|
6890
|
+
* listen (a phone), and its `CadrePeer` row stays unsigned — so unresolvable —
|
|
6891
|
+
* until it self-publishes over a connection; this node therefore has to open
|
|
6892
|
+
* that connection from the addresses it was handed, on this launch or a later
|
|
6893
|
+
* one.
|
|
6894
|
+
*
|
|
6895
|
+
* Nothing is dialed here: the drone has not received the seed yet. After
|
|
6896
|
+
* delivering it, call {@link reconcileControlCohort} to dial straight away;
|
|
6897
|
+
* otherwise the next timed reconcile pass does. A pass already in flight is
|
|
6898
|
+
* joined rather than restarted, and one that listed siblings before this add
|
|
6899
|
+
* does not dial the drone, so a caller waiting for the connection should allow
|
|
6900
|
+
* for one more timed pass.
|
|
5396
6901
|
*/
|
|
5397
6902
|
async addDrone(options: AddDroneOptions): Promise<DroneInitResult> {
|
|
5398
6903
|
if (!this.seedBootstrapService) {
|
|
@@ -5400,7 +6905,12 @@ export class CadreNode implements SAppIdLookup {
|
|
|
5400
6905
|
}
|
|
5401
6906
|
// The drone's `CadrePeer` insert notifies the membership hub on the way to the
|
|
5402
6907
|
// seed, so the per-stream gate already admits it here.
|
|
5403
|
-
|
|
6908
|
+
const result = await this.seedBootstrapService.addDrone(options);
|
|
6909
|
+
// Same queueing as authorizePeer: a phone adding its first always-on node has
|
|
6910
|
+
// no control connection, so the insert usually committed local-only.
|
|
6911
|
+
this.noteControlWrite(options.dronePeerId, 'authorize');
|
|
6912
|
+
this.retainDialTarget(options.dronePeerId, options.droneMultiaddrs, 'addDrone');
|
|
6913
|
+
return result;
|
|
5404
6914
|
}
|
|
5405
6915
|
|
|
5406
6916
|
/**
|
|
@@ -5479,6 +6989,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
5479
6989
|
const tempService = new SeedBootstrapService({
|
|
5480
6990
|
partyId: invite.partyId,
|
|
5481
6991
|
trustPolicy: this.config.seedTrustPolicy,
|
|
6992
|
+
dialBudget: this.controlDialBudget(),
|
|
5482
6993
|
});
|
|
5483
6994
|
if (this.controlNode && this.controlDatabase) {
|
|
5484
6995
|
tempService.initialize(this.controlNode, this.controlDatabase, { registerHandler: false });
|
|
@@ -5507,7 +7018,15 @@ export class CadreNode implements SAppIdLookup {
|
|
|
5507
7018
|
this.strandSolicitationService = new StrandSolicitationService({
|
|
5508
7019
|
...options,
|
|
5509
7020
|
partyId: this.config.controlNetwork.partyId,
|
|
5510
|
-
cadrePeerAddrs: this.getMultiaddrs()
|
|
7021
|
+
cadrePeerAddrs: this.getMultiaddrs(),
|
|
7022
|
+
// Overrides any caller-supplied hook, exactly like partyId/cadrePeerAddrs: only
|
|
7023
|
+
// this node can say which strand-network addresses it is actually listening on,
|
|
7024
|
+
// and a wrong answer here seeds a joiner's mesh with addresses that reach nobody.
|
|
7025
|
+
resolveStrandAddrs: (strandId: string) => this.getStrandMultiaddrs(strandId),
|
|
7026
|
+
// Overridden on the same grounds: only this node holds the running strand
|
|
7027
|
+
// instance's database and this party's `StrandPartyKey` identity, and a wrong
|
|
7028
|
+
// issuer here would admit joiners under someone else's authority.
|
|
7029
|
+
issueMembershipInvite: (strandId: string) => this.issueStrandMembershipInvite(strandId)
|
|
5511
7030
|
});
|
|
5512
7031
|
|
|
5513
7032
|
// Register as responder on the control node
|
|
@@ -5569,11 +7088,190 @@ export class CadreNode implements SAppIdLookup {
|
|
|
5569
7088
|
this.initializeStrandSolicitation();
|
|
5570
7089
|
}
|
|
5571
7090
|
|
|
5572
|
-
|
|
7091
|
+
const result = await this.strandSolicitationService!.formStrand(
|
|
5573
7092
|
invitation,
|
|
5574
7093
|
disclosure,
|
|
5575
7094
|
this.controlNode
|
|
5576
7095
|
);
|
|
7096
|
+
this.recordCrossPartyStrandAddrs(result.strandId, result.strandAddrs);
|
|
7097
|
+
// A closed host strand's approval carries the joiner's own membership invitation —
|
|
7098
|
+
// persist this party's identity key and stage the invitation for bring-up to redeem.
|
|
7099
|
+
// Runs AFTER the addr recording so a persistence failure (which throws — see the
|
|
7100
|
+
// method) still leaves the cross-party seed in place for a manual recovery.
|
|
7101
|
+
if (result.membershipInvite) {
|
|
7102
|
+
await this.adoptFormationMembershipInvite(result.strandId, result.membershipInvite);
|
|
7103
|
+
}
|
|
7104
|
+
return result;
|
|
7105
|
+
}
|
|
7106
|
+
|
|
7107
|
+
/**
|
|
7108
|
+
* Adopt an approved closed-strand formation's membership half on the JOINER:
|
|
7109
|
+
*
|
|
7110
|
+
* 1. Mint-or-reuse this party's own strand identity — {@link ensureStrandPartyKey}
|
|
7111
|
+
* with no explicit key returns the stored `StrandPartyKey` row when one exists (a
|
|
7112
|
+
* re-formation by an existing member party: lost addresses, an app reinstall with
|
|
7113
|
+
* an intact control DB) and mints + persists a fresh one otherwise. The invitation
|
|
7114
|
+
* will admit THIS key's public half as the `Strand.Member`.
|
|
7115
|
+
* 2. Stage the invitation in {@link pendingMembershipInvites} for the strand
|
|
7116
|
+
* bring-up flow (`strand-node-binds-member-peer`) to redeem via `consumeInvite`.
|
|
7117
|
+
*
|
|
7118
|
+
* Throws — failing the whole {@link formStrand} — when the identity cannot be
|
|
7119
|
+
* persisted: a joiner "joined" without a persistable identity could never become a
|
|
7120
|
+
* member, and failing loudly beats a silent half-member. The formation itself has
|
|
7121
|
+
* already succeeded by then and its one-time token is SPENT, so the error says so:
|
|
7122
|
+
* recovery is fixing the underlying cause (no owner signing key / control DB write
|
|
7123
|
+
* rejected) and redeeming a FRESH invitation.
|
|
7124
|
+
*/
|
|
7125
|
+
private async adoptFormationMembershipInvite(strandId: string, invite: StrandMembershipInvite): Promise<void> {
|
|
7126
|
+
try {
|
|
7127
|
+
await this.ensureStrandPartyKey(strandId);
|
|
7128
|
+
} catch (error) {
|
|
7129
|
+
throw new Error(
|
|
7130
|
+
`Formation for strand ${strandId} was approved (its one-time token is spent), but ` +
|
|
7131
|
+
'persisting this party\'s membership identity (StrandPartyKey) failed — the joiner ' +
|
|
7132
|
+
'cannot become a member without it. Fix the underlying cause, then redeem a fresh ' +
|
|
7133
|
+
'invitation (the delivered one dies with this error).',
|
|
7134
|
+
{ cause: error }
|
|
7135
|
+
);
|
|
7136
|
+
}
|
|
7137
|
+
// NOTE: staging alone is not enough on a node whose reconciler for this strand has
|
|
7138
|
+
// already finished. `StrandMembershipReconciler` latches a terminal stopped state on
|
|
7139
|
+
// its `done` path and is rebuilt only by a strand relaunch, so a RE-formation against
|
|
7140
|
+
// an already-launched strand stages an invitation no loop will ever look at. Harmless
|
|
7141
|
+
// for a party that is still a member (the invitation would only be burned), and the
|
|
7142
|
+
// live defect for one that has been REMOVED — which is exactly when an app re-forms.
|
|
7143
|
+
// Tracked as `backlog/bug-removed-party-cannot-redeem-its-way-back`; pinned by
|
|
7144
|
+
// `strand-party-removal-via-formation-e2e.integration.ts` (test 2).
|
|
7145
|
+
this.pendingMembershipInvites.set(strandId, invite);
|
|
7146
|
+
log('formStrand: staged membership invitation for strand %s (party key persisted)', strandId);
|
|
7147
|
+
}
|
|
7148
|
+
|
|
7149
|
+
/**
|
|
7150
|
+
* The pending single-use membership invitation a closed-strand formation carried back
|
|
7151
|
+
* for `strandId`, or `undefined` when none is staged — the seam the strand bring-up
|
|
7152
|
+
* membership reconciler (`strand-membership-reconciler.ts`) reads to redeem the
|
|
7153
|
+
* joiner's `Strand.Member` seat. `undefined` therefore also means "already redeemed,
|
|
7154
|
+
* burned, or found dead": the reconciler clears the entry as soon as it settles the
|
|
7155
|
+
* invitation, so a caller polling this sees it disappear on its own. In-memory only;
|
|
7156
|
+
* see {@link pendingMembershipInvites} for lifetime and re-formation semantics.
|
|
7157
|
+
*/
|
|
7158
|
+
getPendingMembershipInvite(strandId: string): StrandMembershipInvite | undefined {
|
|
7159
|
+
return this.pendingMembershipInvites.get(strandId);
|
|
7160
|
+
}
|
|
7161
|
+
|
|
7162
|
+
/**
|
|
7163
|
+
* Responder-side issuer behind the formation manager's `issueMembershipInvite` seam
|
|
7164
|
+
* (see `StrandFormationManagerOptions.issueMembershipInvite` for the contract this
|
|
7165
|
+
* implements): mint a single-use `Strand.Invite` against the LIVE host strand so a
|
|
7166
|
+
* validated joiner can seat its own `Strand.Member` row.
|
|
7167
|
+
*
|
|
7168
|
+
* - Open host strand → `null` (no members, nothing to invite into).
|
|
7169
|
+
* - Closed host strand whose founder launch was refused as pre-split
|
|
7170
|
+
* ({@link strandLaunchRefusals}) → rethrow the recorded `PreSplitStrandIdentityError`;
|
|
7171
|
+
* the manager maps it to the NON-retryable `HOST_STRAND_MUST_BE_RECREATED_REASON`.
|
|
7172
|
+
* - Closed host strand with no `StrandPartyKey` row → throw: this party's identity is
|
|
7173
|
+
* the invite's issuing manager, and without it nothing can sign the issuance. (The
|
|
7174
|
+
* founder's publish/launch paths mint it, so this is a not-yet-converged sibling.)
|
|
7175
|
+
* The manager maps the throw to a clean retryable rejection BEFORE the formation
|
|
7176
|
+
* token is spent.
|
|
7177
|
+
* - Closed host strand with no running local instance/database → throw, same mapping:
|
|
7178
|
+
* a joiner admitted without an invitation would look joined and never become a
|
|
7179
|
+
* member, and a responder not running the strand cannot serve its sync anyway.
|
|
7180
|
+
* - Closed host strand whose LIVE rows carry the pre-split fingerprint
|
|
7181
|
+
* (`assertNotPreSplitStrand`) → throw `PreSplitStrandIdentityError`, same mapping as
|
|
7182
|
+
* the recorded refusal. Covers the responders that never ran a refused founder launch:
|
|
7183
|
+
* a sibling machine of the founding party, or a node restarted since the refusal.
|
|
7184
|
+
*
|
|
7185
|
+
* The recorded refusal is checked first: it is an in-memory read and the only
|
|
7186
|
+
* permanent diagnosis. Identity is checked BEFORE the runtime: it is the cheaper read
|
|
7187
|
+
* and the more actionable diagnosis when both are missing (a missing runtime is
|
|
7188
|
+
* transient, a missing identity is not), and it keeps the branch reachable without
|
|
7189
|
+
* standing a strand runtime up.
|
|
7190
|
+
*
|
|
7191
|
+
* The invitation expires `MEMBERSHIP_INVITE_TTL_MS` from now — see that constant for
|
|
7192
|
+
* the slow-joiner / lost-result tradeoff.
|
|
7193
|
+
*
|
|
7194
|
+
* NOTE: the issuing identity must be a `Strand.Manager` (the schema's `InviteValid`
|
|
7195
|
+
* gate), and only the FOUNDING party's key is seated as one. Today only the founder
|
|
7196
|
+
* party can host a bound formation at all — a joining party never gets the host
|
|
7197
|
+
* strand's control `Strand` row, so `resolveStrand` reports `missing` on it — so this
|
|
7198
|
+
* never bites. If a joined party is ever able to host formations into a strand it
|
|
7199
|
+
* joined (re-invite / multi-hop join), issuance here fails the manager gate and every
|
|
7200
|
+
* such redemption rejects with the retryable-sounding
|
|
7201
|
+
* `MEMBERSHIP_INVITE_UNAVAILABLE_REASON` forever; that flow needs manager delegation,
|
|
7202
|
+
* not a retry.
|
|
7203
|
+
*/
|
|
7204
|
+
private async issueStrandMembershipInvite(strandId: string): Promise<StrandMembershipInvite | null> {
|
|
7205
|
+
if (!this.controlDatabase) {
|
|
7206
|
+
throw new Error(`Cannot issue a membership invitation for strand ${strandId}: control database unavailable`);
|
|
7207
|
+
}
|
|
7208
|
+
const row = await this.controlDatabase.queryStrand(strandId);
|
|
7209
|
+
if (!row) {
|
|
7210
|
+
// The bound resolution saw this row moments ago; a vanished row is a concurrent
|
|
7211
|
+
// unpublish — reject rather than invite into a strand this party just removed.
|
|
7212
|
+
throw new Error(`Cannot issue a membership invitation for strand ${strandId}: its Strand row is gone`);
|
|
7213
|
+
}
|
|
7214
|
+
if (row.Type !== 'c') {
|
|
7215
|
+
return null;
|
|
7216
|
+
}
|
|
7217
|
+
const refusal = this.strandLaunchRefusals.get(strandId);
|
|
7218
|
+
if (refusal) {
|
|
7219
|
+
throw refusal;
|
|
7220
|
+
}
|
|
7221
|
+
const partyKey = await this.controlDatabase.queryStrandPartyKey(strandId);
|
|
7222
|
+
if (partyKey === null) {
|
|
7223
|
+
throw new Error(
|
|
7224
|
+
`Cannot issue a membership invitation for closed strand ${strandId}: this party holds ` +
|
|
7225
|
+
'no StrandPartyKey row for it (identity not yet converged from the machine that published it)'
|
|
7226
|
+
);
|
|
7227
|
+
}
|
|
7228
|
+
const db = this.strandManager.getInstance(strandId)?.database?.getDatabase();
|
|
7229
|
+
if (!db) {
|
|
7230
|
+
throw new Error(
|
|
7231
|
+
`Cannot issue a membership invitation for closed strand ${strandId}: its runtime is ` +
|
|
7232
|
+
'not live on this responder (not launched, hibernating, or quiescing)'
|
|
7233
|
+
);
|
|
7234
|
+
}
|
|
7235
|
+
if (row.MemberPrivateKey) {
|
|
7236
|
+
await assertNotPreSplitStrand(db, strandId, strandMemberKeyPair(row.MemberPrivateKey).publicKeyB64);
|
|
7237
|
+
}
|
|
7238
|
+
return await issueInvite(db, {
|
|
7239
|
+
managerKeyPair: strandMemberKeyPair(partyKey),
|
|
7240
|
+
expiration: Date.now() + MEMBERSHIP_INVITE_TTL_MS
|
|
7241
|
+
});
|
|
7242
|
+
}
|
|
7243
|
+
|
|
7244
|
+
/**
|
|
7245
|
+
* Remember the responder's strand-network addresses for a strand this node just
|
|
7246
|
+
* formed, so the strand's discovery seed has something to work with when the app
|
|
7247
|
+
* launches it (see {@link crossPartyStrandAddrs}).
|
|
7248
|
+
*
|
|
7249
|
+
* Scoped strictly to `strandId`: these addresses reach ONE strand node of ONE other
|
|
7250
|
+
* party and must never seed another strand's mesh or the control peerStore. An empty
|
|
7251
|
+
* list is recorded as no entry at all rather than an empty one, so a later formation
|
|
7252
|
+
* against the same strand that DOES disclose addresses is not shadowed.
|
|
7253
|
+
*/
|
|
7254
|
+
private recordCrossPartyStrandAddrs(strandId: string, strandAddrs: readonly string[]): void {
|
|
7255
|
+
if (strandAddrs.length === 0) {
|
|
7256
|
+
log('formStrand: responder disclosed no strand addrs for %s — cross-party seed stays empty', strandId);
|
|
7257
|
+
return;
|
|
7258
|
+
}
|
|
7259
|
+
// Capped at the same `MAX_STRAND_ADDRS` one arriving list is, because this
|
|
7260
|
+
// ACCUMULATES: re-forming against the same strand (the documented recovery for a
|
|
7261
|
+
// responder whose relay reservation rotated) unions a fresh list onto the old one,
|
|
7262
|
+
// and nothing here can tell a dead entry from a live one. Without the cap, the seed
|
|
7263
|
+
// — and the peerStore rows every refresh pass re-merges from it — would grow by up
|
|
7264
|
+
// to 16 entries per redemption for the node's lifetime. Oldest entries fall off the
|
|
7265
|
+
// end, which is the right end to lose: `unionAddrs` keeps the freshest list first.
|
|
7266
|
+
const merged = unionAddrs(strandAddrs, this.crossPartyStrandAddrs.get(strandId) ?? [])
|
|
7267
|
+
.slice(0, MAX_STRAND_ADDRS);
|
|
7268
|
+
this.crossPartyStrandAddrs.set(strandId, merged);
|
|
7269
|
+
// A strand already running when this lands (a re-formation) would otherwise wait out
|
|
7270
|
+
// the rest of its ~10-minute refresh throttle before the new addresses reached its
|
|
7271
|
+
// address book. Clearing the stamp makes the next 15 s reconcile tick merge them,
|
|
7272
|
+
// which matters precisely because re-forming is the recovery path for a dead entry.
|
|
7273
|
+
this.strandPeerAddrRefreshAt.delete(strandId);
|
|
7274
|
+
log('formStrand: recorded %d cross-party strand addr(s) for %s', merged.length, strandId);
|
|
5577
7275
|
}
|
|
5578
7276
|
|
|
5579
7277
|
/**
|