@serfab/cadre-core 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +255 -253
- package/dist/bootstrap-peer-store-file.d.ts +13 -0
- package/dist/bootstrap-peer-store-file.js +42 -0
- package/dist/bootstrap-peer-store-file.js.map +1 -0
- package/dist/bootstrap-peer-store.d.ts +73 -0
- package/dist/bootstrap-peer-store.js +128 -0
- package/dist/bootstrap-peer-store.js.map +1 -0
- package/dist/cadre-node.d.ts +1121 -81
- package/dist/cadre-node.js +2276 -262
- package/dist/cadre-node.js.map +1 -1
- package/dist/control-authorization.d.ts +93 -0
- package/dist/control-authorization.js +57 -0
- package/dist/control-authorization.js.map +1 -0
- package/dist/control-cohort.d.ts +26 -0
- package/dist/control-cohort.js +26 -0
- package/dist/control-cohort.js.map +1 -1
- package/dist/control-database.d.ts +959 -54
- package/dist/control-database.js +1421 -148
- package/dist/control-database.js.map +1 -1
- package/dist/control-formation-recorder.d.ts +90 -10
- package/dist/control-formation-recorder.js +130 -14
- package/dist/control-formation-recorder.js.map +1 -1
- package/dist/control-schema.d.ts +1 -1
- package/dist/control-schema.js +651 -72
- package/dist/control-schema.js.map +1 -1
- package/dist/control-stream.d.ts +38 -4
- package/dist/control-stream.js +70 -4
- package/dist/control-stream.js.map +1 -1
- package/dist/control-write-retry.d.ts +161 -0
- package/dist/control-write-retry.js +412 -0
- package/dist/control-write-retry.js.map +1 -0
- package/dist/delegate-admission.d.ts +134 -0
- package/dist/delegate-admission.js +244 -0
- package/dist/delegate-admission.js.map +1 -0
- package/dist/device-token.d.ts +11 -11
- package/dist/device-token.js +14 -13
- package/dist/device-token.js.map +1 -1
- package/dist/ed25519-key.d.ts +26 -0
- package/dist/ed25519-key.js +56 -1
- package/dist/ed25519-key.js.map +1 -1
- package/dist/file-durable-slot.d.ts +20 -0
- package/dist/file-durable-slot.js +53 -0
- package/dist/file-durable-slot.js.map +1 -0
- package/dist/formation-approval.d.ts +157 -0
- package/dist/formation-approval.js +375 -0
- package/dist/formation-approval.js.map +1 -0
- package/dist/fs-atomic.d.ts +38 -0
- package/dist/fs-atomic.js +97 -0
- package/dist/fs-atomic.js.map +1 -0
- package/dist/identity-key.d.ts +49 -0
- package/dist/identity-key.js +81 -0
- package/dist/identity-key.js.map +1 -0
- package/dist/index.d.ts +17 -7
- package/dist/index.js +40 -6
- package/dist/index.js.map +1 -1
- package/dist/key-store-file.d.ts +2 -24
- package/dist/key-store-file.js +10 -77
- package/dist/key-store-file.js.map +1 -1
- package/dist/membership-connection-gater.d.ts +146 -0
- package/dist/membership-connection-gater.js +180 -0
- package/dist/membership-connection-gater.js.map +1 -0
- package/dist/node-local-snapshot.d.ts +109 -0
- package/dist/node-local-snapshot.js +182 -0
- package/dist/node-local-snapshot.js.map +1 -0
- package/dist/peer-addr-book.d.ts +92 -0
- package/dist/peer-addr-book.js +220 -0
- package/dist/peer-addr-book.js.map +1 -0
- package/dist/peer-authorization.d.ts +135 -18
- package/dist/peer-authorization.js +152 -21
- package/dist/peer-authorization.js.map +1 -1
- package/dist/peer-record.d.ts +21 -8
- package/dist/peer-record.js +26 -10
- package/dist/peer-record.js.map +1 -1
- package/dist/relay-addrs.d.ts +45 -0
- package/dist/relay-addrs.js +103 -0
- package/dist/relay-addrs.js.map +1 -0
- package/dist/relay-reservation.d.ts +266 -0
- package/dist/relay-reservation.js +613 -0
- package/dist/relay-reservation.js.map +1 -0
- package/dist/seed-bootstrap.d.ts +228 -56
- package/dist/seed-bootstrap.js +340 -198
- package/dist/seed-bootstrap.js.map +1 -1
- package/dist/seed-trust-policy.d.ts +47 -21
- package/dist/seed-trust-policy.js +24 -15
- package/dist/seed-trust-policy.js.map +1 -1
- package/dist/strand-addr-protocol.d.ts +42 -13
- package/dist/strand-addr-protocol.js +58 -52
- package/dist/strand-addr-protocol.js.map +1 -1
- package/dist/strand-backfill.d.ts +109 -0
- package/dist/strand-backfill.js +279 -0
- package/dist/strand-backfill.js.map +1 -0
- package/dist/strand-cohort.d.ts +4 -37
- package/dist/strand-cohort.js +4 -15
- package/dist/strand-cohort.js.map +1 -1
- package/dist/strand-database.d.ts +11 -16
- package/dist/strand-database.js +19 -7
- package/dist/strand-database.js.map +1 -1
- package/dist/strand-formation-manager.d.ts +36 -7
- package/dist/strand-formation-manager.js +115 -15
- package/dist/strand-formation-manager.js.map +1 -1
- package/dist/strand-formation-protocol.d.ts +103 -4
- package/dist/strand-formation-protocol.js +232 -4
- package/dist/strand-formation-protocol.js.map +1 -1
- package/dist/strand-instance-manager.d.ts +88 -19
- package/dist/strand-instance-manager.js +178 -34
- package/dist/strand-instance-manager.js.map +1 -1
- package/dist/strand-member-registry.d.ts +5 -5
- package/dist/strand-member-registry.js +18 -10
- package/dist/strand-member-registry.js.map +1 -1
- package/dist/strand-membership-writer.d.ts +484 -88
- package/dist/strand-membership-writer.js +751 -128
- package/dist/strand-membership-writer.js.map +1 -1
- package/dist/strand-solicitation.d.ts +99 -33
- package/dist/strand-solicitation.js +100 -52
- package/dist/strand-solicitation.js.map +1 -1
- package/dist/strand-transport-key.d.ts +40 -0
- package/dist/strand-transport-key.js +56 -0
- package/dist/strand-transport-key.js.map +1 -0
- package/dist/strand-wake-protocol.d.ts +9 -7
- package/dist/strand-wake-protocol.js +22 -46
- package/dist/strand-wake-protocol.js.map +1 -1
- package/dist/strand-watcher.d.ts +17 -2
- package/dist/strand-watcher.js +47 -3
- package/dist/strand-watcher.js.map +1 -1
- package/dist/trusted-owner-store-file.d.ts +12 -0
- package/dist/trusted-owner-store-file.js +39 -0
- package/dist/trusted-owner-store-file.js.map +1 -0
- package/dist/trusted-owner-store.d.ts +64 -0
- package/dist/trusted-owner-store.js +126 -0
- package/dist/trusted-owner-store.js.map +1 -0
- package/dist/types.d.ts +307 -44
- package/dist/types.js +13 -0
- package/dist/types.js.map +1 -1
- package/package.json +20 -8
- package/src/arachnode-stub.ts +159 -159
- package/src/bootstrap-peer-store-file.ts +44 -0
- package/src/bootstrap-peer-store.ts +172 -0
- package/src/cadre-node.ts +5158 -2981
- package/src/canonical-json.ts +30 -30
- package/src/control-authorization.ts +114 -0
- package/src/control-cohort.ts +130 -103
- package/src/control-database.ts +2406 -948
- package/src/control-formation-recorder.ts +305 -151
- package/src/control-schema.ts +651 -72
- package/src/control-stream.ts +205 -128
- package/src/control-write-retry.ts +483 -0
- package/src/delegate-admission.ts +302 -0
- package/src/device-token.ts +94 -89
- package/src/diagnostics/connection-path.ts +277 -277
- package/src/diagnostics/webrtc-turn-tracker.ts +222 -222
- package/src/ed25519-key.ts +66 -1
- package/src/enrollment.ts +199 -199
- package/src/file-durable-slot.ts +61 -0
- package/src/formation-approval.ts +604 -0
- package/src/fs-atomic.ts +107 -0
- package/src/hibernation-manager.ts +437 -437
- package/src/identity-key.ts +109 -0
- package/src/index.ts +137 -5
- package/src/key-store-file.ts +132 -203
- package/src/key-store.ts +99 -99
- package/src/membership-connection-gater.ts +204 -0
- package/src/node-local-snapshot.ts +260 -0
- package/src/peer-addr-book.ts +250 -0
- package/src/peer-authorization.ts +259 -81
- package/src/peer-record.ts +164 -146
- package/src/push-credentials.ts +84 -84
- package/src/push-node.ts +64 -64
- package/src/push-notifier-apns.ts +247 -247
- package/src/push-notifier-fcm.ts +220 -220
- package/src/push-notifier-shared.ts +28 -28
- package/src/push-notifier.ts +52 -52
- package/src/relay-addrs.ts +111 -0
- package/src/relay-reservation.ts +823 -0
- package/src/seed-bootstrap.ts +409 -209
- package/src/seed-trust-policy.ts +140 -107
- package/src/strand-addr-protocol.ts +441 -410
- package/src/strand-backfill.ts +366 -0
- package/src/strand-cohort.ts +38 -77
- package/src/strand-database.ts +202 -204
- package/src/strand-formation-manager.ts +491 -368
- package/src/strand-formation-protocol.ts +692 -391
- package/src/strand-instance-manager.ts +201 -46
- package/src/strand-member-key.ts +39 -39
- package/src/strand-member-registry.ts +182 -177
- package/src/strand-membership-writer.ts +938 -146
- package/src/strand-solicitation.ts +472 -407
- package/src/strand-transport-key.ts +58 -0
- package/src/strand-wake-payload.ts +32 -32
- package/src/strand-wake-protocol.ts +314 -331
- package/src/strand-watcher.ts +314 -251
- package/src/trusted-owner-store-file.ts +41 -0
- package/src/trusted-owner-store.ts +171 -0
- package/src/types.ts +1309 -1032
package/dist/cadre-node.js
CHANGED
|
@@ -1,27 +1,38 @@
|
|
|
1
1
|
import debug from 'debug';
|
|
2
2
|
import { toString as uint8ArrayToString, fromString as uint8ArrayFromString } from 'uint8arrays';
|
|
3
|
-
import {
|
|
4
|
-
import { peerIdFromString } from '@libp2p/peer-id';
|
|
3
|
+
import { peerIdFromString, peerIdFromPrivateKey } from '@libp2p/peer-id';
|
|
5
4
|
import { createLibp2pNode } from '@optimystic/db-p2p';
|
|
5
|
+
import { wrapStorageWithCache, disposeStorageCache } from '@serfab/quereus-plugin-sereus';
|
|
6
6
|
import { multiaddr } from '@multiformats/multiaddr';
|
|
7
|
-
import { DEFAULT_CHECKIN_WINDOW_MS } from './types.js';
|
|
7
|
+
import { CONTROL_CLUSTER_POLICY, CONTROL_REPLICATION_BREADTH, DEFAULT_CHECKIN_WINDOW_MS } from './types.js';
|
|
8
8
|
import { sign } from '@optimystic/quereus-plugin-crypto';
|
|
9
|
-
import { ed25519KeyPairFromLibp2p } from './ed25519-key.js';
|
|
9
|
+
import { ed25519KeyPairFromLibp2p, ed25519PublicKeyFromPrivate, requireEd25519PublicKeyB64 } from './ed25519-key.js';
|
|
10
|
+
import { strandTransportKey } from './strand-transport-key.js';
|
|
10
11
|
import { DEFAULT_IDENTITY_KEY_ID } from './key-store.js';
|
|
12
|
+
import { loadOrCreateIdentityKey } from './identity-key.js';
|
|
13
|
+
import { MemoryTrustedOwnerStore } from './trusted-owner-store.js';
|
|
14
|
+
import { MemoryBootstrapPeerStore } from './bootstrap-peer-store.js';
|
|
15
|
+
import { mergePeerAddrs, groupAddrsByPeerId } from './peer-addr-book.js';
|
|
16
|
+
import { verifyCadrePeerVoucher } from './peer-authorization.js';
|
|
11
17
|
import { ed25519PublicKeyB64FromPeerId } from './seed-bootstrap.js';
|
|
12
|
-
import { signPeerRecord, verifyPeerRecordSignature, isPeerRecordFresh, orderSignalingFirst, isSignalingAddr, currentMemberTrustPolicy, DEFAULT_PEER_RECORD_MAX_AGE_MS, DEFAULT_PEER_RECORD_HEARTBEAT_MS } from './peer-record.js';
|
|
18
|
+
import { signPeerRecord, verifyPeerRecordSignature, isPeerRecordFresh, orderSignalingFirst, isSignalingAddr, trailingPeerId, currentMemberTrustPolicy, DEFAULT_PEER_RECORD_MAX_AGE_MS, DEFAULT_PEER_RECORD_HEARTBEAT_MS } from './peer-record.js';
|
|
13
19
|
import { signDeviceTokenRecord, verifyDeviceTokenSignature, isPushPlatform } from './device-token.js';
|
|
14
20
|
import { StrandWatcher } from './strand-watcher.js';
|
|
15
21
|
import { StrandInstanceManager } from './strand-instance-manager.js';
|
|
16
|
-
import { deriveCohortMembers
|
|
17
|
-
import { selectControlCohortDials, DEFAULT_CONTROL_COHORT_RECONCILE_MS, DEFAULT_CONTROL_COHORT_TARGET_DEGREE } from './control-cohort.js';
|
|
22
|
+
import { deriveCohortMembers } from './strand-cohort.js';
|
|
23
|
+
import { selectControlCohortDials, DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS, DEFAULT_CONTROL_COHORT_RECONCILE_MS, DEFAULT_CONTROL_COHORT_TARGET_DEGREE } from './control-cohort.js';
|
|
24
|
+
import { withDeadline } from './control-stream.js';
|
|
18
25
|
import { EnrollmentService } from './enrollment.js';
|
|
19
26
|
import { HibernationManager } from './hibernation-manager.js';
|
|
20
27
|
import { ControlDatabase } from './control-database.js';
|
|
21
28
|
import { SeedBootstrapService } from './seed-bootstrap.js';
|
|
22
29
|
import { StrandSolicitationService } from './strand-solicitation.js';
|
|
30
|
+
import { createMembershipConnectionGater, DEFAULT_ENROLLMENT_WINDOW_MS } from './membership-connection-gater.js';
|
|
23
31
|
import { StrandWakeService, dialWake } from './strand-wake-protocol.js';
|
|
24
32
|
import { StrandAddrService, collectStrandAddrs } from './strand-addr-protocol.js';
|
|
33
|
+
import { DelegateAdmissionStore, extractCircuitRelayTargets, dueRelayAnnounces, pruneStoppedStrandAnnounces, peerStrandKey } from './delegate-admission.js';
|
|
34
|
+
import { resolveListenAddrs } from './relay-addrs.js';
|
|
35
|
+
import { superviseRelayReservation, resolveRelayReservationState } from './relay-reservation.js';
|
|
25
36
|
import { PushFanoutService } from './push-fanout.js';
|
|
26
37
|
import { summarizeConnectionPaths, classifyTransport } from './diagnostics/connection-path.js';
|
|
27
38
|
import { TurnRelayTracker } from './diagnostics/webrtc-turn-tracker.js';
|
|
@@ -33,6 +44,45 @@ const timing = debug('sereus:cadre:timing');
|
|
|
33
44
|
* `RTCPeerConnection` reaching `connected` and libp2p surfacing the connection.
|
|
34
45
|
*/
|
|
35
46
|
const TURN_CONSUME_WINDOW_MS = 1000;
|
|
47
|
+
/**
|
|
48
|
+
* How often a running strand re-resolves its siblings' strand-network addresses
|
|
49
|
+
* and re-merges them into its own libp2p address book
|
|
50
|
+
* ({@link CadreNode.refreshStrandPeerAddrs}). Overridable per node via
|
|
51
|
+
* `network.controlCohort.strandAddrRefreshMs`.
|
|
52
|
+
*
|
|
53
|
+
* Ten minutes sits comfortably inside the peerStore's one-hour address expiry
|
|
54
|
+
* (`MAX_ADDRESS_AGE`, see `peer-addr-book.ts`) with headroom for several missed
|
|
55
|
+
* passes, and far above the 15 s reconcile cadence the refresh rides on, so the
|
|
56
|
+
* strand-addr RPC fan-out stays cheap.
|
|
57
|
+
*/
|
|
58
|
+
export const STRAND_PEER_ADDR_REFRESH_MS = 10 * 60 * 1000;
|
|
59
|
+
/**
|
|
60
|
+
* A circuit relay as a strand-addr RPC target: dialed by peerId, with its
|
|
61
|
+
* direct addr riding along as the fallback for a relay we hold no connection to.
|
|
62
|
+
*/
|
|
63
|
+
function relayStrandAddrPeer(relay) {
|
|
64
|
+
return { peerId: relay.relayPeerId, addrs: [multiaddr(relay.relayAddr)] };
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Build the signing callback the `ControlDatabase` writers expect: they hand it the
|
|
68
|
+
* canonical row-bound message BYTES (see `buildAuthorizationMessage`), and it ed25519-signs
|
|
69
|
+
* them directly (no pre-hash) with the owner private key, returning a base64url signature.
|
|
70
|
+
*/
|
|
71
|
+
function signMessageWith(privateKeyB64) {
|
|
72
|
+
return (message) => sign(message, privateKeyB64, 'ed25519', 'bytes', 'base64url', 'base64url');
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Reject a blank identifier before it reaches the control database, where it would fail an
|
|
76
|
+
* authorization CHECK as an opaque constraint error rather than an actionable message.
|
|
77
|
+
* Returns the trimmed value so callers write the same bytes they validated.
|
|
78
|
+
*/
|
|
79
|
+
function requireNonBlank(value, label) {
|
|
80
|
+
const trimmed = value.trim();
|
|
81
|
+
if (trimmed.length === 0) {
|
|
82
|
+
throw new Error(`A ${label} is required (received an empty or whitespace-only value)`);
|
|
83
|
+
}
|
|
84
|
+
return trimmed;
|
|
85
|
+
}
|
|
36
86
|
/**
|
|
37
87
|
* CadreNode is the main entry point for a cadre member.
|
|
38
88
|
* It manages:
|
|
@@ -44,6 +94,15 @@ const TURN_CONSUME_WINDOW_MS = 1000;
|
|
|
44
94
|
*/
|
|
45
95
|
export class CadreNode {
|
|
46
96
|
constructor(config) {
|
|
97
|
+
/**
|
|
98
|
+
* Node-local, NON-replicated trusted-owner anchor (see
|
|
99
|
+
* `trusted-owner-store.ts`). Constructed (or adopted from
|
|
100
|
+
* `config.trustedOwners.store`) by {@link initializeTrustedOwnerStore} during
|
|
101
|
+
* {@link start}; deliberately NOT cleared by {@link cleanup}, so an in-memory
|
|
102
|
+
* anchor survives a stop()→start() cycle of the same node instance. Never
|
|
103
|
+
* sourced from replicated control state.
|
|
104
|
+
*/
|
|
105
|
+
this.trustedOwnerStore = null;
|
|
47
106
|
this.controlNode = null;
|
|
48
107
|
this.controlDatabase = null;
|
|
49
108
|
this.strandWatcher = null;
|
|
@@ -66,6 +125,14 @@ export class CadreNode {
|
|
|
66
125
|
this.pushFanoutService = null;
|
|
67
126
|
/** Backing field for the {@link running} / {@link isRunning} getters. */
|
|
68
127
|
this._running = false;
|
|
128
|
+
/**
|
|
129
|
+
* The control database's own raw storage (cache-wrapped), resolved once per
|
|
130
|
+
* `start()` and released in {@link cleanup}. Owning it is what keeps
|
|
131
|
+
* `config.storage.provider('control')` a once-per-runtime call: a `stop()` then
|
|
132
|
+
* `start()` cycle on this object re-resolves against a live cache, and never
|
|
133
|
+
* orphans the previous wrapper's registration in the shared cache pool.
|
|
134
|
+
*/
|
|
135
|
+
this.controlStorage = null;
|
|
69
136
|
/**
|
|
70
137
|
* In-flight {@link serviceWake} operations keyed by strandId. Coalesces
|
|
71
138
|
* concurrent on-demand wakes for the same strand into one runtime build + one
|
|
@@ -80,6 +147,25 @@ export class CadreNode {
|
|
|
80
147
|
*/
|
|
81
148
|
this.windowWaiters = new Set();
|
|
82
149
|
this.eventHandlers = new Map();
|
|
150
|
+
/**
|
|
151
|
+
* Relay multiaddrs the caller asked {@link reserveRelays} to reserve through.
|
|
152
|
+
* Empty (the default) means nobody asked, so {@link getRelayReservationState}
|
|
153
|
+
* reports `none` — this whole surface is opt-in and dormant for the CLI/host
|
|
154
|
+
* nodes, which use the configured `network.relayAddrs` route instead.
|
|
155
|
+
*/
|
|
156
|
+
this.relayReserveAddrs = [];
|
|
157
|
+
/**
|
|
158
|
+
* The running retry loop for {@link relayReserveAddrs}, or `null` when nobody
|
|
159
|
+
* asked for a reservation / there was no control node to supervise. It owns the
|
|
160
|
+
* in-flight flag, the last failure and the next-attempt time; this class only
|
|
161
|
+
* starts it, stops it and reads it.
|
|
162
|
+
*/
|
|
163
|
+
this.relayReserveSupervisor = null;
|
|
164
|
+
/**
|
|
165
|
+
* Reason {@link reserveRelays} produced no reservation when there is no
|
|
166
|
+
* supervisor to hold one — i.e. the pre-start `control node unavailable` case.
|
|
167
|
+
*/
|
|
168
|
+
this.relayReserveError = null;
|
|
83
169
|
/** Map of strandId -> sAppConfig for sAppId filtering and management */
|
|
84
170
|
this.sAppConfigs = new Map();
|
|
85
171
|
/**
|
|
@@ -89,6 +175,116 @@ export class CadreNode {
|
|
|
89
175
|
* control-network node never needs to dial back to the manager.
|
|
90
176
|
*/
|
|
91
177
|
this.latestInviteAddresses = null;
|
|
178
|
+
/**
|
|
179
|
+
* Epoch ms until which the control-network inbound gate admits
|
|
180
|
+
* not-yet-authorized peers (see {@link admitInboundControlConnection} /
|
|
181
|
+
* {@link openEnrollmentWindow}). 0 = no window open. Opened automatically by
|
|
182
|
+
* {@link createInvite}; deliberately NOT reset on stop() — a stop/start cycle
|
|
183
|
+
* inside an outstanding invite's validity must not strand the invitee.
|
|
184
|
+
*
|
|
185
|
+
* NOTE: in-memory only, so a PROCESS restart mid-invite closes the door until
|
|
186
|
+
* the owner re-mints or the invitee is authorized. Fine while invites are
|
|
187
|
+
* short-lived and redeemed promptly; if long-lived invites or restart-prone
|
|
188
|
+
* hosts become normal, persist the window (or derive it from the issued-invite
|
|
189
|
+
* records `SeedBootstrapService` already keeps).
|
|
190
|
+
*/
|
|
191
|
+
this.enrollmentWindowUntil = 0;
|
|
192
|
+
/**
|
|
193
|
+
* Lazily-parsed PeerIds of `controlNetwork.bootstrapNodes` (see
|
|
194
|
+
* {@link getBootstrapPeerIds}) — infrastructure the inbound gate always admits.
|
|
195
|
+
*/
|
|
196
|
+
this.bootstrapPeerIds = null;
|
|
197
|
+
/**
|
|
198
|
+
* Materialized in-memory snapshot of the AUTHORIZED member peer ids (the
|
|
199
|
+
* {@link listAuthorizedMembers} result), consulted by the per-stream
|
|
200
|
+
* control-DB gate ({@link authorizeInboundControlStream}). That predicate
|
|
201
|
+
* runs on EVERY inbound Optimystic control-DB stream and must never await a
|
|
202
|
+
* control-DB read: those reads pull blocks over the very protocols the
|
|
203
|
+
* predicate gates, which closes a circular wait that ends in mutual denial
|
|
204
|
+
* (the upstream gate is fail-closed on timeout, unlike the fail-open
|
|
205
|
+
* connection gater). So the set is refreshed OUT OF BAND instead — after
|
|
206
|
+
* start, on every {@link reconcileControlCohort} pass (15s cadence), and
|
|
207
|
+
* immediately after every LOCAL membership mutation so a just-vouched peer
|
|
208
|
+
* is admitted without waiting for the timer. That last refresh is AUTOMATIC:
|
|
209
|
+
* the control database notifies {@link refreshMembershipGate} after every
|
|
210
|
+
* committed `CadrePeer` write (see `ControlDatabase.mutateCadrePeer`), so no
|
|
211
|
+
* writer has to remember. A change that arrives by REPLICATION is picked up
|
|
212
|
+
* on the next timed refresh — bounded staleness, acceptable because this gate
|
|
213
|
+
* is defense in depth: rows an unadmitted peer manages to write are still
|
|
214
|
+
* disbelieved at read time by the voucher-anchored predicate.
|
|
215
|
+
*/
|
|
216
|
+
this.authorizedControlPeers = new Set();
|
|
217
|
+
/**
|
|
218
|
+
* Coalescing state for {@link refreshMembershipGate}: a pending "the snapshot
|
|
219
|
+
* is stale" flag, the single in-flight drain that consumes it, and the depth
|
|
220
|
+
* of open {@link deferMembershipGateRefresh} scopes (a burst of writes inside
|
|
221
|
+
* one scope collapses to a single refresh at scope exit).
|
|
222
|
+
*/
|
|
223
|
+
this.membershipGateDirty = false;
|
|
224
|
+
this.membershipGateDrain = null;
|
|
225
|
+
this.membershipGateDeferDepth = 0;
|
|
226
|
+
/**
|
|
227
|
+
* In-memory delegate admission grants (see `delegate-admission.ts`): the
|
|
228
|
+
* strand-node transport peerIds this party's members have announced over the
|
|
229
|
+
* strand-addr RPC, admitted at the CONNECTION level so a member's NAT'd
|
|
230
|
+
* strand node can hold a circuit-relay reservation on this node. Consulted
|
|
231
|
+
* ONLY by {@link admitInboundControlConnection}; the fail-closed per-stream
|
|
232
|
+
* gate ({@link authorizeInboundControlStream}) never honors it.
|
|
233
|
+
*/
|
|
234
|
+
this.delegateAdmission = new DelegateAdmissionStore();
|
|
235
|
+
/**
|
|
236
|
+
* When this node last ANNOUNCED a delegate, keyed `<targetPeerId>\n<strandId>`
|
|
237
|
+
* (the peer announced TO, not the delegate). Throttles
|
|
238
|
+
* {@link refreshDelegateGrants} to once per `DELEGATE_GRANT_TTL_MS / 2` per
|
|
239
|
+
* (relay, strand) so the 15 s reconcile tick never becomes per-tick RPC
|
|
240
|
+
* chatter; the launch/resume announce passes record here too. Keys whose
|
|
241
|
+
* strand is no longer running are pruned on each reconcile pass.
|
|
242
|
+
*/
|
|
243
|
+
this.delegateAnnounceAt = new Map();
|
|
244
|
+
/**
|
|
245
|
+
* When each running strand last had its siblings' strand addresses re-resolved
|
|
246
|
+
* into its own address book, keyed by strandId. Throttles
|
|
247
|
+
* {@link refreshStrandPeerAddrs} to once per {@link STRAND_PEER_ADDR_REFRESH_MS}
|
|
248
|
+
* per strand so the 15 s reconcile tick never becomes per-tick RPC chatter.
|
|
249
|
+
* Entries for strands that are no longer running are pruned on each pass, so a
|
|
250
|
+
* hibernated-then-resumed strand refreshes immediately rather than inheriting a
|
|
251
|
+
* stale stamp.
|
|
252
|
+
*/
|
|
253
|
+
this.strandPeerAddrRefreshAt = new Map();
|
|
254
|
+
/**
|
|
255
|
+
* Cold-start bootstrap dial targets: the owner-flagged peers of every seed
|
|
256
|
+
* this node has applied, keyed by peer id, valued by the seed's multiaddr
|
|
257
|
+
* strings (parsed lazily, at dial time). Written by
|
|
258
|
+
* {@link recordSeedBootstrapPeers}; read only by
|
|
259
|
+
* {@link dialColdStartBootstrap} while the control database still has no
|
|
260
|
+
* siblings to dial.
|
|
261
|
+
*
|
|
262
|
+
* A STORE rather than a plain map (see `bootstrap-peer-store.ts`), because the
|
|
263
|
+
* targets must outlive the process: a node seeded into a party it could not
|
|
264
|
+
* reach has nothing else on disk naming that party's addresses (`applySeed`
|
|
265
|
+
* writes no control row, and `CadrePeer` fills in only after a connection
|
|
266
|
+
* succeeds), so an in-memory-only set left it stranded permanently across a
|
|
267
|
+
* restart. Constructed (or adopted from `config.bootstrapPeers.store`) by
|
|
268
|
+
* {@link initializeBootstrapPeerStore} during {@link start}; deliberately NOT
|
|
269
|
+
* cleared by {@link cleanup}, so it survives a stop()→start() cycle of the same
|
|
270
|
+
* node instance — same lifecycle as {@link trustedOwnerStore}. Durability
|
|
271
|
+
* depends on the injected backend: Node gets a file-backed store from the CLI,
|
|
272
|
+
* RN/browser stay ephemeral for now.
|
|
273
|
+
*
|
|
274
|
+
* Deliberately NOT the libp2p peer store, which {@link peerStoreAddrs} already
|
|
275
|
+
* reads for the steady-state path. The peer store is shared with everything
|
|
276
|
+
* libp2p discovers, so "dial every entry" would grow into dialing arbitrary
|
|
277
|
+
* discovered peers as the node lives longer; this store holds exactly the peers
|
|
278
|
+
* an owner-signed, trust-anchored seed nominated as owners. `applySeed` merges
|
|
279
|
+
* the same addresses into the peer store as well, so the two never disagree —
|
|
280
|
+
* this one is just the precisely-scoped subset.
|
|
281
|
+
*
|
|
282
|
+
* A later seed OVERWRITES an entry rather than merging, so a re-seed after an
|
|
283
|
+
* owner's address changes replaces the stale address instead of accumulating
|
|
284
|
+
* dead ones. Entries are never evicted: they are the node's only way back into
|
|
285
|
+
* the party if it is ever stranded again.
|
|
286
|
+
*/
|
|
287
|
+
this.bootstrapPeerStore = null;
|
|
92
288
|
/** Initial self-registration timer (see {@link scheduleSelfRegistration}). */
|
|
93
289
|
this.selfRegistrationTimer = null;
|
|
94
290
|
/** TTL heartbeat that re-publishes the self record before it goes stale. */
|
|
@@ -118,15 +314,38 @@ export class CadreNode {
|
|
|
118
314
|
this.registerSelfInFlight = null;
|
|
119
315
|
// ── Write-while-alone re-replication (control-write-ensure-replicated) ──────
|
|
120
316
|
/**
|
|
121
|
-
* Re-replication queue for owner
|
|
122
|
-
* node was alone — `controlNode.getConnections().length === 0` at write
|
|
123
|
-
* means the Optimystic commit was local-only (the block's cluster was ≤1)
|
|
124
|
-
* never broadcast. Maps the affected subject peerId →
|
|
125
|
-
* the cohort grows.
|
|
126
|
-
*
|
|
127
|
-
*
|
|
317
|
+
* Re-replication queue for owner `authorizePeer` writes that committed while
|
|
318
|
+
* this node was alone — `controlNode.getConnections().length === 0` at write
|
|
319
|
+
* time means the Optimystic commit was local-only (the block's cluster was ≤1)
|
|
320
|
+
* and never broadcast. Maps the affected subject peerId → 'authorize', to
|
|
321
|
+
* re-issue once the cohort grows. Removals are NOT tracked here — a delete
|
|
322
|
+
* leaves a `Revocation` tombstone, tracked by {@link pendingRevocations}; a
|
|
323
|
+
* removePeer also clears any queued authorize for the same subject (the row is
|
|
324
|
+
* gone, so re-issuing the insert would be wrong). Drained on the 0→≥1
|
|
325
|
+
* control-connection transition by {@link drainPendingControlReplication}.
|
|
128
326
|
*/
|
|
129
327
|
this.pendingPeerWrites = new Map();
|
|
328
|
+
/**
|
|
329
|
+
* Re-replication queue for `Revocation` tombstones that committed while this
|
|
330
|
+
* node was alone, keyed on the retired `StampId`. Fed by the control DB's
|
|
331
|
+
* committed-delete seam ({@link noteGuardedDelete}), so one queue covers all
|
|
332
|
+
* four guarded tables (`CadrePeer` / `DeviceToken` / `Strand` /
|
|
333
|
+
* `ValidationKey`). Drained (owner-signed `ReissuedAt` bump, which re-broadcasts
|
|
334
|
+
* the tombstone) by {@link drainPendingRevocations}. Cleared on stop alongside
|
|
335
|
+
* {@link pendingPeerWrites} — the tombstone rows are durable, so the next
|
|
336
|
+
* lifetime's first-growth sweep re-covers anything dropped here.
|
|
337
|
+
*/
|
|
338
|
+
this.pendingRevocations = new Map();
|
|
339
|
+
/**
|
|
340
|
+
* Guards the once-per-lifetime first-growth revocation sweep (re-issue EVERY
|
|
341
|
+
* locally-held tombstone, covering removals from before this node started).
|
|
342
|
+
* Deliberately separate from {@link reconstructedLocalOnlyWrites} and set only
|
|
343
|
+
* after a sweep pass SUCCEEDS (including the nothing-held case) — a throwing
|
|
344
|
+
* sweep retries on the next growth edge instead of being lost for the lifetime.
|
|
345
|
+
* Reset on stop, so a stop()→start() cycle sweeps again (that second start is
|
|
346
|
+
* exactly a "removals from before this lifetime" case).
|
|
347
|
+
*/
|
|
348
|
+
this.reissuedHeldRevocations = false;
|
|
130
349
|
/** This node's own `CadrePeer` self-write committed local-only (re-touched on growth). */
|
|
131
350
|
this.pendingSelfPeerWrite = false;
|
|
132
351
|
/** This node's own `DeviceToken` self-write committed local-only (re-touched on growth). */
|
|
@@ -282,6 +501,14 @@ export class CadreNode {
|
|
|
282
501
|
return;
|
|
283
502
|
}
|
|
284
503
|
log('Starting CadreNode for party: %s', this.config.controlNetwork.partyId);
|
|
504
|
+
// NOTE: the only direct `console.*` in this library — a boot-time operator warning, not a
|
|
505
|
+
// diagnostic trace (those use `debug`, which an operator never sees without `DEBUG=`). If a
|
|
506
|
+
// second such warning ever appears here, route both through a `CadreNodeEvents` entry the
|
|
507
|
+
// embedder surfaces instead of growing a console surface inside a library.
|
|
508
|
+
if (this.config.network?.announceAddrs && this.config.network.announceAddrs.length > 0) {
|
|
509
|
+
console.warn('network.announceAddrs is set but not yet supported (no upstream db-p2p option to apply it) ' +
|
|
510
|
+
'— this node will keep advertising its listen/relay addresses instead.');
|
|
511
|
+
}
|
|
285
512
|
try {
|
|
286
513
|
const tTotal = performance.now();
|
|
287
514
|
// Install the WebRTC TURN-relay tracker BEFORE any libp2p bring-up, so it
|
|
@@ -292,6 +519,16 @@ export class CadreNode {
|
|
|
292
519
|
// libp2p/network bring-up, so a misconfiguration or an access-denied secure
|
|
293
520
|
// store fails closed before a node is created.
|
|
294
521
|
await this.resolveIdentityKey();
|
|
522
|
+
// Bring up the node-local trusted-owner anchor (and seed config pins)
|
|
523
|
+
// before any network bring-up: a mis-scoped injected store fails closed
|
|
524
|
+
// here, and out-of-band pins are anchored before the first seed/peer
|
|
525
|
+
// interaction could consult them.
|
|
526
|
+
await this.initializeTrustedOwnerStore();
|
|
527
|
+
// Bring up the node-local cold-start bootstrap-peer store alongside the
|
|
528
|
+
// anchor, and for the same two reasons: a mis-scoped injected store fails
|
|
529
|
+
// closed before any network bring-up, and the retained dial targets are
|
|
530
|
+
// loaded before the first reconcile pass could consult them.
|
|
531
|
+
this.initializeBootstrapPeerStore();
|
|
295
532
|
// Create the control network libp2p node
|
|
296
533
|
let t0 = performance.now();
|
|
297
534
|
this.controlNode = await this.createControlNode();
|
|
@@ -313,6 +550,15 @@ export class CadreNode {
|
|
|
313
550
|
await this.controlDatabase.initialize();
|
|
314
551
|
timing('[start] controlDatabase.initialize: %dms', Math.round(performance.now() - t0));
|
|
315
552
|
log('Control database initialized');
|
|
553
|
+
// Attach the per-stream gate to the control DB's membership hub, so every
|
|
554
|
+
// committed `CadrePeer` write re-materializes the authorized snapshot on its
|
|
555
|
+
// own. Wired before anything can write. `_running` is still false for the
|
|
556
|
+
// rest of start() and the refresh early-returns while it is — a write in that
|
|
557
|
+
// window is covered by start's own pass below.
|
|
558
|
+
this.controlDatabase.setMembershipChangeListener((reason) => this.refreshMembershipGate(reason));
|
|
559
|
+
// Committed-delete seam → write-while-alone revocation queue. Same wiring
|
|
560
|
+
// window argument as the membership listener above.
|
|
561
|
+
this.controlDatabase.setGuardedDeleteListener((revocation) => this.noteGuardedDelete(revocation));
|
|
316
562
|
// Create strand queryable using the control database
|
|
317
563
|
const queryable = this.createStrandQueryable();
|
|
318
564
|
// Create and start the strand watcher with sAppId lookup
|
|
@@ -340,9 +586,14 @@ export class CadreNode {
|
|
|
340
586
|
// resolving a strand's bootstrap seed asks us for our live strand-network
|
|
341
587
|
// multiaddrs (its CadrePeer row only knows our *control* address). Gated on
|
|
342
588
|
// AUTHORIZED membership; answers only for strands we are actively meshing.
|
|
589
|
+
// The same RPC doubles as the delegate-announce channel: a member's request
|
|
590
|
+
// may carry the derived peerId its strand node runs as, and we record an
|
|
591
|
+
// admission grant so our connection gate (and thus our relay server, when
|
|
592
|
+
// enabled) admits it — see delegate-admission.ts.
|
|
343
593
|
this.strandAddrService = new StrandAddrService({
|
|
344
594
|
isMember: (peerId) => this.isAuthorizedMember(peerId),
|
|
345
595
|
getStrandMultiaddrs: (strandId) => this.getStrandMultiaddrs(strandId),
|
|
596
|
+
onDelegateAnnounce: (announcer, strandId, delegate) => this.grantDelegateAdmission(announcer, strandId, delegate),
|
|
346
597
|
});
|
|
347
598
|
this.strandAddrService.initialize(this.controlNode);
|
|
348
599
|
// Server-side push-wake fan-out: only when push is configured.
|
|
@@ -358,6 +609,11 @@ export class CadreNode {
|
|
|
358
609
|
// delayed refresh path) so no early `connection:open` is missed — the
|
|
359
610
|
// write-while-alone re-replication drain must fire on the first 0→≥1 edge.
|
|
360
611
|
this.wireControlConnectionListeners();
|
|
612
|
+
// Seed the per-stream gate's materialized authorized set now that the
|
|
613
|
+
// control DB is up (non-blocking; the helper never rejects). Until it
|
|
614
|
+
// lands the snapshot is empty, which the stream gate treats as cold
|
|
615
|
+
// start and admits.
|
|
616
|
+
void this.refreshMembershipGate('start');
|
|
361
617
|
// Schedule self-registration in background
|
|
362
618
|
this.scheduleSelfRegistration();
|
|
363
619
|
}
|
|
@@ -401,7 +657,18 @@ export class CadreNode {
|
|
|
401
657
|
return;
|
|
402
658
|
}
|
|
403
659
|
log('Stopping CadreNode');
|
|
660
|
+
// Stop the retry loop BEFORE tearing the node down, not after: `cleanup()`
|
|
661
|
+
// stops the control node partway through, and a tick that fires during it
|
|
662
|
+
// would dial and then poll a half-torn-down node for the whole reserve
|
|
663
|
+
// timeout — logging spurious dial failures and holding the process open.
|
|
664
|
+
this.relayReserveSupervisor?.stop();
|
|
665
|
+
this.relayReserveSupervisor = null;
|
|
404
666
|
await this.cleanup();
|
|
667
|
+
// Drop the rest of the relay-reservation posture: a stopped node holds no
|
|
668
|
+
// reservation, so a restarted instance must not report the previous run's
|
|
669
|
+
// `reserved`.
|
|
670
|
+
this.relayReserveAddrs = [];
|
|
671
|
+
this.relayReserveError = null;
|
|
405
672
|
this._running = false;
|
|
406
673
|
this.emit('control:disconnected', undefined);
|
|
407
674
|
log('CadreNode stopped');
|
|
@@ -436,11 +703,11 @@ export class CadreNode {
|
|
|
436
703
|
* before any network bring-up. Resolution order:
|
|
437
704
|
*
|
|
438
705
|
* 1. Both `keyStore` and `privateKey` set ⇒ configuration error (throws).
|
|
439
|
-
* 2. `keyStore` set ⇒
|
|
440
|
-
* {@link DEFAULT_IDENTITY_KEY_ID})
|
|
441
|
-
*
|
|
442
|
-
*
|
|
443
|
-
*
|
|
706
|
+
* 2. `keyStore` set ⇒ {@link loadOrCreateIdentityKey} against `identityKeyId`
|
|
707
|
+
* (default {@link DEFAULT_IDENTITY_KEY_ID}): load when the slot is present,
|
|
708
|
+
* generate + persist when it is empty. A rejected `get` (access denied /
|
|
709
|
+
* backend failure) PROPAGATES — we never generate a new key on a read error,
|
|
710
|
+
* which would silently orphan the real identity.
|
|
444
711
|
* 3. `privateKey` set ⇒ use it directly.
|
|
445
712
|
* 4. Neither ⇒ leave undefined; libp2p generates an ephemeral key.
|
|
446
713
|
*
|
|
@@ -457,21 +724,11 @@ export class CadreNode {
|
|
|
457
724
|
'configure at most one source for the node identity');
|
|
458
725
|
}
|
|
459
726
|
if (keyStore) {
|
|
460
|
-
|
|
461
|
-
//
|
|
462
|
-
//
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
// Corrupt/garbage bytes throw here; surface loudly rather than
|
|
466
|
-
// regenerating (which would orphan the real identity).
|
|
467
|
-
this.identityKey = privateKeyFromProtobuf(bytes);
|
|
468
|
-
log('Identity key loaded from key store (slot present)');
|
|
469
|
-
return;
|
|
470
|
-
}
|
|
471
|
-
const generated = await generateKeyPair('Ed25519');
|
|
472
|
-
await keyStore.set(keyId, privateKeyToProtobuf(generated));
|
|
473
|
-
this.identityKey = generated;
|
|
474
|
-
log('Identity key generated and persisted to key store (first run)');
|
|
727
|
+
// Shared with the embedding app: `reference-app-rn` resolves the very same
|
|
728
|
+
// key before constructing this node so it can sign its ICE-manifest request
|
|
729
|
+
// with it (identity-key.ts). One copy of the rule, so neither side can
|
|
730
|
+
// drift into generating a second identity.
|
|
731
|
+
this.identityKey = await loadOrCreateIdentityKey(keyStore, identityKeyId ?? DEFAULT_IDENTITY_KEY_ID);
|
|
475
732
|
return;
|
|
476
733
|
}
|
|
477
734
|
if (privateKey) {
|
|
@@ -480,19 +737,155 @@ export class CadreNode {
|
|
|
480
737
|
}
|
|
481
738
|
// Neither configured: libp2p generates an ephemeral key internally.
|
|
482
739
|
}
|
|
740
|
+
/**
|
|
741
|
+
* Construct (or adopt) the node-local trusted-owner anchor and seed the
|
|
742
|
+
* out-of-band pinned keys from `config.trustedOwners`. The store is NEVER
|
|
743
|
+
* sourced from the replicated control DB — its entries come only from
|
|
744
|
+
* genesis self-trust ({@link initializeSeedBootstrap}), config pins (here),
|
|
745
|
+
* or runtime enrollment pins ({@link trustOwnerKeys}).
|
|
746
|
+
*
|
|
747
|
+
* Idempotent across stop()→start(): the store instance is kept, and
|
|
748
|
+
* re-seeding config pins is a no-op ({@link TrustedOwnerStore.trust} is
|
|
749
|
+
* idempotent). An injected store scoped to a different party is a
|
|
750
|
+
* configuration error (fail closed before any network bring-up).
|
|
751
|
+
*/
|
|
752
|
+
async initializeTrustedOwnerStore() {
|
|
753
|
+
const { trustedOwners } = this.config;
|
|
754
|
+
const partyId = this.config.controlNetwork.partyId;
|
|
755
|
+
if (!this.trustedOwnerStore) {
|
|
756
|
+
const store = trustedOwners?.store ?? new MemoryTrustedOwnerStore(partyId);
|
|
757
|
+
if (store.partyId !== partyId) {
|
|
758
|
+
throw new Error(`CadreNodeConfig: trustedOwners.store is scoped to party ${store.partyId}, ` +
|
|
759
|
+
`but this node serves party ${partyId} — refusing to mix trust anchors`);
|
|
760
|
+
}
|
|
761
|
+
this.trustedOwnerStore = store;
|
|
762
|
+
}
|
|
763
|
+
// Validate every pin before trusting any: a malformed entry must not leave
|
|
764
|
+
// earlier, valid pins anchored while start() then fails on a later one —
|
|
765
|
+
// config pins land all-or-nothing, same as the runtime seam below.
|
|
766
|
+
const pinnedKeys = (trustedOwners?.pinnedKeys ?? []).map(key => requireEd25519PublicKeyB64(key, 'pinned owner key'));
|
|
767
|
+
for (const key of pinnedKeys) {
|
|
768
|
+
await this.trustedOwnerStore.trust(key, trustedOwners?.pinnedSource ?? 'operator');
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
/**
|
|
772
|
+
* Construct (or adopt) the node-local cold-start bootstrap-peer store (see
|
|
773
|
+
* {@link bootstrapPeerStore}). Synchronous: an injected store has already
|
|
774
|
+
* loaded its persisted targets by the time it is handed in (its `open` is the
|
|
775
|
+
* async part), and the in-memory fallback has nothing to load.
|
|
776
|
+
*
|
|
777
|
+
* Idempotent across stop()→start(): the store instance is kept, so retained
|
|
778
|
+
* targets survive a restart of the same node instance even with the ephemeral
|
|
779
|
+
* default. An injected store scoped to a different party is a configuration
|
|
780
|
+
* error (fail closed before any network bring-up) — a foreign party's addresses
|
|
781
|
+
* must never enter this node's dial loop.
|
|
782
|
+
*/
|
|
783
|
+
initializeBootstrapPeerStore() {
|
|
784
|
+
const partyId = this.config.controlNetwork.partyId;
|
|
785
|
+
if (this.bootstrapPeerStore) {
|
|
786
|
+
return;
|
|
787
|
+
}
|
|
788
|
+
const store = this.config.bootstrapPeers?.store ?? new MemoryBootstrapPeerStore(partyId);
|
|
789
|
+
if (store.partyId !== partyId) {
|
|
790
|
+
throw new Error(`CadreNodeConfig: bootstrapPeers.store is scoped to party ${store.partyId}, ` +
|
|
791
|
+
`but this node serves party ${partyId} — refusing to mix cold-start dial targets`);
|
|
792
|
+
}
|
|
793
|
+
this.bootstrapPeerStore = store;
|
|
794
|
+
}
|
|
795
|
+
/**
|
|
796
|
+
* The node-local cold-start bootstrap-peer store (null before {@link start}) —
|
|
797
|
+
* the retained owner addresses {@link reconcileControlCohort}'s cold-start
|
|
798
|
+
* branch re-dials. Exposed for diagnostics and for a host that wants to show
|
|
799
|
+
* "what would this node dial if it is stranded?".
|
|
800
|
+
*/
|
|
801
|
+
getBootstrapPeerStore() {
|
|
802
|
+
return this.bootstrapPeerStore;
|
|
803
|
+
}
|
|
804
|
+
/**
|
|
805
|
+
* The node-local trusted-owner anchor (null before {@link start}). This is
|
|
806
|
+
* the set the authorized-membership predicate and seed-trust anchor consult —
|
|
807
|
+
* never the replicated `OwnerKey` table, which any stranger can pollute.
|
|
808
|
+
*/
|
|
809
|
+
getTrustedOwnerStore() {
|
|
810
|
+
return this.trustedOwnerStore;
|
|
811
|
+
}
|
|
812
|
+
/**
|
|
813
|
+
* Persist out-of-band-established owner keys into the node-local anchor —
|
|
814
|
+
* the runtime enrollment seam: call with `CadreInvite.ownerKeys` when
|
|
815
|
+
* redeeming an invite (BEFORE the first `applySeed`, so the anchor already
|
|
816
|
+
* holds the pins when seed trust consults it), or with an operator-supplied
|
|
817
|
+
* pin. Idempotent. ('genesis' provenance is reserved for the node's own
|
|
818
|
+
* founding key, seeded internally by {@link initializeSeedBootstrap}.)
|
|
819
|
+
*
|
|
820
|
+
* Validates every key's shape before trusting any of them (all-or-nothing):
|
|
821
|
+
* a malformed entry anywhere in `keys` rejects the whole call before a
|
|
822
|
+
* single key is anchored. For the invite route this means a `CadreInvite`
|
|
823
|
+
* carrying one malformed `ownerKeys` entry fails the redemption outright —
|
|
824
|
+
* consistent with the anchor's existing whole-record-or-nothing policy for
|
|
825
|
+
* a corrupt persisted entry (see `trusted-owner-store.ts`'s
|
|
826
|
+
* `unusableEntry: 'discard-all'`) — rather than silently anchoring a subset
|
|
827
|
+
* and leaving the caller to notice a key went missing.
|
|
828
|
+
*/
|
|
829
|
+
async trustOwnerKeys(keys, source) {
|
|
830
|
+
if (!this.trustedOwnerStore) {
|
|
831
|
+
throw new Error('CadreNode must be started before trusting owner keys');
|
|
832
|
+
}
|
|
833
|
+
const validated = Array.from(keys, key => requireEd25519PublicKeyB64(key, 'pinned owner key'));
|
|
834
|
+
for (const key of validated) {
|
|
835
|
+
await this.trustedOwnerStore.trust(key, source);
|
|
836
|
+
}
|
|
837
|
+
}
|
|
483
838
|
async createControlNode() {
|
|
484
|
-
|
|
839
|
+
return await createLibp2pNode(this.buildControlNodeOptions());
|
|
840
|
+
}
|
|
841
|
+
/**
|
|
842
|
+
* The control database's cache-wrapped raw storage, resolved from
|
|
843
|
+
* `config.storage.provider` on first use and then held in {@link controlStorage}
|
|
844
|
+
* for the rest of this runtime.
|
|
845
|
+
*
|
|
846
|
+
* Resolving once is the contract `RawStorageProvider` (types.ts) states: a provider is
|
|
847
|
+
* called once per scope per runtime lifetime. Re-entering it per call would mint a
|
|
848
|
+
* second store over one backend — a fresh, cold cache, with the previous wrapper's
|
|
849
|
+
* registration orphaned in the process-wide pool.
|
|
850
|
+
*
|
|
851
|
+
* Lazy rather than eager in `start()` so the pure-unit call path
|
|
852
|
+
* (`cadre-node-control-node-options.spec.ts` calls `buildControlNodeOptions` on a
|
|
853
|
+
* bare `new CadreNode`) still resolves — and, on that path too, only once.
|
|
854
|
+
*
|
|
855
|
+
* Wrapped in the write-through raw-storage cache (quereus-plugin-sereus's
|
|
856
|
+
* `cached-storage.ts`) because a control start's cost is its raw-storage
|
|
857
|
+
* operation count.
|
|
858
|
+
*/
|
|
859
|
+
resolveControlStorage() {
|
|
860
|
+
if (this.controlStorage) {
|
|
861
|
+
return this.controlStorage;
|
|
862
|
+
}
|
|
863
|
+
const provider = this.config.storage?.provider;
|
|
864
|
+
if (!provider) {
|
|
865
|
+
return undefined;
|
|
866
|
+
}
|
|
867
|
+
const resolved = typeof provider === 'function' ? provider('control') : provider;
|
|
868
|
+
this.controlStorage = wrapStorageWithCache(resolved, 'control');
|
|
869
|
+
return this.controlStorage;
|
|
870
|
+
}
|
|
871
|
+
/**
|
|
872
|
+
* Map this node's config onto the control network's libp2p node options.
|
|
873
|
+
*
|
|
874
|
+
* Split out of {@link createControlNode} purely so the mapping is assertable without
|
|
875
|
+
* standing up a real libp2p node — `packages/cadre-core/test/cadre-node-control-node-options.spec.ts`
|
|
876
|
+
* calls it on a bare `new CadreNode(config)`. Read nothing else into the split; the
|
|
877
|
+
* only caller in production is `createControlNode`.
|
|
878
|
+
*/
|
|
879
|
+
buildControlNodeOptions() {
|
|
880
|
+
const { controlNetwork, network, profile } = this.config;
|
|
485
881
|
const identityKey = this.identityKey;
|
|
882
|
+
const listenAddrs = resolveListenAddrs(network);
|
|
486
883
|
// Determine relay mode: if explicitly set in config, use that;
|
|
487
884
|
// otherwise default to true for storage profile nodes (better connectivity/uptime)
|
|
488
885
|
const enableRelay = network?.enableRelay ?? (profile === 'storage');
|
|
489
|
-
//
|
|
490
|
-
//
|
|
491
|
-
const controlStorageProvider =
|
|
492
|
-
? (typeof storage.provider === 'function'
|
|
493
|
-
? storage.provider('control')
|
|
494
|
-
: storage.provider)
|
|
495
|
-
: undefined;
|
|
886
|
+
// The control node's own storage, resolved once per runtime (see
|
|
887
|
+
// {@link resolveControlStorage}) rather than per call.
|
|
888
|
+
const controlStorageProvider = this.resolveControlStorage();
|
|
496
889
|
const nodeOptions = {
|
|
497
890
|
port: 0,
|
|
498
891
|
bootstrapNodes: controlNetwork.bootstrapNodes,
|
|
@@ -500,15 +893,279 @@ export class CadreNode {
|
|
|
500
893
|
storage: controlStorageProvider,
|
|
501
894
|
fretProfile: profile === 'storage' ? 'core' : 'edge',
|
|
502
895
|
relay: enableRelay,
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
896
|
+
// Fixed, and deliberately above any party's node count: every member reads the
|
|
897
|
+
// whole control database, so a cohort that excludes a member leaves it dependent
|
|
898
|
+
// on read repair — which cannot converge at a two-member cohort. Not a knob;
|
|
899
|
+
// see CONTROL_REPLICATION_BREADTH. `assumedClusterSize: 2` is declared explicitly in
|
|
900
|
+
// CONTROL_CLUSTER_POLICY: only the admission gate defaults to 2, while the read-repair
|
|
901
|
+
// corroboration floor would otherwise fall back to this clusterSize of 16.
|
|
902
|
+
clusterSize: CONTROL_REPLICATION_BREADTH,
|
|
903
|
+
clusterPolicy: CONTROL_CLUSTER_POLICY,
|
|
904
|
+
arachnode: { enableRingZulu: profile === 'storage' },
|
|
506
905
|
...(identityKey && { privateKey: identityKey }),
|
|
507
906
|
...(network?.transports && { transports: network.transports }),
|
|
508
|
-
|
|
509
|
-
|
|
907
|
+
// Configured `listenAddrs`, plus a `/p2p-circuit` listener for every
|
|
908
|
+
// `network.relayAddrs` entry — that listener is what reserves the relay slot.
|
|
909
|
+
...(listenAddrs && { listenAddrs }),
|
|
910
|
+
// The CONTROL node composes the membership admission gate onto any
|
|
911
|
+
// caller-supplied gater (deny from either wins on inbound; all other
|
|
912
|
+
// hooks pass through). Strand cohort nodes keep the raw configured gater
|
|
913
|
+
// (see strand-instance-manager.ts) — their peers are legitimately
|
|
914
|
+
// cross-party, so cadre membership must not gate them.
|
|
915
|
+
connectionGater: createMembershipConnectionGater({ admitInbound: (remotePeerId) => this.admitInboundControlConnection(remotePeerId) }, network?.connectionGater),
|
|
916
|
+
// Fail-closed per-stream authorization for the four Optimystic control-DB
|
|
917
|
+
// protocols — the members-only layer the connection gater's stranger
|
|
918
|
+
// carve-outs cannot express (see authorizeInboundControlStream). Control
|
|
919
|
+
// node only: strand cohort nodes serve cross-party peers.
|
|
920
|
+
authorizeInboundStream: (remotePeerId, protocol) => this.authorizeInboundControlStream(remotePeerId, protocol)
|
|
510
921
|
};
|
|
511
|
-
return
|
|
922
|
+
return nodeOptions;
|
|
923
|
+
}
|
|
924
|
+
/**
|
|
925
|
+
* Decide whether an inbound CONTROL-network connection from `remotePeerId`
|
|
926
|
+
* should be admitted — the policy behind the connection gater wired in
|
|
927
|
+
* {@link createControlNode}. Deny only on a positive "unauthorized outsider
|
|
928
|
+
* while no stranger path is open" determination; everything ambiguous admits
|
|
929
|
+
* and defers to the fail-closed per-stream gates (see
|
|
930
|
+
* `membership-connection-gater.ts` for the layer's rationale and the
|
|
931
|
+
* stranger-open protocol allowlist).
|
|
932
|
+
*
|
|
933
|
+
* Admits when ANY of:
|
|
934
|
+
* 1. a shared-baseline check admits ({@link admitControlPeerUnconditionally}
|
|
935
|
+
* — not running / DB torn down, absent-or-empty trusted-owner anchor, or
|
|
936
|
+
* configured bootstrap/relay infrastructure);
|
|
937
|
+
* 2. an enrollment window is open ({@link openEnrollmentWindow}, opened by
|
|
938
|
+
* {@link createInvite}) — the invitee dials in before it is authorized;
|
|
939
|
+
* 3. the peer holds a live DELEGATE ADMISSION GRANT — an authorized member
|
|
940
|
+
* announced it (over the strand-addr RPC) as the transport peerId of its
|
|
941
|
+
* own strand node, so a NAT'd member's strand node can hold a
|
|
942
|
+
* circuit-relay reservation here (see `delegate-admission.ts`).
|
|
943
|
+
* Connection only: the per-stream gate below never honors a grant;
|
|
944
|
+
* 4. the authorized-member set is empty — cold start: the rows that would
|
|
945
|
+
* authorize anyone arrive by replication over these very connections;
|
|
946
|
+
* 5. the peer IS an authorized member; or
|
|
947
|
+
* 6. an open invitation is OUTSTANDING — at least one unexpired,
|
|
948
|
+
* not-fully-consumed invitation this node minted or persisted (see
|
|
949
|
+
* `StrandSolicitationService.hasOutstandingInvitation`). A formation
|
|
950
|
+
* initiator is another party's peer by design and its token is only
|
|
951
|
+
* checkable inside the protocol, so the gate asks the coarser question
|
|
952
|
+
* "does this node expect a stranger at all?". Merely REGISTERING the
|
|
953
|
+
* responder ({@link initializeStrandSolicitation}) no longer suspends
|
|
954
|
+
* stranger denial — eager registration (as `reference-app-rn` does at
|
|
955
|
+
* bring-up) and `formStrand`'s lazy initialization both leave the gate
|
|
956
|
+
* armed, because neither mints an invitation.
|
|
957
|
+
*
|
|
958
|
+
* Ordering is semantically free (the checks are OR'd) but decides who pays:
|
|
959
|
+
* checks 1-3 are in-memory, 4/5 share one control-DB read, and only a peer
|
|
960
|
+
* already on the deny path reaches check 6's invitation lookup.
|
|
961
|
+
*
|
|
962
|
+
* Caveats of check 6, both self-healing:
|
|
963
|
+
* - the in-memory mint registry dies with the process, so after a restart
|
|
964
|
+
* only invitations persisted as `FormationInvite` rows still hold the
|
|
965
|
+
* exemption open (re-mint otherwise — same story as the enrollment window);
|
|
966
|
+
* - a peer holding a token whose `FormationInvite` row has not replicated to
|
|
967
|
+
* this node yet is denied even though the formation handler would have
|
|
968
|
+
* accepted it, exactly like the unreplicated-membership-row case below.
|
|
969
|
+
*
|
|
970
|
+
* NOTE: check 4/5 runs a control-DB read per inbound connection
|
|
971
|
+
* (`listAuthorizedMembers`); connections are rare and cadres small, and this
|
|
972
|
+
* layer is fail-open behind `ADMISSION_DECISION_TIMEOUT_MS`, so the live
|
|
973
|
+
* read is safe here — unlike the per-stream gate, which must consult the
|
|
974
|
+
* materialized {@link authorizedControlPeers} snapshot instead.
|
|
975
|
+
* NOTE: a sibling whose membership row has not yet replicated to
|
|
976
|
+
* this node is denied until the row converges (typically via the owner);
|
|
977
|
+
* either side's next outbound reconcile dial (outbound is never gated)
|
|
978
|
+
* re-establishes the link — self-healing, but visible as a transient deny.
|
|
979
|
+
*/
|
|
980
|
+
async admitInboundControlConnection(remotePeerId) {
|
|
981
|
+
if (this.admitControlPeerUnconditionally(remotePeerId)) {
|
|
982
|
+
return true;
|
|
983
|
+
}
|
|
984
|
+
if (Date.now() < this.enrollmentWindowUntil) {
|
|
985
|
+
return true;
|
|
986
|
+
}
|
|
987
|
+
if (this.delegateAdmission.has(remotePeerId)) {
|
|
988
|
+
return true;
|
|
989
|
+
}
|
|
990
|
+
const authorized = await this.listAuthorizedMembers();
|
|
991
|
+
if (authorized.length === 0) {
|
|
992
|
+
return true;
|
|
993
|
+
}
|
|
994
|
+
if (authorized.some((m) => m.peerId === remotePeerId)) {
|
|
995
|
+
return true;
|
|
996
|
+
}
|
|
997
|
+
try {
|
|
998
|
+
if (await this.strandSolicitationService?.hasOutstandingInvitation()) {
|
|
999
|
+
return true;
|
|
1000
|
+
}
|
|
1001
|
+
}
|
|
1002
|
+
catch (error) {
|
|
1003
|
+
log('admitInboundControlConnection: outstanding-invitation check threw for %s — admitting (fail-open): %o', remotePeerId, error);
|
|
1004
|
+
return true;
|
|
1005
|
+
}
|
|
1006
|
+
log('admitInboundControlConnection: DENYING inbound from %s — not an authorized member and no enrollment path open', remotePeerId);
|
|
1007
|
+
return false;
|
|
1008
|
+
}
|
|
1009
|
+
/**
|
|
1010
|
+
* Hold the control-network inbound gate open for not-yet-authorized peers
|
|
1011
|
+
* until `untilEpochMs` (extends, never shrinks, an already-open window).
|
|
1012
|
+
* {@link createInvite} calls this automatically; a host running an
|
|
1013
|
+
* out-of-band enrollment flow (e.g. accepting a phone whose invite this node
|
|
1014
|
+
* never minted) can open it explicitly before the stranger dials in.
|
|
1015
|
+
*/
|
|
1016
|
+
openEnrollmentWindow(untilEpochMs) {
|
|
1017
|
+
this.enrollmentWindowUntil = Math.max(this.enrollmentWindowUntil, untilEpochMs);
|
|
1018
|
+
log('Enrollment window open until %d', this.enrollmentWindowUntil);
|
|
1019
|
+
}
|
|
1020
|
+
/**
|
|
1021
|
+
* Record (or refresh) a delegate admission grant: `delegatePeerId` is the
|
|
1022
|
+
* transport peerId of `announcerPeerId`'s strand-`strandId` node, admitted
|
|
1023
|
+
* at the CONNECTION level (all a circuit-relay reservation needs) for
|
|
1024
|
+
* `DELEGATE_GRANT_TTL_MS`. Called by the strand-addr responder after its
|
|
1025
|
+
* authorized-membership gate has passed; public so tests can drive the
|
|
1026
|
+
* admission policy without a full strand launch. A re-announce for the same
|
|
1027
|
+
* (announcer, strand) REPLACES the previous delegate rather than
|
|
1028
|
+
* accumulating. Never honored by {@link authorizeInboundControlStream}.
|
|
1029
|
+
*/
|
|
1030
|
+
grantDelegateAdmission(announcerPeerId, strandId, delegatePeerId) {
|
|
1031
|
+
this.delegateAdmission.grant(announcerPeerId, strandId, delegatePeerId);
|
|
1032
|
+
}
|
|
1033
|
+
/** Is `remotePeerId` covered by a live delegate admission grant? */
|
|
1034
|
+
hasDelegateAdmission(remotePeerId) {
|
|
1035
|
+
return this.delegateAdmission.has(remotePeerId);
|
|
1036
|
+
}
|
|
1037
|
+
/**
|
|
1038
|
+
* The admission checks SHARED by the connection gate
|
|
1039
|
+
* ({@link admitInboundControlConnection}) and the per-stream control-DB gate
|
|
1040
|
+
* ({@link authorizeInboundControlStream}) — factored so the two layers cannot
|
|
1041
|
+
* drift. All in-memory and cheap (the stream gate runs this per inbound
|
|
1042
|
+
* stream). Returns true when the peer must be admitted BEFORE any membership
|
|
1043
|
+
* source is consulted:
|
|
1044
|
+
* - the node is not fully up (`start()` in progress / DB torn down) — both
|
|
1045
|
+
* gates exist from libp2p bring-up, before the control DB does;
|
|
1046
|
+
* - the trusted-owner anchor is absent or empty — an un-enrolled node has
|
|
1047
|
+
* no basis to judge anyone and MUST accept its enrollment seed;
|
|
1048
|
+
* - the peer is one of the configured control bootstrap/relay nodes —
|
|
1049
|
+
* operator-configured infrastructure, not cadre members.
|
|
1050
|
+
* False only means "no unconditional admit": the caller then judges the peer
|
|
1051
|
+
* against its own membership source (a live DB read for the fail-open
|
|
1052
|
+
* connection gate; the materialized snapshot for the fail-closed stream gate).
|
|
1053
|
+
*/
|
|
1054
|
+
admitControlPeerUnconditionally(remotePeerId) {
|
|
1055
|
+
if (!this._running || !this.controlDatabase) {
|
|
1056
|
+
return true;
|
|
1057
|
+
}
|
|
1058
|
+
if (!this.trustedOwnerStore || this.trustedOwnerStore.all().size === 0) {
|
|
1059
|
+
return true;
|
|
1060
|
+
}
|
|
1061
|
+
return this.getBootstrapPeerIds().has(remotePeerId);
|
|
1062
|
+
}
|
|
1063
|
+
/**
|
|
1064
|
+
* Per-stream authorization for the four Optimystic control-DB protocols
|
|
1065
|
+
* (`/optimystic/control-<party>/{repo,cluster,sync,block-transfer}/…`),
|
|
1066
|
+
* wired as `authorizeInboundStream` in {@link createControlNode} — the
|
|
1067
|
+
* fail-closed layer behind the fail-open connection gater. Control node
|
|
1068
|
+
* ONLY: strand cohort nodes legitimately serve cross-party peers.
|
|
1069
|
+
*
|
|
1070
|
+
* The STRICT SUBSET of {@link admitInboundControlConnection}: the same
|
|
1071
|
+
* "no basis to judge" admissions (shared via
|
|
1072
|
+
* {@link admitControlPeerUnconditionally}), minus the stranger carve-outs
|
|
1073
|
+
* (enrollment window, outstanding open invitation, delegate admission
|
|
1074
|
+
* grant). The first two exist so a stranger can reach `/sereus/seed/1.0.0`
|
|
1075
|
+
* and `/sereus/formation/1.0.0` — neither is gated here, and admitting a
|
|
1076
|
+
* stranger to `repo` during an enrollment window is exactly the hole this
|
|
1077
|
+
* gate closes. A DELEGATE-admitted connection (a member's strand node
|
|
1078
|
+
* holding a circuit-relay reservation, see `delegate-admission.ts`) is
|
|
1079
|
+
* likewise exactly a case this gate must still refuse: the delegate gets
|
|
1080
|
+
* the connection and never the control DB.
|
|
1081
|
+
*
|
|
1082
|
+
* Admits when ANY of:
|
|
1083
|
+
* 1. a shared-baseline check admits (not running / no DB, empty anchor,
|
|
1084
|
+
* configured bootstrap infra);
|
|
1085
|
+
* 2. the materialized authorized set is empty — cold start, before the
|
|
1086
|
+
* rows that would authorize anyone have replicated in;
|
|
1087
|
+
* 3. the peer IS in the materialized authorized set.
|
|
1088
|
+
*
|
|
1089
|
+
* Deliberately SYNCHRONOUS and pure in-memory — see
|
|
1090
|
+
* {@link authorizedControlPeers} for why a control-DB read here would
|
|
1091
|
+
* deadlock into mutual denial. With a sync predicate the upstream deadline
|
|
1092
|
+
* (`authorizeInboundStreamTimeoutMs`) never trips, so it stays at its
|
|
1093
|
+
* default.
|
|
1094
|
+
*
|
|
1095
|
+
* NOTE: the snapshot keys on PEER ID, so a node with no persistent identity
|
|
1096
|
+
* key never lands a `CadrePeer` row ({@link registerSelf} skips) and its
|
|
1097
|
+
* siblings deny its control-DB streams once their own snapshot is non-empty —
|
|
1098
|
+
* owner status is a KEY, not a peer id, so being the owner does not exempt it.
|
|
1099
|
+
* Harmless today (a real deployment persists its identity; the ephemeral-owner
|
|
1100
|
+
* case appears only in tests, which configure the owner as bootstrap infra —
|
|
1101
|
+
* `push-wake-e2e.integration.ts` design note 3). If an ephemeral-identity node
|
|
1102
|
+
* ever becomes a supported deployment, this gate needs a key-based admission.
|
|
1103
|
+
*/
|
|
1104
|
+
authorizeInboundControlStream(remotePeerId, protocol) {
|
|
1105
|
+
if (this.admitControlPeerUnconditionally(remotePeerId)) {
|
|
1106
|
+
return true;
|
|
1107
|
+
}
|
|
1108
|
+
if (this.authorizedControlPeers.size === 0) {
|
|
1109
|
+
return true;
|
|
1110
|
+
}
|
|
1111
|
+
if (this.authorizedControlPeers.has(remotePeerId)) {
|
|
1112
|
+
return true;
|
|
1113
|
+
}
|
|
1114
|
+
// Upstream logs its own denial line; this one carries the WHY.
|
|
1115
|
+
log('authorizeInboundControlStream: DENYING %s on %s — not in the materialized authorized set (%d member(s))', remotePeerId, protocol, this.authorizedControlPeers.size);
|
|
1116
|
+
return false;
|
|
1117
|
+
}
|
|
1118
|
+
/**
|
|
1119
|
+
* Refresh {@link authorizedControlPeers} from the control DB, best-effort: a
|
|
1120
|
+
* failed read keeps the previous snapshot (never clears it), so a transient
|
|
1121
|
+
* DB error can neither flip the stream gate's cold-start carve-out back open
|
|
1122
|
+
* nor drop a legitimate member mid-flight. Never rejects.
|
|
1123
|
+
*
|
|
1124
|
+
* The SOLE caller is {@link drainMembershipGate}, which serializes refreshes —
|
|
1125
|
+
* so two reads can no longer settle out of order.
|
|
1126
|
+
*/
|
|
1127
|
+
async refreshAuthorizedControlPeers(reason) {
|
|
1128
|
+
if (!this._running || !this.controlDatabase) {
|
|
1129
|
+
return;
|
|
1130
|
+
}
|
|
1131
|
+
try {
|
|
1132
|
+
const members = await this.listAuthorizedMembers();
|
|
1133
|
+
this.authorizedControlPeers = new Set(members.map((m) => m.peerId));
|
|
1134
|
+
log('refreshAuthorizedControlPeers(%s): %d authorized peer(s)', reason, this.authorizedControlPeers.size);
|
|
1135
|
+
}
|
|
1136
|
+
catch (error) {
|
|
1137
|
+
log('refreshAuthorizedControlPeers(%s) failed — keeping previous snapshot: %o', reason, error);
|
|
1138
|
+
}
|
|
1139
|
+
}
|
|
1140
|
+
/**
|
|
1141
|
+
* PeerIds of the configured control bootstrap nodes (relay/bootstrap
|
|
1142
|
+
* infrastructure — always admitted inbound). Parsed once, lazily; an
|
|
1143
|
+
* address without a `/p2p/<id>` component contributes nothing.
|
|
1144
|
+
*/
|
|
1145
|
+
getBootstrapPeerIds() {
|
|
1146
|
+
if (!this.bootstrapPeerIds) {
|
|
1147
|
+
const ids = new Set();
|
|
1148
|
+
for (const addr of this.config.controlNetwork.bootstrapNodes) {
|
|
1149
|
+
try {
|
|
1150
|
+
// Last `/p2p/<id>` component: on a plain bootstrap addr the node's own
|
|
1151
|
+
// id; on a circuit addr the dial target (which is what gets admitted).
|
|
1152
|
+
let id;
|
|
1153
|
+
for (const component of multiaddr(addr).getComponents()) {
|
|
1154
|
+
if (component.name === 'p2p' && component.value) {
|
|
1155
|
+
id = component.value;
|
|
1156
|
+
}
|
|
1157
|
+
}
|
|
1158
|
+
if (id) {
|
|
1159
|
+
ids.add(id);
|
|
1160
|
+
}
|
|
1161
|
+
}
|
|
1162
|
+
catch (error) {
|
|
1163
|
+
log('getBootstrapPeerIds: skipping unparsable bootstrap addr %s: %o', addr, error);
|
|
1164
|
+
}
|
|
1165
|
+
}
|
|
1166
|
+
this.bootstrapPeerIds = ids;
|
|
1167
|
+
}
|
|
1168
|
+
return this.bootstrapPeerIds;
|
|
512
1169
|
}
|
|
513
1170
|
createStrandQueryable() {
|
|
514
1171
|
return {
|
|
@@ -558,7 +1215,11 @@ export class CadreNode {
|
|
|
558
1215
|
* PeerId (the resolved node identity from `keyStore`/`config.privateKey`).
|
|
559
1216
|
* - If the row already exists: a self-signed UPDATE bumping `UpdatedAt`.
|
|
560
1217
|
* - If not, and the node is its own owner: an owner-signed INSERT that
|
|
561
|
-
* also carries the self-signature.
|
|
1218
|
+
* also carries the self-signature. That INSERT is idempotent, so if an owner
|
|
1219
|
+
* {@link authorizePeer} of this node's own id seated the row inside the
|
|
1220
|
+
* read-then-insert window it no-ops — the publish then falls through to the
|
|
1221
|
+
* UPDATE path (re-reading and re-signing against the row that landed) rather
|
|
1222
|
+
* than leaving the authorize's null `Sig` in place, and reports `refreshed`.
|
|
562
1223
|
* - Otherwise: logs and returns (a non-owner node with no row yet must
|
|
563
1224
|
* wait for an owner to insert it; it can then self-refresh).
|
|
564
1225
|
*
|
|
@@ -599,27 +1260,70 @@ export class CadreNode {
|
|
|
599
1260
|
const peerId = this.controlNode.peerId.toString();
|
|
600
1261
|
const addrs = await this.collectSelfAddrs();
|
|
601
1262
|
const existing = await this.controlDatabase.queryPeerRecord(peerId);
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
}
|
|
612
|
-
if (this.seedBootstrapService) {
|
|
1263
|
+
if (!existing) {
|
|
1264
|
+
if (!this.seedBootstrapService) {
|
|
1265
|
+
// "No row" also covers "revoked": queryPeerRecord reads a row whose StampId is
|
|
1266
|
+
// retired in Revocation as absent, so a removed node lands here every heartbeat
|
|
1267
|
+
// and stops refreshing its own record — correct, but say so, or the message reads
|
|
1268
|
+
// as "never added" for a node that was removed.
|
|
1269
|
+
log('registerSelf: no readable CadrePeer row for self (never added, or removed and its stamp retired) and no owner service to self-insert; skipping');
|
|
1270
|
+
return 'skipped';
|
|
1271
|
+
}
|
|
613
1272
|
// First-time row: requires an owner signature (the node is its own
|
|
614
1273
|
// owner). insertSelfPeerRecord throws if no owner key is present.
|
|
615
|
-
|
|
616
|
-
if (this.
|
|
617
|
-
this.
|
|
618
|
-
|
|
619
|
-
|
|
1274
|
+
const record = this.signSelfRecord(peerId, signingKey, addrs, null);
|
|
1275
|
+
if (await this.seedBootstrapService.insertSelfPeerRecord(record)) {
|
|
1276
|
+
if (this.committedAlone())
|
|
1277
|
+
this.pendingSelfPeerWrite = true;
|
|
1278
|
+
log('registerSelf: inserted own CadrePeer record (owner-signed, updatedAt=%d, %d addrs, sig=%s…)', record.updatedAt, addrs.length, record.sig.slice(0, 16));
|
|
1279
|
+
return 'inserted';
|
|
1280
|
+
}
|
|
1281
|
+
// An owner authorize of this node's OWN peer id seated the row (null `Sig`)
|
|
1282
|
+
// inside the read-then-insert window, so the idempotent insert no-op'd. Fall
|
|
1283
|
+
// through to the self-update path, which re-signs against the row that landed.
|
|
1284
|
+
//
|
|
1285
|
+
// NOTE: this path reports 'refreshed' — honest about the write, which really was
|
|
1286
|
+
// an UPDATE, though the caller's intent was a first publish. No caller branches on
|
|
1287
|
+
// 'inserted' today (the CLI only logs it); if one ever needs "this was the row's
|
|
1288
|
+
// first publish", add a fourth SelfRegistrationOutcome rather than re-labelling.
|
|
1289
|
+
log('registerSelf: own CadrePeer row appeared mid-publish (concurrent authorize); self-updating to carry the signature');
|
|
1290
|
+
}
|
|
1291
|
+
// Reached on the fall-through above, where the pre-race read missed the row a
|
|
1292
|
+
// concurrent authorize then seated — so re-read to get that row's `UpdatedAt`, which
|
|
1293
|
+
// the strictly-greater self-update rule is measured against. The re-read can still
|
|
1294
|
+
// come back null (see the branch below).
|
|
1295
|
+
//
|
|
1296
|
+
// NOTE: when `existing` is set it predates this publish, so a removePeer(self) racing
|
|
1297
|
+
// in would leave the UPDATE below matching no rows while still reporting 'refreshed'.
|
|
1298
|
+
// Closing that needs updateSelfPeerRecord to report rows-affected — only worth doing
|
|
1299
|
+
// if removing self ever becomes something that happens concurrently in practice.
|
|
1300
|
+
const current = existing ?? await this.controlDatabase.queryPeerRecord(peerId);
|
|
1301
|
+
if (!current) {
|
|
1302
|
+
// Two causes, and the second is a steady state rather than a race: the row really
|
|
1303
|
+
// vanished mid-publish (next refresh re-inserts), or this node was REVOKED — its row
|
|
1304
|
+
// is physically present, so insertCadrePeer's in-lock existence check no-op'd above,
|
|
1305
|
+
// but queryPeerRecord reads it as absent because its stamp is retired. The revoked
|
|
1306
|
+
// case repeats every heartbeat and never re-inserts; that is intended (a revoked node
|
|
1307
|
+
// has nothing to publish), so this stays a skip, not an error.
|
|
1308
|
+
log('registerSelf: own CadrePeer row not readable (vanished mid-publish, or this node is revoked and its stamp retired); skipping');
|
|
1309
|
+
return 'skipped';
|
|
620
1310
|
}
|
|
621
|
-
|
|
622
|
-
|
|
1311
|
+
const record = this.signSelfRecord(peerId, signingKey, addrs, current);
|
|
1312
|
+
await this.controlDatabase.updateSelfPeerRecord(record);
|
|
1313
|
+
if (this.committedAlone())
|
|
1314
|
+
this.pendingSelfPeerWrite = true;
|
|
1315
|
+
log('registerSelf: refreshed own CadrePeer record (updatedAt=%d, %d addrs, sig=%s…)', record.updatedAt, addrs.length, record.sig.slice(0, 16));
|
|
1316
|
+
return 'refreshed';
|
|
1317
|
+
}
|
|
1318
|
+
/**
|
|
1319
|
+
* Sign this node's address record for publication, stamped strictly later than
|
|
1320
|
+
* `existing` (the row it is about to replace, or null for a first insert) so a
|
|
1321
|
+
* same-millisecond re-publish still satisfies the monotonic `UpdatedAt` rule the
|
|
1322
|
+
* `CadrePeer.AuthorizedUpdate` self-branch enforces.
|
|
1323
|
+
*/
|
|
1324
|
+
signSelfRecord(peerId, signingKey, addrs, existing) {
|
|
1325
|
+
const updatedAt = Math.max(Date.now(), (existing?.updatedAt ?? 0) + 1);
|
|
1326
|
+
return signPeerRecord({ peerId, publicKey: signingKey.publicKeyB64, addrs, updatedAt }, signingKey.privateKeyB64);
|
|
623
1327
|
}
|
|
624
1328
|
/**
|
|
625
1329
|
* The ed25519 keypair (base64url) the node signs its own record with — the key
|
|
@@ -751,13 +1455,17 @@ export class CadreNode {
|
|
|
751
1455
|
this.turnConnectionOpenHandler = null;
|
|
752
1456
|
this.turnConnectionCloseHandler = null;
|
|
753
1457
|
// Reset the write-while-alone re-replication state so a stop()→start() cycle
|
|
754
|
-
// re-arms the growth edge and re-runs
|
|
755
|
-
//
|
|
756
|
-
//
|
|
757
|
-
//
|
|
1458
|
+
// re-arms the growth edge and re-runs BOTH one-shot passes. Any queued
|
|
1459
|
+
// local-only writes are moot once the node is torn down: owner inserts are
|
|
1460
|
+
// re-covered by the next start's reconstruction, and delete tombstones by its
|
|
1461
|
+
// first-growth revocation sweep — which is why the sweep flag must reset too,
|
|
1462
|
+
// or the second lifetime would skip the sweep and strand exactly the
|
|
1463
|
+
// committed-alone tombstones it exists to carry.
|
|
758
1464
|
this.hasControlConnection = false;
|
|
759
1465
|
this.reconstructedLocalOnlyWrites = false;
|
|
1466
|
+
this.reissuedHeldRevocations = false;
|
|
760
1467
|
this.pendingPeerWrites.clear();
|
|
1468
|
+
this.pendingRevocations.clear();
|
|
761
1469
|
this.pendingSelfPeerWrite = false;
|
|
762
1470
|
this.pendingSelfDeviceWrite = false;
|
|
763
1471
|
}
|
|
@@ -795,7 +1503,10 @@ export class CadreNode {
|
|
|
795
1503
|
}
|
|
796
1504
|
// Self-signature over (peerId, addrs, updatedAt).
|
|
797
1505
|
if (!verifyPeerRecordSignature(record)) {
|
|
798
|
-
|
|
1506
|
+
// Field detail distinguishes "holding the owner-vouch revision (Sig null,
|
|
1507
|
+
// no addrs — a not-yet-self-published row)" from a genuinely corrupt or
|
|
1508
|
+
// mixed-revision row. Sig/addrs are public replicated data.
|
|
1509
|
+
log('resolvePeerAddrs: signature verification failed for %s (updatedAt=%d, addrs=%o, sig=%s)', peerId, record.updatedAt, record.addrs, record.sig ? `${record.sig.slice(0, 16)}…` : '(empty)');
|
|
799
1510
|
return [];
|
|
800
1511
|
}
|
|
801
1512
|
// Freshness: never hand back a dead relay reservation.
|
|
@@ -884,15 +1595,91 @@ export class CadreNode {
|
|
|
884
1595
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
885
1596
|
return;
|
|
886
1597
|
}
|
|
1598
|
+
// Ride this pass's cadence to refresh the per-stream gate's materialized
|
|
1599
|
+
// authorized set — membership changes that ARRIVED BY REPLICATION (rather
|
|
1600
|
+
// than a local write) are picked up here, bounding the snapshot's staleness
|
|
1601
|
+
// to the reconcile interval.
|
|
1602
|
+
// NOTE: this refresh and the sibling enumeration below each run their own
|
|
1603
|
+
// CadrePeer query (two reads per pass), plus a third from
|
|
1604
|
+
// `refreshStrandPeerAddrs` on the passes where a strand is due AND this node
|
|
1605
|
+
// holds a control connection (one read for the whole pass, not one per
|
|
1606
|
+
// strand); if those reads ever get costly, share one row-set across all
|
|
1607
|
+
// three.
|
|
1608
|
+
await this.refreshMembershipGate('reconcile');
|
|
1609
|
+
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
1610
|
+
return;
|
|
1611
|
+
}
|
|
1612
|
+
// Refresh relay delegate grants BEFORE the sibling enumeration below: a
|
|
1613
|
+
// solo cadre with a party relay has no siblings to dial but must still keep
|
|
1614
|
+
// its running strands' grants alive (a dropped relay connection re-dials
|
|
1615
|
+
// the reservation and faces the connection gate again).
|
|
1616
|
+
await this.refreshDelegateGrants();
|
|
1617
|
+
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
1618
|
+
return;
|
|
1619
|
+
}
|
|
1620
|
+
// Then re-warm each running strand's own address book from its siblings, on
|
|
1621
|
+
// its own (much longer) throttle. Same reasoning as warmSiblingAddrBook
|
|
1622
|
+
// below, one layer down: replication runs on the STRAND network, and every
|
|
1623
|
+
// layer under cadre-core dials strand peers by bare peer id.
|
|
1624
|
+
await this.refreshStrandPeerAddrs();
|
|
1625
|
+
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
1626
|
+
return;
|
|
1627
|
+
}
|
|
887
1628
|
const selfPeerId = this.controlNode.peerId.toString();
|
|
1629
|
+
// Reap guarded rows this node still holds for incarnations an already-committed
|
|
1630
|
+
// Revocation tombstone retires — the half of a removal replication cannot carry
|
|
1631
|
+
// (a delete of an already-absent row replays as nothing). Connected-only: a reap
|
|
1632
|
+
// is a write, and a write committed alone is local-only and forks this node's own
|
|
1633
|
+
// history, which is the condition this work exists to stop creating
|
|
1634
|
+
// (tickets/blocked/forked-control-collection-sync-livelocks.md). Nothing is urgent
|
|
1635
|
+
// here — the stale row already reads as absent everywhere — so waiting for
|
|
1636
|
+
// connectivity costs nothing and keeps the reap fork-CLOSING rather than
|
|
1637
|
+
// fork-creating.
|
|
1638
|
+
//
|
|
1639
|
+
// BEFORE step 1, not after: the pass early-returns when it finds no siblings, and a
|
|
1640
|
+
// cadre whose only sibling row is the tombstoned one lands in exactly that branch
|
|
1641
|
+
// (listMembers() filters retired stamps, so the row needing the reap is invisible to
|
|
1642
|
+
// the sibling count). Placed after step 1 it would never run for the smallest and
|
|
1643
|
+
// most likely case.
|
|
1644
|
+
//
|
|
1645
|
+
// Shares no mutable state with drainPendingRevocations, which re-issues tombstones
|
|
1646
|
+
// while this consumes them: different tables (Revocation update vs guarded-row
|
|
1647
|
+
// delete) and both take the write lock per statement. The two can overlap freely —
|
|
1648
|
+
// do not add a mutex.
|
|
1649
|
+
//
|
|
1650
|
+
// NOTE: the gate is sampled ONCE for the whole sweep, so a disconnect landing
|
|
1651
|
+
// mid-sweep lets the remaining rows commit alone — the fork this gate exists to
|
|
1652
|
+
// avoid. Bounded by how long a sweep runs, which is bounded by the backlog of
|
|
1653
|
+
// unreaped rows (0 in steady state). If that backlog can ever be large, move the
|
|
1654
|
+
// check inside reapRevokedRows' loop via an injected predicate.
|
|
1655
|
+
if (this.getControlConnectionCount() > 0) {
|
|
1656
|
+
try {
|
|
1657
|
+
const reaped = await this.controlDatabase.reapRevokedRows(selfPeerId);
|
|
1658
|
+
if (reaped > 0) {
|
|
1659
|
+
log('reconcileControlCohort: reaped %d revoked control row(s)', reaped);
|
|
1660
|
+
}
|
|
1661
|
+
}
|
|
1662
|
+
catch (error) {
|
|
1663
|
+
// Best-effort like every other step in this pass: reconnecting siblings
|
|
1664
|
+
// outranks garbage collection, so a reap failure never aborts the reconcile.
|
|
1665
|
+
log('reconcileControlCohort: reap pass failed (continuing): %o', error);
|
|
1666
|
+
}
|
|
1667
|
+
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
1668
|
+
return;
|
|
1669
|
+
}
|
|
1670
|
+
}
|
|
888
1671
|
// 1. Enumerate known siblings. This membership read is itself a pull-on-read
|
|
889
1672
|
// that helps the CadrePeer table converge — a reader-only node converges
|
|
890
1673
|
// purely by these reads.
|
|
891
1674
|
const members = await this.listMembers();
|
|
892
1675
|
const siblings = members.filter((m) => m.peerId !== selfPeerId);
|
|
893
1676
|
if (siblings.length === 0) {
|
|
894
|
-
//
|
|
895
|
-
// dial
|
|
1677
|
+
// No rows to dial from. Either a solo cadre (nothing to do) or a COLD
|
|
1678
|
+
// START whose seed dial never landed — and those look identical from here,
|
|
1679
|
+
// because filling this table needs a connection and getting a connection
|
|
1680
|
+
// needs this table. Fall back to the seed's bootstrap addresses; a solo
|
|
1681
|
+
// cadre has none and this is a no-op. Must not throw or busy-loop.
|
|
1682
|
+
await this.dialColdStartBootstrap();
|
|
896
1683
|
return;
|
|
897
1684
|
}
|
|
898
1685
|
// Re-guard after the await: a stop() may have raced the membership read.
|
|
@@ -900,6 +1687,11 @@ export class CadreNode {
|
|
|
900
1687
|
return;
|
|
901
1688
|
}
|
|
902
1689
|
// 2. Classify backbone (owner) members and select a bounded dial set.
|
|
1690
|
+
// NOTE: deliberately the REPLICATED table, not the node-local anchor — this
|
|
1691
|
+
// only *prefers* owner peers as dial targets, so pollution costs a wasted
|
|
1692
|
+
// dial while anchoring would drop legitimate co-owners this node never
|
|
1693
|
+
// pinned. Same call as `SeedPeer.isOwner` in seed-bootstrap.ts. Move it to
|
|
1694
|
+
// the anchor if owner status here ever gates something trusted.
|
|
903
1695
|
const ownerKeys = await this.controlDatabase.getOwnerKeys();
|
|
904
1696
|
if (!this._running || !this.controlNode) {
|
|
905
1697
|
return;
|
|
@@ -911,9 +1703,16 @@ export class CadreNode {
|
|
|
911
1703
|
// Don't silently bound coverage — surface what the out-degree cap dropped.
|
|
912
1704
|
log('reconcileControlCohort: capped %d non-owner sibling(s) at targetDegree=%d', cappedNonOwner, targetDegree);
|
|
913
1705
|
}
|
|
914
|
-
// 3.
|
|
1706
|
+
// 3. Warm the libp2p address book with EVERY sibling's verified addresses,
|
|
1707
|
+
// keeping what resolved for the dial loop below.
|
|
1708
|
+
const resolved = await this.warmSiblingAddrBook(siblings);
|
|
1709
|
+
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
1710
|
+
return;
|
|
1711
|
+
}
|
|
1712
|
+
// 4. Skip already-connected peers (no re-dial / churn for live connections).
|
|
915
1713
|
const connected = new Set(this.controlNode.getConnections().map((c) => c.remotePeer.toString()));
|
|
916
|
-
//
|
|
1714
|
+
// 5. Dial each selected, not-yet-connected sibling, best-effort, from the
|
|
1715
|
+
// addresses step 3 already resolved for it.
|
|
917
1716
|
let dialed = 0;
|
|
918
1717
|
for (const sibling of dials) {
|
|
919
1718
|
if (!this._running || !this.controlNode) {
|
|
@@ -922,34 +1721,109 @@ export class CadreNode {
|
|
|
922
1721
|
if (connected.has(sibling.peerId)) {
|
|
923
1722
|
continue;
|
|
924
1723
|
}
|
|
925
|
-
if (await this.dialControlSibling(sibling)) {
|
|
1724
|
+
if (await this.dialControlSibling(sibling, resolved.get(sibling.peerId) ?? [])) {
|
|
926
1725
|
dialed++;
|
|
927
1726
|
}
|
|
928
1727
|
}
|
|
929
1728
|
log('reconcileControlCohort: pass complete (siblings=%d, selected=%d, dialed=%d)', siblings.length, dials.length, dialed);
|
|
930
1729
|
}
|
|
931
1730
|
/**
|
|
932
|
-
* Resolve
|
|
933
|
-
*
|
|
1731
|
+
* Resolve every sibling's signed address record once and merge what resolves
|
|
1732
|
+
* into the control node's libp2p **address book** (peerStore). Returns the
|
|
1733
|
+
* resolved addresses per sibling so the dial loop reuses them rather than
|
|
1734
|
+
* re-resolving — one `queryPeerRecord` per sibling per pass, not two.
|
|
1735
|
+
*
|
|
1736
|
+
* Deliberately every sibling, not the {@link selectControlCohortDials} subset,
|
|
1737
|
+
* and deliberately including already-connected ones: everything below
|
|
1738
|
+
* cadre-core dials by bare peer id (Optimystic's cluster/repo clients, FRET
|
|
1739
|
+
* ping/announce), so the address book has to be warm BEFORE a live connection
|
|
1740
|
+
* drops, and a sibling the out-degree cap declines to dial is still one those
|
|
1741
|
+
* layers may need to reach.
|
|
1742
|
+
*
|
|
1743
|
+
* Only `resolvePeerAddrs` output is merged — never the cold-start
|
|
1744
|
+
* `peerStoreAddrs` fallback, which came out of the address book to begin with
|
|
1745
|
+
* and would restamp unverified seed addresses indefinitely. A sibling that
|
|
1746
|
+
* resolves to nothing (revoked, stale, untrusted) is not written at all, so its
|
|
1747
|
+
* existing entry ages out on its own.
|
|
1748
|
+
*
|
|
1749
|
+
* NOTE: this resolves EVERY sibling serially before the dial loop below runs,
|
|
1750
|
+
* so it costs one record query per sibling per reconcile pass (~15s) and each
|
|
1751
|
+
* one delays the pass's first dial. A cadre is a handful of devices, so today
|
|
1752
|
+
* that is a few extra local reads; if cadres ever grow large, batch the records
|
|
1753
|
+
* into one query, merge only on change, or move the warm pass after the dials.
|
|
1754
|
+
*/
|
|
1755
|
+
async warmSiblingAddrBook(siblings) {
|
|
1756
|
+
const resolved = new Map();
|
|
1757
|
+
const counts = { merged: 0, restamped: 0, skipped: 0, failed: 0 };
|
|
1758
|
+
for (const sibling of siblings) {
|
|
1759
|
+
if (!this._running || !this.controlNode) {
|
|
1760
|
+
break;
|
|
1761
|
+
}
|
|
1762
|
+
const addrs = await this.resolveSiblingAddrs(sibling.peerId);
|
|
1763
|
+
resolved.set(sibling.peerId, addrs);
|
|
1764
|
+
// Re-guarded after the resolve await as well as before it: the write is
|
|
1765
|
+
// the half that matters, and a torn-down node must never be written to.
|
|
1766
|
+
const controlNode = this.controlNode;
|
|
1767
|
+
if (!this._running || !controlNode) {
|
|
1768
|
+
break;
|
|
1769
|
+
}
|
|
1770
|
+
// `mergePeerAddrs` owns the whole best-effort contract — including parsing
|
|
1771
|
+
// a malformed `CadrePeer.PeerId` — so one bad row cannot abort the pass.
|
|
1772
|
+
counts[await mergePeerAddrs(controlNode, sibling.peerId, addrs)]++;
|
|
1773
|
+
}
|
|
1774
|
+
log('reconcileControlCohort: address book warmed (siblings=%d, merged=%d, restamped=%d, skipped=%d, failed=%d)', resolved.size, counts.merged, counts.restamped, counts.skipped, counts.failed);
|
|
1775
|
+
return resolved;
|
|
1776
|
+
}
|
|
1777
|
+
/**
|
|
1778
|
+
* {@link resolvePeerAddrs} for one sibling, best-effort: a control-DB read
|
|
1779
|
+
* failure yields `[]` (and the cold-start fallback then gets its turn) rather
|
|
1780
|
+
* than aborting the whole pass, like every other step here.
|
|
1781
|
+
*/
|
|
1782
|
+
async resolveSiblingAddrs(peerId) {
|
|
1783
|
+
try {
|
|
1784
|
+
return await this.resolvePeerAddrs(peerId);
|
|
1785
|
+
}
|
|
1786
|
+
catch (error) {
|
|
1787
|
+
log('reconcileControlCohort: resolving addrs for %s failed (continuing): %o', peerId, error);
|
|
1788
|
+
return [];
|
|
1789
|
+
}
|
|
1790
|
+
}
|
|
1791
|
+
/**
|
|
1792
|
+
* Budget for ONE proactive dial of the reconcile pass — steady-state sibling
|
|
1793
|
+
* ({@link dialControlSibling}) and cold-start bootstrap peer
|
|
1794
|
+
* ({@link dialBootstrapPeer}) alike, since both feed a whole address list to
|
|
1795
|
+
* one `dial()` and both run in a sequential loop that the next entry waits on.
|
|
934
1796
|
*
|
|
935
|
-
*
|
|
936
|
-
*
|
|
937
|
-
* exactly like {@link SeedBootstrapService.applySeed}'s owner-dial loop. A
|
|
938
|
-
* failed dial is simply retried on the next pass.
|
|
1797
|
+
* See {@link DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS} for why the pass bounds
|
|
1798
|
+
* this itself rather than inheriting each address's libp2p attempt timeout.
|
|
939
1799
|
*/
|
|
940
|
-
|
|
1800
|
+
controlDialTimeoutMs() {
|
|
1801
|
+
return this.config.network?.controlCohort?.dialTimeoutMs
|
|
1802
|
+
?? DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS;
|
|
1803
|
+
}
|
|
1804
|
+
/**
|
|
1805
|
+
* Dial one sibling from its already-resolved addresses, best-effort. Returns
|
|
1806
|
+
* whether a dial was attempted (false when no address resolves).
|
|
1807
|
+
*
|
|
1808
|
+
* A per-peer failure (NAT, offline, relay down, connection-gater denial, or
|
|
1809
|
+
* the {@link controlDialTimeoutMs} budget expiring) is logged and swallowed so
|
|
1810
|
+
* one unreachable sibling never aborts the pass — exactly like
|
|
1811
|
+
* {@link SeedBootstrapService.applySeed}'s owner-dial loop. A failed dial is
|
|
1812
|
+
* simply retried on the next pass.
|
|
1813
|
+
*/
|
|
1814
|
+
async dialControlSibling(sibling, resolved) {
|
|
941
1815
|
const controlNode = this.controlNode;
|
|
942
1816
|
if (!controlNode) {
|
|
943
1817
|
return false;
|
|
944
1818
|
}
|
|
945
|
-
const addrs = await this.resolveControlDialAddrs(sibling.peerId);
|
|
1819
|
+
const addrs = await this.resolveControlDialAddrs(sibling.peerId, resolved);
|
|
946
1820
|
if (addrs.length === 0) {
|
|
947
1821
|
log('reconcileControlCohort: no dialable control address for sibling %s; skipping', sibling.peerId);
|
|
948
1822
|
return false;
|
|
949
1823
|
}
|
|
950
1824
|
try {
|
|
951
1825
|
log('reconcileControlCohort: dialing sibling %s (%d addr(s))', sibling.peerId, addrs.length);
|
|
952
|
-
await controlNode.dial(addrs);
|
|
1826
|
+
await withDeadline(this.controlDialTimeoutMs(), `reconcileControlCohort dial of sibling ${sibling.peerId}`, (signal) => controlNode.dial(addrs, { signal }));
|
|
953
1827
|
return true;
|
|
954
1828
|
}
|
|
955
1829
|
catch (error) {
|
|
@@ -958,16 +1832,16 @@ export class CadreNode {
|
|
|
958
1832
|
}
|
|
959
1833
|
}
|
|
960
1834
|
/**
|
|
961
|
-
*
|
|
1835
|
+
* A sibling's control-network dial addresses for the reconcile pass.
|
|
962
1836
|
*
|
|
963
|
-
* Primary (steady state): the signed, fresh, trust-gated control addresses
|
|
964
|
-
*
|
|
965
|
-
*
|
|
966
|
-
*
|
|
967
|
-
*
|
|
1837
|
+
* Primary (steady state): the signed, fresh, trust-gated control addresses
|
|
1838
|
+
* `resolved` for it by {@link warmSiblingAddrBook} — passed in rather than
|
|
1839
|
+
* re-resolved, so the pass makes one record query per sibling. Fallback (cold
|
|
1840
|
+
* start): the libp2p peerStore entries `applySeed` populated, used only while
|
|
1841
|
+
* the record is not yet resolvable. Returns `[]` (never throws) when neither
|
|
1842
|
+
* yields an address — that sibling is skipped this pass.
|
|
968
1843
|
*/
|
|
969
|
-
async resolveControlDialAddrs(peerId) {
|
|
970
|
-
const resolved = await this.resolvePeerAddrs(peerId);
|
|
1844
|
+
async resolveControlDialAddrs(peerId, resolved) {
|
|
971
1845
|
if (resolved.length > 0) {
|
|
972
1846
|
return resolved;
|
|
973
1847
|
}
|
|
@@ -993,6 +1867,149 @@ export class CadreNode {
|
|
|
993
1867
|
return [];
|
|
994
1868
|
}
|
|
995
1869
|
}
|
|
1870
|
+
/**
|
|
1871
|
+
* Retain a just-applied seed's owner-flagged peers as cold-start bootstrap
|
|
1872
|
+
* dial targets (see {@link bootstrapPeerStore}).
|
|
1873
|
+
*
|
|
1874
|
+
* Called for every seed this node accepts, on BOTH intake paths — the
|
|
1875
|
+
* {@link applySeed} wrapper (which may run on a throwaway service, so the
|
|
1876
|
+
* service itself is the wrong place to keep this) and the inbound
|
|
1877
|
+
* `/sereus/seed/1.0.0` handler via `onSeedApplied`. Peers with no address are
|
|
1878
|
+
* skipped: there is nothing to dial.
|
|
1879
|
+
*
|
|
1880
|
+
* `isOwner` is the seed's own claim about its peers, exactly as
|
|
1881
|
+
* `SeedBootstrapService.applySeed` uses it to choose its dial targets. That is
|
|
1882
|
+
* sound here for the same reason it is sound there — the whole seed is
|
|
1883
|
+
* signature-checked against a trust-anchored signer before this runs, and the
|
|
1884
|
+
* flag only *selects a dial target*; a dial grants no authority.
|
|
1885
|
+
*
|
|
1886
|
+
* Self is excluded. `createSeed` projects EVERY `CadrePeer` row, so an owner
|
|
1887
|
+
* that applies a seed minted after it joined finds itself in the owner list;
|
|
1888
|
+
* retaining that would make the cold-start pass dial this node forever (the
|
|
1889
|
+
* steady-state pass filters self out of its sibling list for the same reason).
|
|
1890
|
+
*/
|
|
1891
|
+
recordSeedBootstrapPeers(seed) {
|
|
1892
|
+
const store = this.bootstrapPeerStore;
|
|
1893
|
+
if (!store) {
|
|
1894
|
+
// Unreachable in production: start() builds the store before any network
|
|
1895
|
+
// bring-up, and both intake paths need a started node (an uninitialized
|
|
1896
|
+
// SeedBootstrapService rejects the seed, so noteAppliedSeed returns early).
|
|
1897
|
+
log('recordSeedBootstrapPeers: no bootstrap-peer store yet; retaining nothing');
|
|
1898
|
+
return;
|
|
1899
|
+
}
|
|
1900
|
+
const selfPeerId = this.controlNode?.peerId.toString();
|
|
1901
|
+
for (const peer of seed.peers) {
|
|
1902
|
+
if (!peer.isOwner || peer.multiaddrs.length === 0 || peer.peerId === selfPeerId) {
|
|
1903
|
+
continue;
|
|
1904
|
+
}
|
|
1905
|
+
// Sync-visible by contract; the promise tracks durability only. A persist
|
|
1906
|
+
// failure costs restart survival, never this session's retry set — the same
|
|
1907
|
+
// trade `SeedBootstrapService.anchorAcceptedSigner` makes — so log and carry
|
|
1908
|
+
// on rather than failing the seed that was already accepted.
|
|
1909
|
+
void store.record(peer.peerId, peer.multiaddrs).catch((error) => {
|
|
1910
|
+
log('recordSeedBootstrapPeers: persisting bootstrap peer %s failed (retained in memory): %o', peer.peerId, error);
|
|
1911
|
+
});
|
|
1912
|
+
}
|
|
1913
|
+
}
|
|
1914
|
+
/**
|
|
1915
|
+
* Cold-start branch of the reconcile pass: re-dial the seed's owner peers
|
|
1916
|
+
* while the control database still has no siblings to dial.
|
|
1917
|
+
*
|
|
1918
|
+
* `SeedBootstrapService.applySeed` dials those owners exactly ONCE, best-effort.
|
|
1919
|
+
* When that single dial fails — owner momentarily down, relay reservation not
|
|
1920
|
+
* up yet, NAT traversal lost the race — the joining node has an empty
|
|
1921
|
+
* `CadrePeer` table and no connection, and the steady-state pass above cannot
|
|
1922
|
+
* help: it dials only siblings enumerated from that very table. Retrying here
|
|
1923
|
+
* is the only way back in.
|
|
1924
|
+
*
|
|
1925
|
+
* Unbounded, at the reconcile cadence, with no backoff: the branch is already
|
|
1926
|
+
* gated by "the control database has no siblings", so it stops the moment the
|
|
1927
|
+
* node is actually in the party, and a node that is stranded MUST keep trying —
|
|
1928
|
+
* a give-up rule would turn a transient outage into a permanent one. The steady
|
|
1929
|
+
* cost is one dial per bootstrap peer per pass (seeds nominate one or a few
|
|
1930
|
+
* owners), each failing fast against an unreachable address.
|
|
1931
|
+
* NOTE: if seeds ever carry many owner peers, or the reconcile cadence tightens
|
|
1932
|
+
* well below its 15 s default, add per-peer backoff here.
|
|
1933
|
+
*
|
|
1934
|
+
* Best-effort per peer, exactly like {@link dialControlSibling}: one dead
|
|
1935
|
+
* address never aborts the pass.
|
|
1936
|
+
*/
|
|
1937
|
+
async dialColdStartBootstrap() {
|
|
1938
|
+
const controlNode = this.controlNode;
|
|
1939
|
+
// Snapshot up front: the store hands back a copy, so a seed applied mid-pass
|
|
1940
|
+
// cannot mutate what we are iterating.
|
|
1941
|
+
const targets = this.bootstrapPeerStore?.all();
|
|
1942
|
+
if (!controlNode || !targets || targets.size === 0) {
|
|
1943
|
+
return;
|
|
1944
|
+
}
|
|
1945
|
+
// Same skip rule as step 3 of the steady-state pass: no churn on live links.
|
|
1946
|
+
const connected = new Set(controlNode.getConnections().map((c) => c.remotePeer.toString()));
|
|
1947
|
+
let dialed = 0;
|
|
1948
|
+
for (const [peerId, entry] of targets) {
|
|
1949
|
+
if (!this._running || !this.controlNode) {
|
|
1950
|
+
return;
|
|
1951
|
+
}
|
|
1952
|
+
if (connected.has(peerId)) {
|
|
1953
|
+
continue;
|
|
1954
|
+
}
|
|
1955
|
+
if (await this.dialBootstrapPeer(peerId, entry.addrs)) {
|
|
1956
|
+
dialed++;
|
|
1957
|
+
}
|
|
1958
|
+
}
|
|
1959
|
+
log('reconcileControlCohort: cold-start pass complete (bootstrap=%d, connected=%d, dialed=%d)', targets.size, connected.size, dialed);
|
|
1960
|
+
}
|
|
1961
|
+
/**
|
|
1962
|
+
* Bind a bootstrap peer's seed addresses to its peer id, so the dial
|
|
1963
|
+
* authenticates the peer it is aiming at rather than trusting whoever answers.
|
|
1964
|
+
*
|
|
1965
|
+
* An address that already carries `/p2p/<id>` must carry THIS peer's id or it
|
|
1966
|
+
* is dropped (a seed that disagrees with itself is not a dial target); one
|
|
1967
|
+
* that carries none — an `addressOverrides`-published record, say — gets the
|
|
1968
|
+
* id encapsulated. Unencapsulatable addresses are dropped rather than dialed
|
|
1969
|
+
* bare.
|
|
1970
|
+
*/
|
|
1971
|
+
bootstrapDialAddrs(peerId, addrs) {
|
|
1972
|
+
const out = [];
|
|
1973
|
+
for (const addr of this.parseMultiaddrs(addrs)) {
|
|
1974
|
+
const embedded = trailingPeerId(addr);
|
|
1975
|
+
if (embedded === peerId) {
|
|
1976
|
+
out.push(addr);
|
|
1977
|
+
continue;
|
|
1978
|
+
}
|
|
1979
|
+
if (embedded !== null) {
|
|
1980
|
+
log('reconcileControlCohort(cold-start): addr %s names peer %s, not %s; dropping', addr.toString(), embedded, peerId);
|
|
1981
|
+
continue;
|
|
1982
|
+
}
|
|
1983
|
+
try {
|
|
1984
|
+
out.push(addr.encapsulate(`/p2p/${peerId}`));
|
|
1985
|
+
}
|
|
1986
|
+
catch (error) {
|
|
1987
|
+
log('reconcileControlCohort(cold-start): cannot bind %s to %s: %o', addr.toString(), peerId, error);
|
|
1988
|
+
}
|
|
1989
|
+
}
|
|
1990
|
+
return out;
|
|
1991
|
+
}
|
|
1992
|
+
/** Dial one cold-start bootstrap peer, best-effort. Returns whether it connected. */
|
|
1993
|
+
async dialBootstrapPeer(peerId, addrs) {
|
|
1994
|
+
const controlNode = this.controlNode;
|
|
1995
|
+
if (!controlNode) {
|
|
1996
|
+
return false;
|
|
1997
|
+
}
|
|
1998
|
+
const parsed = this.bootstrapDialAddrs(peerId, addrs);
|
|
1999
|
+
if (parsed.length === 0) {
|
|
2000
|
+
log('reconcileControlCohort(cold-start): no dialable address for bootstrap peer %s; skipping', peerId);
|
|
2001
|
+
return false;
|
|
2002
|
+
}
|
|
2003
|
+
try {
|
|
2004
|
+
log('reconcileControlCohort(cold-start): dialing bootstrap peer %s (%d addr(s))', peerId, parsed.length);
|
|
2005
|
+
await withDeadline(this.controlDialTimeoutMs(), `reconcileControlCohort cold-start dial of bootstrap peer ${peerId}`, (signal) => controlNode.dial(parsed, { signal }));
|
|
2006
|
+
return true;
|
|
2007
|
+
}
|
|
2008
|
+
catch (error) {
|
|
2009
|
+
log('reconcileControlCohort(cold-start): dial of bootstrap peer %s failed (continuing): %o', peerId, error);
|
|
2010
|
+
return false;
|
|
2011
|
+
}
|
|
2012
|
+
}
|
|
996
2013
|
// ============================================================================
|
|
997
2014
|
// Write-while-alone re-replication (control-write-ensure-replicated)
|
|
998
2015
|
//
|
|
@@ -1017,9 +2034,12 @@ export class CadreNode {
|
|
|
1017
2034
|
* is caught by the cohort's periodic pull-on-read instead. Re-issuing an already-
|
|
1018
2035
|
* replicated row is harmless (an idempotent monotonic bump), so the coarse proxy
|
|
1019
2036
|
* is acceptable as the agreed first cut.
|
|
2037
|
+
*
|
|
2038
|
+
* Reads {@link getControlConnectionCount} — the public, embedder-facing form of
|
|
2039
|
+
* the same sample — so the two cannot drift.
|
|
1020
2040
|
*/
|
|
1021
2041
|
committedAlone() {
|
|
1022
|
-
return
|
|
2042
|
+
return this.getControlConnectionCount() === 0;
|
|
1023
2043
|
}
|
|
1024
2044
|
/**
|
|
1025
2045
|
* Record (or clear) a just-committed owner membership write in the
|
|
@@ -1028,27 +2048,47 @@ export class CadreNode {
|
|
|
1028
2048
|
* cohort grows; otherwise the write replicated and any stale queue entry for this
|
|
1029
2049
|
* subject is dropped.
|
|
1030
2050
|
*
|
|
1031
|
-
* A `remove`
|
|
1032
|
-
*
|
|
1033
|
-
*
|
|
1034
|
-
*
|
|
2051
|
+
* A `remove` never queues here — a delete's re-replicable half is its
|
|
2052
|
+
* `Revocation` tombstone, which the committed-delete seam routes into
|
|
2053
|
+
* {@link pendingRevocations} ({@link noteGuardedDelete}). The `remove` arm only
|
|
2054
|
+
* drops any queued authorize for the subject (the row is gone; re-issuing the
|
|
2055
|
+
* insert would resurrect it) and, when the commit was alone, logs the
|
|
2056
|
+
* security-relevant window loudly.
|
|
1035
2057
|
*/
|
|
1036
2058
|
noteControlWrite(peerId, kind) {
|
|
1037
|
-
if (
|
|
2059
|
+
if (kind === 'remove') {
|
|
1038
2060
|
this.pendingPeerWrites.delete(peerId);
|
|
2061
|
+
if (this.committedAlone()) {
|
|
2062
|
+
log('removePeer(%s) committed while ALONE (0 control connections): the deletion is ' +
|
|
2063
|
+
'local-only, so a revoked peer may persist in the cohort until the queued ' +
|
|
2064
|
+
'Revocation tombstone is re-issued on cohort growth (drainPendingRevocations).', peerId);
|
|
2065
|
+
}
|
|
1039
2066
|
return;
|
|
1040
2067
|
}
|
|
1041
|
-
this.
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
'local-only, so a revoked peer may persist in the cohort until re-replication. The ' +
|
|
1045
|
-
'on-growth re-issue is best-effort (a physical delete cannot be replayed without a ' +
|
|
1046
|
-
'schema tombstone — see control-delete-while-alone-tombstone).', peerId);
|
|
2068
|
+
if (!this.committedAlone()) {
|
|
2069
|
+
this.pendingPeerWrites.delete(peerId);
|
|
2070
|
+
return;
|
|
1047
2071
|
}
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
2072
|
+
this.pendingPeerWrites.set(peerId, 'authorize');
|
|
2073
|
+
log('authorizePeer(%s) committed while alone (0 control connections); queued for ' +
|
|
2074
|
+
're-replication on cohort growth', peerId);
|
|
2075
|
+
}
|
|
2076
|
+
/**
|
|
2077
|
+
* Committed-delete seam handler ({@link GuardedDeleteListener}, wired in
|
|
2078
|
+
* {@link start}): a guarded-table delete and its `Revocation` tombstone
|
|
2079
|
+
* committed. If the node was alone the tombstone is local-only — queue it for
|
|
2080
|
+
* an owner-signed re-issue on cohort growth; otherwise it broadcast, so drop
|
|
2081
|
+
* any stale queue entry for the stamp. Synchronous by contract — bookkeeping
|
|
2082
|
+
* only, never throws into the delete path.
|
|
2083
|
+
*/
|
|
2084
|
+
noteGuardedDelete(revocation) {
|
|
2085
|
+
if (!this.committedAlone()) {
|
|
2086
|
+
this.pendingRevocations.delete(revocation.stampId);
|
|
2087
|
+
return;
|
|
1051
2088
|
}
|
|
2089
|
+
this.pendingRevocations.set(revocation.stampId, revocation);
|
|
2090
|
+
log('%s delete of %s committed while alone (0 control connections); Revocation tombstone ' +
|
|
2091
|
+
'(stamp %s) queued for re-issue on cohort growth', revocation.tableName, revocation.rowKey, revocation.stampId);
|
|
1052
2092
|
}
|
|
1053
2093
|
/**
|
|
1054
2094
|
* Drain the write-while-alone re-replication queue: re-issue the writes that
|
|
@@ -1079,8 +2119,19 @@ export class CadreNode {
|
|
|
1079
2119
|
return;
|
|
1080
2120
|
}
|
|
1081
2121
|
const firstGrowth = !this.reconstructedLocalOnlyWrites;
|
|
1082
|
-
log('Draining control re-replication (reason=%s, firstGrowth=%s, pendingPeers=%d, self=%s, deviceToken=%s)', reason, firstGrowth, this.pendingPeerWrites.size, this.pendingSelfPeerWrite, this.pendingSelfDeviceWrite);
|
|
1083
|
-
// 1.
|
|
2122
|
+
log('Draining control re-replication (reason=%s, firstGrowth=%s, pendingPeers=%d, pendingRevocations=%d, self=%s, deviceToken=%s)', reason, firstGrowth, this.pendingPeerWrites.size, this.pendingRevocations.size, this.pendingSelfPeerWrite, this.pendingSelfDeviceWrite);
|
|
2123
|
+
// 1. Revocation tombstones FIRST: a removal that never reached the cohort
|
|
2124
|
+
// leaves a revoked peer live elsewhere, which outranks every re-touch
|
|
2125
|
+
// below. Ordering is not cosmetic — every `CadrePeer` write here (self row
|
|
2126
|
+
// included) can burn tens of seconds in upstream retry livelock after a
|
|
2127
|
+
// collection fork, while the tombstone lives in a different collection
|
|
2128
|
+
// that is merely behind, not forked. No membership-gate refresh around
|
|
2129
|
+
// this step: a re-issue only bumps `ReissuedAt` — LOCAL membership
|
|
2130
|
+
// visibility changed at delete time (deleteCadrePeer already notified),
|
|
2131
|
+
// and receiving-node staleness is the pre-existing pull-on-read /
|
|
2132
|
+
// periodic-refresh property.
|
|
2133
|
+
await this.drainPendingRevocations();
|
|
2134
|
+
// 2. Self rows. On the first growth always re-touch (covers a self row carried
|
|
1084
2135
|
// over from a prior process that committed local-only before this start);
|
|
1085
2136
|
// afterwards only when a self-write-while-alone was recorded this session.
|
|
1086
2137
|
// registerSelf is idempotent + single-flight, so this never races a
|
|
@@ -1093,15 +2144,15 @@ export class CadreNode {
|
|
|
1093
2144
|
await this.retouchSelfDeviceToken();
|
|
1094
2145
|
this.pendingSelfDeviceWrite = false;
|
|
1095
2146
|
}
|
|
1096
|
-
//
|
|
2147
|
+
// 3. First growth: an owner reconstructs membership rows it may have
|
|
1097
2148
|
// authored that never replicated (covers writes from before this process
|
|
1098
2149
|
// started). Rows already tracked in the in-memory queue are handled by
|
|
1099
|
-
// step
|
|
2150
|
+
// step 4 — skipped here to avoid a double re-touch.
|
|
1100
2151
|
if (firstGrowth) {
|
|
1101
2152
|
this.reconstructedLocalOnlyWrites = true;
|
|
1102
2153
|
await this.reconstructAuthoredMembership();
|
|
1103
2154
|
}
|
|
1104
|
-
//
|
|
2155
|
+
// 4. In-session pending owner authorize writes.
|
|
1105
2156
|
await this.drainPendingPeerWrites();
|
|
1106
2157
|
}
|
|
1107
2158
|
/**
|
|
@@ -1124,7 +2175,70 @@ export class CadreNode {
|
|
|
1124
2175
|
await this.registerDeviceToken(existing.platform, existing.token);
|
|
1125
2176
|
}
|
|
1126
2177
|
catch (error) {
|
|
1127
|
-
log('drain: retouch self DeviceToken failed: %o', error);
|
|
2178
|
+
log('drain: retouch self DeviceToken failed: %o', error);
|
|
2179
|
+
}
|
|
2180
|
+
}
|
|
2181
|
+
/**
|
|
2182
|
+
* Re-issue `Revocation` tombstones so a removal that committed while alone
|
|
2183
|
+
* still reaches the cohort. Two regimes, one method:
|
|
2184
|
+
*
|
|
2185
|
+
* - **First successful pass per process** ({@link reissuedHeldRevocations}
|
|
2186
|
+
* false): sweep EVERY locally-held tombstone from `queryRevocations()` — the
|
|
2187
|
+
* only cover for a removal made before this process started, since the
|
|
2188
|
+
* in-memory queue does not survive a restart. Queued stamps are part of that
|
|
2189
|
+
* same batch (exactly once — one transaction, no double bump).
|
|
2190
|
+
* - **Later passes**: only the tombstones queued in-session by
|
|
2191
|
+
* {@link noteGuardedDelete}.
|
|
2192
|
+
*
|
|
2193
|
+
* Owner-gated: a node with no owner key cannot sign a re-issue, so stray queue
|
|
2194
|
+
* entries are dropped (mirrors {@link drainPendingPeerWrites}). Best-effort — a
|
|
2195
|
+
* failure is logged and leaves the queue (and the sweep flag) untouched for the
|
|
2196
|
+
* next growth edge. A successful `reissueRevocations` exec is NOT proof of
|
|
2197
|
+
* broadcast (the connection that fired the growth edge may not be in the
|
|
2198
|
+
* affected block's cluster — see
|
|
2199
|
+
* tickets/backlog/control-rereplication-broadcast-confirmation); the drain
|
|
2200
|
+
* inherits that known gap, and a full disconnect→reconnect (or the next
|
|
2201
|
+
* process's sweep) re-covers it.
|
|
2202
|
+
*
|
|
2203
|
+
* NOTE: the sweep is O(all tombstones ever) row-updates — plus one owner
|
|
2204
|
+
* signature each — in one transaction, once per lifetime. `Revocation` is
|
|
2205
|
+
* append-only and unbounded growth is declared acceptable for a cadre-sized
|
|
2206
|
+
* party. If the tombstone table ever gets large, bound the sweep (e.g. persist
|
|
2207
|
+
* a node-local high-water mark of what has been re-issued while connected)
|
|
2208
|
+
* instead of re-touching everything; note the `Math.max(...)` spread below also
|
|
2209
|
+
* caps out around 10^5 rows (V8 argument limit) before the size becomes merely
|
|
2210
|
+
* a latency problem.
|
|
2211
|
+
*/
|
|
2212
|
+
async drainPendingRevocations() {
|
|
2213
|
+
const sweep = !this.reissuedHeldRevocations;
|
|
2214
|
+
if (!sweep && this.pendingRevocations.size === 0) {
|
|
2215
|
+
return;
|
|
2216
|
+
}
|
|
2217
|
+
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
2218
|
+
return;
|
|
2219
|
+
}
|
|
2220
|
+
if (!this.seedBootstrapService?.canAuthorize()) {
|
|
2221
|
+
// A non-owner node cannot sign a re-issue; drop any stray entries.
|
|
2222
|
+
this.pendingRevocations.clear();
|
|
2223
|
+
return;
|
|
2224
|
+
}
|
|
2225
|
+
try {
|
|
2226
|
+
const held = await this.controlDatabase.queryRevocations();
|
|
2227
|
+
const rows = sweep ? held : held.filter((r) => this.pendingRevocations.has(r.stampId));
|
|
2228
|
+
if (rows.length > 0) {
|
|
2229
|
+
const reissuedAt = Math.max(Date.now(), Math.max(...rows.map((r) => r.reissuedAt)) + 1);
|
|
2230
|
+
await this.seedBootstrapService.reissueRevocations(rows, reissuedAt);
|
|
2231
|
+
log('drain: re-issued %d revocation tombstone(s)%s', rows.length, sweep ? ' (first-growth sweep)' : '');
|
|
2232
|
+
}
|
|
2233
|
+
this.reissuedHeldRevocations = true;
|
|
2234
|
+
// Per-stamp clear (not clear()) so an entry queued concurrently mid-drain
|
|
2235
|
+
// survives for the next growth edge.
|
|
2236
|
+
for (const row of rows) {
|
|
2237
|
+
this.pendingRevocations.delete(row.stampId);
|
|
2238
|
+
}
|
|
2239
|
+
}
|
|
2240
|
+
catch (error) {
|
|
2241
|
+
log('drain: revocation re-issue failed; leaving queued for next growth: %o', error);
|
|
1128
2242
|
}
|
|
1129
2243
|
}
|
|
1130
2244
|
/**
|
|
@@ -1139,9 +2253,10 @@ export class CadreNode {
|
|
|
1139
2253
|
* row is a no-op-equivalent).
|
|
1140
2254
|
*
|
|
1141
2255
|
* A node with no owner private key skips this entirely — it cannot re-sign
|
|
1142
|
-
* rows for other peers, and rows it merely holds are not its to re-issue.
|
|
1143
|
-
*
|
|
1144
|
-
*
|
|
2256
|
+
* rows for other peers, and rows it merely holds are not its to re-issue.
|
|
2257
|
+
* DELETEs are not reconstructed here — the `Revocation` tombstone sweep in
|
|
2258
|
+
* {@link drainPendingRevocations} covers them (a removed row leaves no
|
|
2259
|
+
* `CadrePeer` trace, but its tombstone is re-issuable).
|
|
1145
2260
|
*/
|
|
1146
2261
|
async reconstructAuthoredMembership() {
|
|
1147
2262
|
if (!this.controlDatabase || !this.controlNode) {
|
|
@@ -1152,10 +2267,25 @@ export class CadreNode {
|
|
|
1152
2267
|
}
|
|
1153
2268
|
const selfPeerId = this.controlNode.peerId.toString();
|
|
1154
2269
|
const members = await this.controlDatabase.queryCadrePeers();
|
|
2270
|
+
// One gate refresh for the whole sweep: every re-touch is a notifying
|
|
2271
|
+
// `CadrePeer` write, and these rows are already in the local snapshot (they
|
|
2272
|
+
// committed here, just local-only), so per-row refreshes would be O(rows)
|
|
2273
|
+
// membership reads for a snapshot that does not change.
|
|
2274
|
+
const touched = await this.deferMembershipGateRefresh('drain-reconstruct', () => this.reissueAuthoredMembershipRows(selfPeerId, members));
|
|
2275
|
+
if (touched > 0) {
|
|
2276
|
+
log('drain: reconstructed %d owner-authored membership row(s) on first cohort growth', touched);
|
|
2277
|
+
}
|
|
2278
|
+
}
|
|
2279
|
+
/**
|
|
2280
|
+
* Body of {@link reconstructAuthoredMembership}'s sweep: re-issue each candidate
|
|
2281
|
+
* owner-authored row, returning how many were re-touched. A per-row failure is
|
|
2282
|
+
* logged and skipped; a teardown mid-sweep stops it.
|
|
2283
|
+
*/
|
|
2284
|
+
async reissueAuthoredMembershipRows(selfPeerId, members) {
|
|
1155
2285
|
let touched = 0;
|
|
1156
2286
|
for (const member of members) {
|
|
1157
2287
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
1158
|
-
return;
|
|
2288
|
+
return touched;
|
|
1159
2289
|
}
|
|
1160
2290
|
if (member.peerId === selfPeerId || this.pendingPeerWrites.has(member.peerId)) {
|
|
1161
2291
|
continue;
|
|
@@ -1173,26 +2303,18 @@ export class CadreNode {
|
|
|
1173
2303
|
log('drain: reconstruction re-touch of %s failed (continuing): %o', member.peerId, error);
|
|
1174
2304
|
}
|
|
1175
2305
|
}
|
|
1176
|
-
|
|
1177
|
-
log('drain: reconstructed %d owner-authored membership row(s) on first cohort growth', touched);
|
|
1178
|
-
}
|
|
2306
|
+
return touched;
|
|
1179
2307
|
}
|
|
1180
2308
|
/**
|
|
1181
2309
|
* Drain the in-session write-while-alone queue: re-issue each pending owner
|
|
1182
|
-
*
|
|
1183
|
-
*
|
|
1184
|
-
*
|
|
1185
|
-
*
|
|
1186
|
-
*
|
|
1187
|
-
*
|
|
1188
|
-
*
|
|
1189
|
-
*
|
|
1190
|
-
* - `remove` (security-relevant): a removePeer-while-alone physically removed the
|
|
1191
|
-
* row locally, so a re-issued DELETE matches nothing and does NOT propagate —
|
|
1192
|
-
* full delete durability requires a schema tombstone (deferred:
|
|
1193
|
-
* `control-delete-while-alone-tombstone`). We still attempt the re-issue
|
|
1194
|
-
* (harmless, and it propagates in the row-still-present case); the commit-alone
|
|
1195
|
-
* was already logged loudly in {@link noteControlWrite}.
|
|
2310
|
+
* authorize as an idempotent monotonic owner UPDATE
|
|
2311
|
+
* ({@link reissuePeerAuthorize}) now that the cohort can broadcast it.
|
|
2312
|
+
* Sequential (no fan-out) to avoid a thundering re-touch; an entry is cleared
|
|
2313
|
+
* only on success, so a failure (or a still-alone re-commit) leaves it queued
|
|
2314
|
+
* for the next growth. An entry is skipped (and cleared) if the row vanished
|
|
2315
|
+
* (raced a delete) or now carries a self-`Sig` (the peer self-published and owns
|
|
2316
|
+
* its republish). Removals are drained by {@link drainPendingRevocations}, not
|
|
2317
|
+
* here.
|
|
1196
2318
|
*/
|
|
1197
2319
|
async drainPendingPeerWrites() {
|
|
1198
2320
|
if (this.pendingPeerWrites.size === 0) {
|
|
@@ -1203,29 +2325,33 @@ export class CadreNode {
|
|
|
1203
2325
|
this.pendingPeerWrites.clear();
|
|
1204
2326
|
return;
|
|
1205
2327
|
}
|
|
1206
|
-
for
|
|
1207
|
-
|
|
2328
|
+
// One gate refresh for the whole drain, not one per entry: these re-issues
|
|
2329
|
+
// replay writes this node already applied locally (and already refreshed
|
|
2330
|
+
// for), so the snapshot only has to be correct once the drain settles.
|
|
2331
|
+
await this.deferMembershipGateRefresh('drain-reissue', () => this.reissuePendingPeerWrites());
|
|
2332
|
+
}
|
|
2333
|
+
/**
|
|
2334
|
+
* Body of {@link drainPendingPeerWrites}' loop. An entry is cleared only on
|
|
2335
|
+
* success (or when there is nothing left to re-issue); a failure is logged and
|
|
2336
|
+
* the entry stays queued for the next cohort growth.
|
|
2337
|
+
*/
|
|
2338
|
+
async reissuePendingPeerWrites() {
|
|
2339
|
+
for (const peerId of [...this.pendingPeerWrites.keys()]) {
|
|
2340
|
+
if (!this._running || !this.controlNode || !this.controlDatabase || !this.seedBootstrapService) {
|
|
1208
2341
|
return;
|
|
1209
2342
|
}
|
|
1210
2343
|
try {
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
continue;
|
|
1217
|
-
}
|
|
1218
|
-
await this.reissuePeerAuthorize(peerId, record.updatedAt);
|
|
1219
|
-
}
|
|
1220
|
-
else {
|
|
1221
|
-
// Best-effort delete re-issue (see method doc): propagates only if the row
|
|
1222
|
-
// is somehow still present locally.
|
|
1223
|
-
await this.seedBootstrapService.removePeer(peerId);
|
|
2344
|
+
const record = await this.controlDatabase.queryPeerRecord(peerId);
|
|
2345
|
+
if (!record || record.sig) {
|
|
2346
|
+
// Removed since, or peer self-published — nothing for the owner to re-issue.
|
|
2347
|
+
this.pendingPeerWrites.delete(peerId);
|
|
2348
|
+
continue;
|
|
1224
2349
|
}
|
|
2350
|
+
await this.reissuePeerAuthorize(peerId, record.updatedAt);
|
|
1225
2351
|
this.pendingPeerWrites.delete(peerId);
|
|
1226
2352
|
}
|
|
1227
2353
|
catch (error) {
|
|
1228
|
-
log('drain: re-issue of %s (
|
|
2354
|
+
log('drain: re-issue of %s (authorize) failed; leaving queued for next growth: %o', peerId, error);
|
|
1229
2355
|
}
|
|
1230
2356
|
}
|
|
1231
2357
|
}
|
|
@@ -1384,6 +2510,13 @@ export class CadreNode {
|
|
|
1384
2510
|
return;
|
|
1385
2511
|
}
|
|
1386
2512
|
if (this.seedBootstrapService) {
|
|
2513
|
+
// NOTE: the `CadrePeer` twin of this read-then-insert needed race recovery because an
|
|
2514
|
+
// owner `authorizePeer` can seat a peer's row for it. Nothing seats a `DeviceToken`
|
|
2515
|
+
// row on a peer's behalf, and this insert has no in-lock existence check, so a lost
|
|
2516
|
+
// race would THROW rather than silently drop the record. If an owner-driven
|
|
2517
|
+
// device-token seeding path is ever added, or two first-publishes can run
|
|
2518
|
+
// concurrently (this method has no single-flight guard, unlike `registerSelf`),
|
|
2519
|
+
// revisit — see `publishSelfRecord`'s fall-through.
|
|
1387
2520
|
await this.seedBootstrapService.insertSelfDeviceToken(record);
|
|
1388
2521
|
if (this.committedAlone())
|
|
1389
2522
|
this.pendingSelfDeviceWrite = true;
|
|
@@ -1404,8 +2537,10 @@ export class CadreNode {
|
|
|
1404
2537
|
* 2. `publicKey <-> peerId` binding — the stored key's libp2p identity is the
|
|
1405
2538
|
* requested peerId,
|
|
1406
2539
|
* 3. a `DeviceToken` row exists with a known {@link PushPlatform},
|
|
1407
|
-
* 4.
|
|
1408
|
-
*
|
|
2540
|
+
* 4. the row's `StampId` is NOT retired in `CadreControl.Revocation` — the
|
|
2541
|
+
* read-side half of the clear (see below),
|
|
2542
|
+
* 5. self-signature verifies against the bound `CadrePeer.PublicKey`,
|
|
2543
|
+
* 6. freshness — `updatedAt` is positive and within `opts.maxAgeMs` (default:
|
|
1409
2544
|
* no ceiling, since a push token is valid until it rotates).
|
|
1410
2545
|
*
|
|
1411
2546
|
* A peer that is not a current member, or whose token has no backing `CadrePeer`
|
|
@@ -1436,6 +2571,22 @@ export class CadreNode {
|
|
|
1436
2571
|
log('resolveDeviceToken: unknown platform %s for %s', record.platform, peerId);
|
|
1437
2572
|
return null;
|
|
1438
2573
|
}
|
|
2574
|
+
// Retired stamp: the read-side mitigation for the write-time race, mirroring what
|
|
2575
|
+
// listAuthorizedMembers does for CadrePeer. Clearing a token retires its StampId
|
|
2576
|
+
// into Revocation, but the schema's NotRevoked CHECK only sees LOCALLY visible
|
|
2577
|
+
// tombstones — a node that converged on a replayed insert before the tombstone can
|
|
2578
|
+
// hold both rows, so the reader must drop the resurrected one.
|
|
2579
|
+
//
|
|
2580
|
+
// NOTE: the freshness ceiling below defaults to INFINITE by design (a suspended
|
|
2581
|
+
// phone must stay push-reachable long after it published), so unlike a peer address
|
|
2582
|
+
// record a stale token never ages out. Stamp retirement is the ONLY thing that
|
|
2583
|
+
// retires a cleared token — do not weaken this check on the assumption that
|
|
2584
|
+
// staleness will eventually cover it.
|
|
2585
|
+
const revokedStamps = await this.controlDatabase.queryRevokedStamps('DeviceToken');
|
|
2586
|
+
if (revokedStamps.has(record.stampId)) {
|
|
2587
|
+
log('resolveDeviceToken: StampId for %s is retired (cleared token, resurrected row)', peerId);
|
|
2588
|
+
return null;
|
|
2589
|
+
}
|
|
1439
2590
|
// Self-signature over (peerId, platform, token, updatedAt), verified against the
|
|
1440
2591
|
// CadrePeer.PublicKey bound to this peerId.
|
|
1441
2592
|
if (!verifyDeviceTokenSignature(record, peerRecord.publicKey)) {
|
|
@@ -1522,7 +2673,6 @@ export class CadreNode {
|
|
|
1522
2673
|
return;
|
|
1523
2674
|
}
|
|
1524
2675
|
try {
|
|
1525
|
-
// Discovery path: no explicit mode — infer from cohort membership.
|
|
1526
2676
|
await this.launchStrand(strand, sAppConfig);
|
|
1527
2677
|
}
|
|
1528
2678
|
catch (error) {
|
|
@@ -1531,17 +2681,21 @@ export class CadreNode {
|
|
|
1531
2681
|
strandId: strand.Id,
|
|
1532
2682
|
error: error instanceof Error ? error : new Error(String(error))
|
|
1533
2683
|
});
|
|
2684
|
+
// Rethrow after the event: the only caller is the StrandWatcher's
|
|
2685
|
+
// onStrandAdded callback, which uses the rejection to decide whether the
|
|
2686
|
+
// strand was really added. Swallowing it here would have the watcher mark
|
|
2687
|
+
// the strand as known-and-added, so no later poll would ever retry it.
|
|
2688
|
+
// The watcher catches and logs, so this never escapes as an unhandled
|
|
2689
|
+
// rejection.
|
|
2690
|
+
throw error;
|
|
1534
2691
|
}
|
|
1535
2692
|
}
|
|
1536
2693
|
async handleStrandRemoved(strandId) {
|
|
1537
2694
|
log('Handling strand removed: %s', strandId);
|
|
1538
2695
|
try {
|
|
1539
|
-
//
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
this.sAppConfigs.delete(strandId);
|
|
1543
|
-
await this.strandManager.stopStrand(strandId);
|
|
1544
|
-
this.emit('strand:stopped', { strandId });
|
|
2696
|
+
// Same local teardown the explicit stop performs — deliberately NOT stopStrand
|
|
2697
|
+
// itself, whose `_running` guard would throw on a poll that lands during shutdown.
|
|
2698
|
+
await this.detachStrand(strandId);
|
|
1545
2699
|
}
|
|
1546
2700
|
catch (error) {
|
|
1547
2701
|
log('Error stopping strand %s: %o', strandId, error);
|
|
@@ -1599,8 +2753,18 @@ export class CadreNode {
|
|
|
1599
2753
|
await this.strandManager.stopAll();
|
|
1600
2754
|
// Clear sApp configs
|
|
1601
2755
|
this.sAppConfigs.clear();
|
|
1602
|
-
//
|
|
2756
|
+
// Drop delegate-admission state: the grants are scoped to the session that
|
|
2757
|
+
// recorded them, so a stop()/start() cycle on this object must not keep
|
|
2758
|
+
// admitting delegates announced under the previous one, nor carry stale
|
|
2759
|
+
// announce timestamps that would suppress the next session's refresh.
|
|
2760
|
+
this.delegateAdmission.clear();
|
|
2761
|
+
this.delegateAnnounceAt.clear();
|
|
2762
|
+
// Close control database (this also shuts down the collection factory).
|
|
2763
|
+
// Detach the membership hub first: nothing this teardown does should drive a
|
|
2764
|
+
// gate refresh, and a notification arriving after this point is a no-op.
|
|
1603
2765
|
if (this.controlDatabase) {
|
|
2766
|
+
this.controlDatabase.setMembershipChangeListener(null);
|
|
2767
|
+
this.controlDatabase.setGuardedDeleteListener(null);
|
|
1604
2768
|
await this.controlDatabase.close();
|
|
1605
2769
|
this.controlDatabase = null;
|
|
1606
2770
|
}
|
|
@@ -1609,6 +2773,16 @@ export class CadreNode {
|
|
|
1609
2773
|
await this.controlNode.stop();
|
|
1610
2774
|
this.controlNode = null;
|
|
1611
2775
|
}
|
|
2776
|
+
// Release the control store's cache registration LAST — after the database and
|
|
2777
|
+
// node that write through it are down. Dropping the field is what lets a
|
|
2778
|
+
// stop()/start() cycle re-resolve the provider against a live cache instead of
|
|
2779
|
+
// handing the restarted node a retired wrapper. Logged, never thrown: a
|
|
2780
|
+
// cache-bookkeeping failure must not abort a teardown.
|
|
2781
|
+
if (this.controlStorage) {
|
|
2782
|
+
const controlStorage = this.controlStorage;
|
|
2783
|
+
this.controlStorage = null;
|
|
2784
|
+
await disposeStorageCache(controlStorage).catch((err) => log('Failed to dispose control storage cache: %o', err));
|
|
2785
|
+
}
|
|
1612
2786
|
}
|
|
1613
2787
|
// Hibernation callbacks
|
|
1614
2788
|
async handleStrandIdle(strandId) {
|
|
@@ -1647,7 +2821,7 @@ export class CadreNode {
|
|
|
1647
2821
|
}
|
|
1648
2822
|
/**
|
|
1649
2823
|
* Wake a strand. If it was hibernating (quiesced — no libp2p node), re-resolve
|
|
1650
|
-
* the cohort discovery seed
|
|
2824
|
+
* the cohort discovery seed exactly as `launchStrand` does and rebuild
|
|
1651
2825
|
* its runtime via the strand manager. If it is still live (e.g. waking an idle
|
|
1652
2826
|
* strand, which retains its resources), just flip the status. Overlapping wake
|
|
1653
2827
|
* triggers are coalesced upstream by `HibernationManager`, so this runs once
|
|
@@ -1667,8 +2841,8 @@ export class CadreNode {
|
|
|
1667
2841
|
this.emit('strand:waking', { strandId });
|
|
1668
2842
|
return;
|
|
1669
2843
|
}
|
|
1670
|
-
// Quiesced: re-resolve volatile cohort
|
|
1671
|
-
//
|
|
2844
|
+
// Quiesced: re-resolve the volatile cohort input (the seed may have grown)
|
|
2845
|
+
// and rebuild the runtime.
|
|
1672
2846
|
log('Waking strand %s — rebuilding strand-network resources', strandId);
|
|
1673
2847
|
await this.resumeStrandRuntime(strandId);
|
|
1674
2848
|
instance.lastActivity = new Date();
|
|
@@ -1676,20 +2850,26 @@ export class CadreNode {
|
|
|
1676
2850
|
log('Strand %s awake (resources rebuilt)', strandId);
|
|
1677
2851
|
}
|
|
1678
2852
|
/**
|
|
1679
|
-
* Rebuild a quiesced strand's runtime, re-resolving the volatile cohort
|
|
1680
|
-
* first: the discovery seed may have grown
|
|
1681
|
-
* shifted the mode `bootstrap → networked` since the strand last ran. Shared by
|
|
2853
|
+
* Rebuild a quiesced strand's runtime, re-resolving the volatile cohort input
|
|
2854
|
+
* first: the discovery seed may have grown since the strand last ran. Shared by
|
|
1682
2855
|
* the wake (`handleStrandWake`) and check-in (`handleStrandCheckIn`) paths so
|
|
1683
|
-
* both apply the same fresh seed
|
|
2856
|
+
* both apply the same fresh seed resolution. `resumeStrand` is idempotent
|
|
1684
2857
|
* (returns the live instance unchanged) as a backstop against double-resume.
|
|
1685
2858
|
*/
|
|
1686
2859
|
async resumeStrandRuntime(strandId) {
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
2860
|
+
// Re-derive the transport peerId (deterministic and cheap — a quiesced
|
|
2861
|
+
// instance has no `libp2pNode` to read it from) so the seed pass announces
|
|
2862
|
+
// the delegate before `resumeStrand` runs `libp2p.start()`.
|
|
2863
|
+
const delegatePeerId = this.identityKey
|
|
2864
|
+
? peerIdFromPrivateKey(await strandTransportKey(this.identityKey, strandId)).toString()
|
|
2865
|
+
: undefined;
|
|
2866
|
+
const bootstrapNodes = await this.resolveCohortSeed(strandId, delegatePeerId);
|
|
2867
|
+
const instance = await this.strandManager.resumeStrand(strandId, { bootstrapNodes });
|
|
2868
|
+
// Same reason as the launch path: `bootstrapNodes` only reaches the address
|
|
2869
|
+
// book through @libp2p/bootstrap discovery, so merge it directly as well.
|
|
2870
|
+
if (instance.libp2pNode) {
|
|
2871
|
+
await this.mergeStrandPeerAddrs(instance.libp2pNode, bootstrapNodes, strandId);
|
|
2872
|
+
}
|
|
1693
2873
|
}
|
|
1694
2874
|
/**
|
|
1695
2875
|
* Real cohort check-in for a hibernating strand (the `onCheckIn` callback).
|
|
@@ -1700,7 +2880,7 @@ export class CadreNode {
|
|
|
1700
2880
|
* realized as a resume → bounded window → re-hibernate-if-idle cycle that
|
|
1701
2881
|
* reuses the existing quiesce/resume primitives rather than a bespoke probe:
|
|
1702
2882
|
*
|
|
1703
|
-
* 1. Resume the strand (rebuild node + db, re-resolve cohort seed
|
|
2883
|
+
* 1. Resume the strand (rebuild node + db, re-resolve the cohort seed) so
|
|
1704
2884
|
* its strand network can reach cohort peers — exactly as a wake does.
|
|
1705
2885
|
* 2. Hold it resumed for a bounded window, during which the app may drive
|
|
1706
2886
|
* reads (pull-on-read) and record activity.
|
|
@@ -1724,7 +2904,7 @@ export class CadreNode {
|
|
|
1724
2904
|
}
|
|
1725
2905
|
try {
|
|
1726
2906
|
// 1. Resume exactly as a wake does: re-resolve the (possibly grown) cohort
|
|
1727
|
-
// seed
|
|
2907
|
+
// seed, then rebuild the runtime.
|
|
1728
2908
|
log('Check-in: resuming strand %s to probe the cohort for pending activity', strandId);
|
|
1729
2909
|
await this.resumeStrandRuntime(strandId);
|
|
1730
2910
|
// 2-3. Bounded window for the strand network to connect + the app to act,
|
|
@@ -1816,20 +2996,26 @@ export class CadreNode {
|
|
|
1816
2996
|
/**
|
|
1817
2997
|
* Add a strand with its sApp configuration.
|
|
1818
2998
|
* The hosting application must provide the sApp schema when creating a strand.
|
|
2999
|
+
*
|
|
3000
|
+
* A rejected call leaves nothing running but DOES leave the sApp config
|
|
3001
|
+
* registered, deliberately: both an explicit retry and the {@link StrandWatcher}'s
|
|
3002
|
+
* automatic relaunch need it. So once the strand's row is visible on the control
|
|
3003
|
+
* network (via {@link publishStrand} or another member), a failed launch keeps
|
|
3004
|
+
* being re-attempted in the background — each failure re-emitting `strand:error`
|
|
3005
|
+
* — until it succeeds. {@link detachStrand} (reached via {@link stopStrand}) is
|
|
3006
|
+
* what abandons a strand for good.
|
|
1819
3007
|
*/
|
|
1820
3008
|
async addStrand(config) {
|
|
1821
3009
|
if (!this._running) {
|
|
1822
3010
|
throw new Error('CadreNode not running');
|
|
1823
3011
|
}
|
|
1824
|
-
const { strandRow, sAppConfig,
|
|
3012
|
+
const { strandRow, sAppConfig, founder } = config;
|
|
1825
3013
|
// Store sApp config for this strand
|
|
1826
3014
|
this.sAppConfigs.set(strandRow.Id, sAppConfig);
|
|
1827
|
-
log('Registered sAppConfig for strand %s (sApp: %s,
|
|
1828
|
-
// Pass `mode` (possibly undefined) through: an explicit mode wins, while a
|
|
1829
|
-
// caller that omits it gets the same cohort-inferred mode as the discovery path.
|
|
3015
|
+
log('Registered sAppConfig for strand %s (sApp: %s, founder: %s)', strandRow.Id, sAppConfig.id, founder ?? false);
|
|
1830
3016
|
// `founder` only flows from the explicit addStrand path — the control-discovered
|
|
1831
3017
|
// join path never founds, so its rows arrive via sync (see handleStrandAdded).
|
|
1832
|
-
return await this.launchStrand(strandRow, sAppConfig,
|
|
3018
|
+
return await this.launchStrand(strandRow, sAppConfig, founder);
|
|
1833
3019
|
}
|
|
1834
3020
|
/**
|
|
1835
3021
|
* Publish a strand row to the shared control database under this node's own
|
|
@@ -1846,7 +3032,7 @@ export class CadreNode {
|
|
|
1846
3032
|
* {@link ed25519KeyPairFromLibp2p} also exposes as the node's owner keypair,
|
|
1847
3033
|
* so peer identity and owner key are one and the same. That key must be
|
|
1848
3034
|
* enrolled in `OwnerKey` (e.g. via {@link ControlDatabase.ensureOwnerKey}
|
|
1849
|
-
* at genesis) or the schema's `Strand.
|
|
3035
|
+
* at genesis) or the schema's `Strand.AuthorizedInsert` constraint rejects the write.
|
|
1850
3036
|
* Failing loudly here is intentional: a silently-unpublished strand would run
|
|
1851
3037
|
* as a local-only island that no peer could ever discover or join.
|
|
1852
3038
|
*
|
|
@@ -1854,30 +3040,99 @@ export class CadreNode {
|
|
|
1854
3040
|
* {@link addStrand}).
|
|
1855
3041
|
* @param type - `'o'` for open (default) or `'c'` for closed.
|
|
1856
3042
|
* @param memberPrivateKey - Optional membership key for a closed strand.
|
|
1857
|
-
* @throws if the node is not started, exposes no owner signing key, or
|
|
1858
|
-
* control DB rejects the (unauthorized) insert.
|
|
3043
|
+
* @throws if the node is not started, exposes no owner signing key, the id is blank, or
|
|
3044
|
+
* the control DB rejects the (unauthorized) insert.
|
|
1859
3045
|
*/
|
|
1860
3046
|
async publishStrand(strandId, type = 'o', memberPrivateKey) {
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
|
|
1864
|
-
const
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
3047
|
+
const signingKey = this.requireOwnerSigningKey(`publish strand ${strandId}`);
|
|
3048
|
+
// Trim/reject here so the id that lands matches the one unpublishStrand looks up: it
|
|
3049
|
+
// trims too, and an untrimmed row would be unreachable by the same string.
|
|
3050
|
+
const trimmed = requireNonBlank(strandId, 'strand id');
|
|
3051
|
+
await this.controlDatabase.insertStrand(trimmed, type, signingKey.publicKeyB64, signMessageWith(signingKey.privateKeyB64), memberPrivateKey);
|
|
3052
|
+
log('Published strand %s (type %s) to control DB under owner %s', trimmed, type, signingKey.publicKeyB64);
|
|
3053
|
+
}
|
|
3054
|
+
/**
|
|
3055
|
+
* Remove this party's `Strand` row from the shared control database — the owner-signed
|
|
3056
|
+
* inverse of {@link publishStrand}.
|
|
3057
|
+
*
|
|
3058
|
+
* Party-wide, unlike {@link stopStrand}, which only stops the strand on THIS node: every
|
|
3059
|
+
* cadre node watching the table sees the row vanish on its next poll (default 5 s) and
|
|
3060
|
+
* stops its own instance. This node converges immediately — the method forces a watcher
|
|
3061
|
+
* poll and stops any still-running local instance before resolving. Removing the row
|
|
3062
|
+
* removes OUR party's participation only: other parties in the strand keep their own
|
|
3063
|
+
* rows and the strand network carries on, so no cross-party sign-off is involved — this
|
|
3064
|
+
* is an owner-signed control-plane write like every other, not "destroy the network".
|
|
3065
|
+
*
|
|
3066
|
+
* Irreversible for a closed strand (`Type='c'`): the row carries `MemberPrivateKey`,
|
|
3067
|
+
* this party's membership secret for that network, and it is stored nowhere else.
|
|
3068
|
+
* With it gone the party can never again admit a member to that closed strand, and a
|
|
3069
|
+
* re-published row would carry a DIFFERENT key that does not match the membership
|
|
3070
|
+
* already written into the strand's RBAC layer. The strand id itself is NOT
|
|
3071
|
+
* blacklisted: a fresh owner-signed {@link publishStrand} re-seats it under a new
|
|
3072
|
+
* stamp — only the unsigned consent re-seat is permanently foreclosed (the removal's
|
|
3073
|
+
* `Revocation` tombstone names the id, which the consent branch of
|
|
3074
|
+
* `Strand.AuthorizedInsert` refuses ever after). Outstanding `FormationInvite` rows
|
|
3075
|
+
* bound to the removed strand are unredeemable while the row is absent (the formation
|
|
3076
|
+
* recorder resolves the strand as missing and rejects cleanly).
|
|
3077
|
+
*
|
|
3078
|
+
* Convergence caveats, same class as {@link enrollValidationKey}'s: a sibling that has
|
|
3079
|
+
* not yet synced keeps running the strand until its own watcher observes the missing
|
|
3080
|
+
* row; a sibling whose `strandFilter` never admitted this strand never observes the
|
|
3081
|
+
* removal AT ALL and keeps running its instance indefinitely — opting out of watching a
|
|
3082
|
+
* strand is also opting out of its party-wide removal, so such a node's only stop is its
|
|
3083
|
+
* own local {@link stopStrand}/`unpublishStrand` call; and a removal committed while
|
|
3084
|
+
* ALONE (0 control connections) deletes the row local-only. The accompanying
|
|
3085
|
+
* `Revocation` tombstone IS queued and re-issued on the next cohort-growth edge
|
|
3086
|
+
* ({@link noteGuardedDelete}/{@link drainPendingRevocations}), so the stamp retirement
|
|
3087
|
+
* — and with it the consent re-seat foreclosure — does propagate; but the physical row
|
|
3088
|
+
* deletion cannot be replayed, and `queryStrands` reads raw (no retired-stamp filter),
|
|
3089
|
+
* so siblings that already hold the row keep running the strand until the collection
|
|
3090
|
+
* itself converges (logged loudly; see the delete-while-alone durability note in
|
|
3091
|
+
* docs/architecture.md).
|
|
3092
|
+
*
|
|
3093
|
+
* A no-op (no throw, no tombstone) when the row is already absent — but a
|
|
3094
|
+
* locally-running instance of that id is still stopped.
|
|
3095
|
+
*
|
|
3096
|
+
* NOTE: control-plane only — the strand's local durable storage is retained. Stopping the
|
|
3097
|
+
* instance closes the `StrandDatabase` and its libp2p node but purges no blocks, so a
|
|
3098
|
+
* closed strand's content stays readable on disk to anyone with the data directory. If a
|
|
3099
|
+
* caller ever needs removal to mean "and erase the local copy", that is a separate purge
|
|
3100
|
+
* step, not a widening of this method.
|
|
3101
|
+
*
|
|
3102
|
+
* @param strandId - The `Strand` row id to remove (as passed to {@link publishStrand}).
|
|
3103
|
+
* @throws if the node is not started, exposes no owner signing key, the id is blank, or
|
|
3104
|
+
* the signer is not an enrolled owner (the schema's `Strand.AuthorizedDelete` rejects
|
|
3105
|
+
* the write and the row survives). A rejection does NOT imply the row survived: the
|
|
3106
|
+
* local stop runs after the control-plane delete has already committed, so a failure
|
|
3107
|
+
* there throws over a completed removal.
|
|
3108
|
+
*/
|
|
3109
|
+
async unpublishStrand(strandId) {
|
|
3110
|
+
const signingKey = this.requireOwnerSigningKey(`unpublish strand ${strandId}`);
|
|
3111
|
+
const trimmed = requireNonBlank(strandId, 'strand id');
|
|
3112
|
+
const removed = await this.controlDatabase.deleteStrand(trimmed, signingKey.publicKeyB64, signMessageWith(signingKey.privateKeyB64));
|
|
3113
|
+
// Gate on `removed`: the absent-row no-op wrote nothing, so warning about an
|
|
3114
|
+
// unreplicated deletion there would send an operator chasing a phantom.
|
|
3115
|
+
if (removed && this.committedAlone()) {
|
|
3116
|
+
log('unpublishStrand(%s) committed while ALONE (0 control connections): the deletion is ' +
|
|
3117
|
+
'local-only. Its Revocation tombstone is queued for re-issue on cohort growth, but the ' +
|
|
3118
|
+
'row deletion itself cannot be replayed, so other nodes may keep running the strand ' +
|
|
3119
|
+
'until the collection converges.', trimmed);
|
|
3120
|
+
}
|
|
3121
|
+
// Converge locally now rather than waiting up to a poll interval. The watcher fires
|
|
3122
|
+
// onStrandRemoved for a strand it tracked; the explicit stop below covers a node whose
|
|
3123
|
+
// strandFilter never admitted this strand (the watcher never knew it, so it will never
|
|
3124
|
+
// fire). StrandInstanceManager.stopStrand no-ops on an unknown id, so the two paths
|
|
3125
|
+
// cannot double-stop into an error.
|
|
3126
|
+
await this.strandWatcher?.forcePoll();
|
|
3127
|
+
if (this.strandManager.getInstance(trimmed)) {
|
|
3128
|
+
await this.stopStrand(trimmed);
|
|
1869
3129
|
}
|
|
1870
|
-
|
|
1871
|
-
// buildAuthorizationMessage); ed25519-sign them directly (no pre-hash) with
|
|
1872
|
-
// the owner private key, returning a base64url signature.
|
|
1873
|
-
const signMessage = (message) => sign(message, signingKey.privateKeyB64, 'ed25519', 'bytes', 'base64url', 'base64url');
|
|
1874
|
-
await this.controlDatabase.insertStrand(strandId, type, signingKey.publicKeyB64, signMessage, memberPrivateKey);
|
|
1875
|
-
log('Published strand %s (type %s) to control DB under owner %s', strandId, type, signingKey.publicKeyB64);
|
|
3130
|
+
log('Unpublished strand %s from control DB under owner %s', trimmed, signingKey.publicKeyB64);
|
|
1876
3131
|
}
|
|
1877
3132
|
/**
|
|
1878
3133
|
* Publish an owner-signed `FormationInvite` (open-invitation token) to the
|
|
1879
3134
|
* shared control database, so a later {@link formStrand} redemption can be
|
|
1880
|
-
* validated against it (the consent branch of `Strand.
|
|
3135
|
+
* validated against it (the consent branch of `Strand.AuthorizedInsert`).
|
|
1881
3136
|
*
|
|
1882
3137
|
* Counterpart to {@link createOpenInvitation}, which only mints the
|
|
1883
3138
|
* out-of-band {@link OpenInvitation} envelope: persisting the matching
|
|
@@ -1897,28 +3152,147 @@ export class CadreNode {
|
|
|
1897
3152
|
* `strandId` (bind a closed/pre-existing host strand for provision-then-record).
|
|
1898
3153
|
*/
|
|
1899
3154
|
async publishFormationInvite(token, sAppId, options = {}) {
|
|
3155
|
+
const signingKey = this.requireOwnerSigningKey(`publish formation invite ${token}`);
|
|
3156
|
+
await this.controlDatabase.insertFormationInvite(token, sAppId, signingKey.publicKeyB64, signMessageWith(signingKey.privateKeyB64), options);
|
|
3157
|
+
// A host may publish an invite whose token was minted elsewhere; registering
|
|
3158
|
+
// it locally opens the connection gate's formation exemption immediately
|
|
3159
|
+
// rather than waiting for the durable row to become readable.
|
|
3160
|
+
this.strandSolicitationService?.registerMintedInvitation(token, options.expiresAtMs ?? Number.POSITIVE_INFINITY);
|
|
3161
|
+
log('Published formation invite %s (sApp %s) under owner %s', token, sAppId, signingKey.publicKeyB64);
|
|
3162
|
+
}
|
|
3163
|
+
/**
|
|
3164
|
+
* Enroll an approver public key allowed to sign off on `ValidationUrl` redemptions.
|
|
3165
|
+
*
|
|
3166
|
+
* A `FormationInvite` carrying a `ValidationUrl` is only redeemable when the approval
|
|
3167
|
+
* that comes back from that URL is signed by a key present in `CadreControl.ValidationKey`
|
|
3168
|
+
* — this is how a party says which outside approver it trusts. Without an enrollment,
|
|
3169
|
+
* every such invitation is unredeemable.
|
|
3170
|
+
*
|
|
3171
|
+
* Signs with the same self-owner key as {@link publishStrand} (the ed25519 key behind
|
|
3172
|
+
* this node's PeerId, which must be an enrolled `OwnerKey`). Throws loudly if the node
|
|
3173
|
+
* isn't started, exposes no signing key, or the key is blank — a blank key would reach
|
|
3174
|
+
* the database and fail the signature CHECK as an opaque constraint error.
|
|
3175
|
+
*
|
|
3176
|
+
* Enrolling this node's OWN owner key is permitted and harmless: the domain/action tags
|
|
3177
|
+
* baked into each authorization digest mean an owner-key signature can never satisfy the
|
|
3178
|
+
* approval rule and an approval can never satisfy an owner rule. No guard is needed.
|
|
3179
|
+
*
|
|
3180
|
+
* NOTE: enrollment is replicated control state, so a key enrolled here is visible to a
|
|
3181
|
+
* sibling cadre node only once control replication converges. A redemption that arrives
|
|
3182
|
+
* at a node which has not caught up is refused as not-enrolled — the same convergence
|
|
3183
|
+
* gap the schema records on `Strand.StampId`. Enroll before circulating the invitations
|
|
3184
|
+
* that depend on the key.
|
|
3185
|
+
*
|
|
3186
|
+
* @param key - Approver public key to enroll (base64url ed25519 public key).
|
|
3187
|
+
*/
|
|
3188
|
+
async enrollValidationKey(key) {
|
|
3189
|
+
const signingKey = this.requireOwnerSigningKey('enroll a validation key');
|
|
3190
|
+
const trimmed = requireEd25519PublicKeyB64(key, 'validation key');
|
|
3191
|
+
await this.controlDatabase.insertValidationKey(trimmed, signingKey.publicKeyB64, signMessageWith(signingKey.privateKeyB64));
|
|
3192
|
+
log('Enrolled validation key %s under owner %s', trimmed, signingKey.publicKeyB64);
|
|
3193
|
+
}
|
|
3194
|
+
/**
|
|
3195
|
+
* Remove an approver public key.
|
|
3196
|
+
*
|
|
3197
|
+
* Narrows who may approve FUTURE redemptions ONLY. The schema's approval CHECKs run at
|
|
3198
|
+
* write time, so a join that was already approved by this key stays valid — removal is
|
|
3199
|
+
* not retroactive and does not re-examine committed rows.
|
|
3200
|
+
*
|
|
3201
|
+
* Rotation is therefore add-then-remove, in that order: removing the only enrolled key
|
|
3202
|
+
* while `ValidationUrl` invitations are outstanding makes every one of them unredeemable
|
|
3203
|
+
* until a new key is enrolled.
|
|
3204
|
+
*
|
|
3205
|
+
* Removing a key that is not enrolled is a silent no-op (no throw, no `Revocation`
|
|
3206
|
+
* tombstone) — see {@link ControlDatabase.deleteValidationKey}.
|
|
3207
|
+
*
|
|
3208
|
+
* @param key - Approver public key to remove (base64url ed25519 public key).
|
|
3209
|
+
*/
|
|
3210
|
+
async removeValidationKey(key) {
|
|
3211
|
+
const signingKey = this.requireOwnerSigningKey('remove a validation key');
|
|
3212
|
+
const trimmed = requireNonBlank(key, 'validation key');
|
|
3213
|
+
await this.controlDatabase.deleteValidationKey(trimmed, signingKey.publicKeyB64, signMessageWith(signingKey.privateKeyB64));
|
|
3214
|
+
log('Removed validation key %s under owner %s', trimmed, signingKey.publicKeyB64);
|
|
3215
|
+
}
|
|
3216
|
+
/**
|
|
3217
|
+
* The approver keys currently enrolled, sorted. Read-only, so unlike
|
|
3218
|
+
* {@link enrollValidationKey} / {@link removeValidationKey} it needs no owner signing
|
|
3219
|
+
* key — only a started node.
|
|
3220
|
+
*/
|
|
3221
|
+
async listValidationKeys() {
|
|
3222
|
+
if (!this._running || !this.controlDatabase) {
|
|
3223
|
+
throw new Error('CadreNode must be started before listing validation keys');
|
|
3224
|
+
}
|
|
3225
|
+
return await this.controlDatabase.queryValidationKeys();
|
|
3226
|
+
}
|
|
3227
|
+
/**
|
|
3228
|
+
* Resolve the owner keypair every owner-signed control write needs — {@link publishStrand},
|
|
3229
|
+
* {@link unpublishStrand}, {@link publishFormationInvite}, {@link enrollValidationKey},
|
|
3230
|
+
* {@link removeValidationKey}
|
|
3231
|
+
* — failing loudly in one two-part shape (not started / no signing key, naming owner
|
|
3232
|
+
* genesis as the fix). Narrows `controlDatabase` for the caller: a non-null return means
|
|
3233
|
+
* `this.controlDatabase` is non-null too.
|
|
3234
|
+
*
|
|
3235
|
+
* @param action - Infinitive phrase naming the attempted write, e.g. `'enroll a
|
|
3236
|
+
* validation key'`; interpolated into both messages.
|
|
3237
|
+
*/
|
|
3238
|
+
requireOwnerSigningKey(action) {
|
|
1900
3239
|
if (!this._running || !this.controlDatabase) {
|
|
1901
|
-
throw new Error(
|
|
3240
|
+
throw new Error(`CadreNode must be started before attempting to ${action}`);
|
|
1902
3241
|
}
|
|
1903
3242
|
const signingKey = this.getSelfSigningKey();
|
|
1904
3243
|
if (!signingKey) {
|
|
1905
|
-
throw new Error(`Cannot
|
|
3244
|
+
throw new Error(`Cannot ${action}: no owner signing key available ` +
|
|
1906
3245
|
'(node identity is unavailable or does not match the node PeerId). Run owner ' +
|
|
1907
|
-
'genesis (ensureOwnerKey + initializeSeedBootstrap) before
|
|
3246
|
+
'genesis (ensureOwnerKey + initializeSeedBootstrap) before writing owner-signed ' +
|
|
3247
|
+
'control state.');
|
|
1908
3248
|
}
|
|
1909
|
-
|
|
1910
|
-
await this.controlDatabase.insertFormationInvite(token, sAppId, signingKey.publicKeyB64, signMessage, options);
|
|
1911
|
-
log('Published formation invite %s (sApp %s) under owner %s', token, sAppId, signingKey.publicKeyB64);
|
|
3249
|
+
return signingKey;
|
|
1912
3250
|
}
|
|
1913
3251
|
/**
|
|
1914
3252
|
* Shared strand launch path for both the explicit (`addStrand`) and the
|
|
1915
3253
|
* control-discovered (`handleStrandAdded`) entry points. Resolves the cohort
|
|
1916
|
-
* seed,
|
|
1917
|
-
*
|
|
3254
|
+
* seed, starts the strand, and registers it with the hibernation manager
|
|
3255
|
+
* before emitting `strand:started`.
|
|
3256
|
+
*
|
|
3257
|
+
* Idempotent no-op when the strand manager already tracks `strand.Id` — the
|
|
3258
|
+
* ordinary founding sequence is `addStrand` (starts it locally) followed by
|
|
3259
|
+
* `publishStrand` (makes it visible), so this node's own `StrandWatcher`
|
|
3260
|
+
* rediscovers a row it already started and calls this again via
|
|
3261
|
+
* `handleStrandAdded`. Without the guard that re-entry would resolve a fresh
|
|
3262
|
+
* (already-connected) cohort seed and re-emit `strand:started` for an
|
|
3263
|
+
* instance that never stopped. The guard runs before the cohort-seed RPC
|
|
3264
|
+
* fan-out, so a rediscovery costs nothing beyond the map lookup.
|
|
1918
3265
|
*/
|
|
1919
|
-
async launchStrand(strand, sAppConfig,
|
|
1920
|
-
const
|
|
1921
|
-
|
|
3266
|
+
async launchStrand(strand, sAppConfig, founder) {
|
|
3267
|
+
const existing = this.strandManager.getInstance(strand.Id);
|
|
3268
|
+
if (existing) {
|
|
3269
|
+
log('launchStrand: strand %s already tracked locally — skipping re-launch', strand.Id);
|
|
3270
|
+
return existing;
|
|
3271
|
+
}
|
|
3272
|
+
// Each strand node gets its own transport identity, derived from the cadre
|
|
3273
|
+
// identity key + strandId (see strand-transport-key.ts). Sharing the
|
|
3274
|
+
// control node's key here gave every node one peerId, which collides at a
|
|
3275
|
+
// shared circuit relay (github.com/gotchoices/sereus/issues/1). The
|
|
3276
|
+
// retained launch config carries this derived key, so hibernate → wake
|
|
3277
|
+
// (resumeStrand) reuses the same peerId.
|
|
3278
|
+
//
|
|
3279
|
+
// NOTE: derivation requires an Ed25519 identity key, so a node configured
|
|
3280
|
+
// with some other key type now fails strand launch outright (surfaced as
|
|
3281
|
+
// `strand:error` / a rejected addStrand) where it previously started the
|
|
3282
|
+
// strand on that key. Ed25519 is already required for every control-DB
|
|
3283
|
+
// signing path, so nothing reachable today hits this; if a non-Ed25519
|
|
3284
|
+
// identity is ever supported, fall back to `undefined` here — libp2p then
|
|
3285
|
+
// generates a random per-strand key, which still avoids the collision but
|
|
3286
|
+
// gives up peerId stability across restarts.
|
|
3287
|
+
const transportKey = this.identityKey
|
|
3288
|
+
? await strandTransportKey(this.identityKey, strand.Id)
|
|
3289
|
+
: undefined;
|
|
3290
|
+
// Derived BEFORE seed resolution so the seed pass doubles as the delegate
|
|
3291
|
+
// announcement and every grant is recorded before `startStrand` runs
|
|
3292
|
+
// `libp2p.start()` (the responder records the grant before replying; the
|
|
3293
|
+
// client awaits the replies).
|
|
3294
|
+
const delegatePeerId = transportKey ? peerIdFromPrivateKey(transportKey).toString() : undefined;
|
|
3295
|
+
const bootstrapNodes = await this.resolveCohortSeed(strand.Id, delegatePeerId);
|
|
1922
3296
|
const instance = await this.strandManager.startStrand({
|
|
1923
3297
|
strandRow: strand,
|
|
1924
3298
|
sAppConfig,
|
|
@@ -1926,44 +3300,343 @@ export class CadreNode {
|
|
|
1926
3300
|
network: this.config.network,
|
|
1927
3301
|
profile: this.config.profile,
|
|
1928
3302
|
defaultLatencyHint: this.config.hibernation?.defaultLatencyHint ?? 'interactive',
|
|
1929
|
-
privateKey:
|
|
1930
|
-
bootstrapNodes
|
|
1931
|
-
mode,
|
|
3303
|
+
privateKey: transportKey,
|
|
3304
|
+
bootstrapNodes,
|
|
1932
3305
|
requireSignedSchemas: this.config.requireSignedSchemas,
|
|
3306
|
+
clusterSize: this.config.strandClusterSize,
|
|
3307
|
+
backfill: this.config.strandBackfill,
|
|
1933
3308
|
founder
|
|
1934
3309
|
});
|
|
3310
|
+
// The seed reached the node as `bootstrapNodes`, which only enters the
|
|
3311
|
+
// address book via @libp2p/bootstrap discovery. Merge it directly too, so a
|
|
3312
|
+
// sibling is dialable by bare peer id from the first moment (see
|
|
3313
|
+
// mergeStrandPeerAddrs; refreshStrandPeerAddrs keeps it warm from here on).
|
|
3314
|
+
if (instance.libp2pNode) {
|
|
3315
|
+
await this.mergeStrandPeerAddrs(instance.libp2pNode, bootstrapNodes, strand.Id);
|
|
3316
|
+
}
|
|
1935
3317
|
this.hibernationManager.trackStrand(instance);
|
|
1936
3318
|
this.emit('strand:started', { strandId: strand.Id });
|
|
1937
3319
|
return instance;
|
|
1938
3320
|
}
|
|
1939
3321
|
/**
|
|
1940
|
-
* Resolve a strand's discovery seed
|
|
1941
|
-
*
|
|
1942
|
-
*
|
|
1943
|
-
*
|
|
1944
|
-
* addresses that must not
|
|
3322
|
+
* Resolve a strand's discovery seed — the dialable strand-network multiaddr
|
|
3323
|
+
* strings for cohort siblings. Membership comes from the control network's
|
|
3324
|
+
* CadrePeer rows; the **strand-network** bootstrap addresses are resolved on
|
|
3325
|
+
* demand over the control mesh via the strand-addr RPC — deliberately NOT
|
|
3326
|
+
* from `CadrePeer.Multiaddr`, which carries *control* addresses that must not
|
|
3327
|
+
* seed the strand mesh.
|
|
1945
3328
|
*
|
|
1946
3329
|
* Only siblings we already hold an open control connection to are RPC'd: they
|
|
1947
3330
|
* are the ones that can answer right now, and dialing them by peerId reuses the
|
|
1948
3331
|
* live connection. When no connected sibling yet runs the strand the seed is
|
|
1949
|
-
* empty (`[]`) —
|
|
1950
|
-
*
|
|
1951
|
-
*
|
|
3332
|
+
* empty (`[]`) — the empty seed self-heals on the next resume / check-in pass.
|
|
3333
|
+
* Returns an empty seed when the control DB or node is absent (not yet
|
|
3334
|
+
* started / torn down).
|
|
3335
|
+
*
|
|
3336
|
+
* When `delegatePeerId` is given (a strand launch/resume is imminent), the
|
|
3337
|
+
* pass doubles as the delegate ANNOUNCEMENT: our circuit relays are merged
|
|
3338
|
+
* into the RPC targets and every request carries the delegate peerId, so each
|
|
3339
|
+
* receiver records an admission grant BEFORE `libp2p.start()` (re-)dials the
|
|
3340
|
+
* relay reservation — the gate denial there is fatal, not degraded. A
|
|
3341
|
+
* party-member relay answers the RPC (its control node admits us as a
|
|
3342
|
+
* member); a dedicated ops/ relay does not speak the protocol and the
|
|
3343
|
+
* per-peer failure folds to `[]` — harmless, it has no membership gate and
|
|
3344
|
+
* needs no grant. The relay's direct addr rides along as the dial fallback
|
|
3345
|
+
* for a relay we are not yet connected to.
|
|
1952
3346
|
*/
|
|
1953
|
-
async resolveCohortSeed(strandId) {
|
|
3347
|
+
async resolveCohortSeed(strandId, delegatePeerId) {
|
|
1954
3348
|
if (!this.controlDatabase || !this.controlNode) {
|
|
1955
|
-
return
|
|
3349
|
+
return [];
|
|
3350
|
+
}
|
|
3351
|
+
const targets = await this.connectedSiblingTargets();
|
|
3352
|
+
const relays = delegatePeerId === undefined ? [] : this.circuitRelayTargets();
|
|
3353
|
+
for (const relay of relays) {
|
|
3354
|
+
if (!targets.some((t) => t.peerId === relay.relayPeerId)) {
|
|
3355
|
+
targets.push(relayStrandAddrPeer(relay));
|
|
3356
|
+
}
|
|
1956
3357
|
}
|
|
1957
|
-
const peers = await this.controlDatabase.queryCadrePeers();
|
|
1958
|
-
const { otherPeerIds, hasOtherPeers } = deriveCohortMembers(peers, this.controlNode.peerId.toString());
|
|
1959
|
-
// RPC only siblings we already have a control connection to — they can answer
|
|
1960
|
-
// now, and `dialProtocol` by peerId reuses the open connection.
|
|
1961
|
-
const connected = new Set(this.controlNode.getConnections().map((c) => c.remotePeer.toString()));
|
|
1962
|
-
const targets = otherPeerIds.filter((id) => connected.has(id));
|
|
1963
3358
|
const bootstrapNodes = targets.length
|
|
1964
|
-
? await collectStrandAddrs(this.controlNode, targets
|
|
3359
|
+
? await collectStrandAddrs(this.controlNode, targets, strandId, { delegatePeerId })
|
|
1965
3360
|
: [];
|
|
1966
|
-
|
|
3361
|
+
// Throttle state for the RELAY targets only — refreshDelegateGrants never
|
|
3362
|
+
// looks up a sibling key, so recording one would only be dead weight.
|
|
3363
|
+
this.recordDelegateAnnounces(relays.map((r) => r.relayPeerId), strandId);
|
|
3364
|
+
return bootstrapNodes;
|
|
3365
|
+
}
|
|
3366
|
+
/**
|
|
3367
|
+
* The co-cadre siblings worth sending a strand-addr RPC to right now: cohort
|
|
3368
|
+
* members (self excluded) we already hold an open control connection to. They
|
|
3369
|
+
* are the ones that can answer immediately, and dialing them by peerId reuses
|
|
3370
|
+
* the live connection instead of opening a second one.
|
|
3371
|
+
*
|
|
3372
|
+
* Shared by {@link resolveCohortSeed} (launch/resume) and
|
|
3373
|
+
* {@link refreshStrandPeerAddrs} (the periodic pass) so both ask the same set.
|
|
3374
|
+
* Empty when the control DB or node is absent (not yet started / torn down).
|
|
3375
|
+
*
|
|
3376
|
+
* The read below is issued even when the node holds no control connection and
|
|
3377
|
+
* the answer is therefore empty by construction. That is deliberate on the
|
|
3378
|
+
* launch path: `control-database-solo-warm-start.spec.ts` exists to prove this
|
|
3379
|
+
* exact read does not stall as an embedder's FIRST awaited control operation,
|
|
3380
|
+
* on a warm cohort no one can reach. A caller that re-enters often enough for
|
|
3381
|
+
* the read to matter should skip the call itself, as
|
|
3382
|
+
* {@link refreshStrandPeerAddrs} does.
|
|
3383
|
+
*
|
|
3384
|
+
* NOTE: this read is UNBOUNDED and sits on the critical path an embedding app
|
|
3385
|
+
* awaits during startup — `addStrand` cannot resolve until it does, and it is
|
|
3386
|
+
* the first control operation an embedder's boot order issues (before genesis,
|
|
3387
|
+
* before seed bootstrap). Measured fine today: the warm-start-alone shape a
|
|
3388
|
+
* report pointed at — a `CadrePeer` list naming peers that are all gone, read
|
|
3389
|
+
* off real files after a restart — completes in milliseconds, and
|
|
3390
|
+
* `control-database-solo-warm-start.spec.ts` covers it under a deadline. If a
|
|
3391
|
+
* control read ever CAN stall (a transactor change that consults the network
|
|
3392
|
+
* for a local row, a storage backend with blocking I/O), give this one a
|
|
3393
|
+
* timeout — and decide then whether the breach fails `addStrand` or degrades
|
|
3394
|
+
* to an empty seed, because those promise callers different things.
|
|
3395
|
+
*/
|
|
3396
|
+
async connectedSiblingTargets() {
|
|
3397
|
+
const controlNode = this.controlNode;
|
|
3398
|
+
if (!this.controlDatabase || !controlNode) {
|
|
3399
|
+
return [];
|
|
3400
|
+
}
|
|
3401
|
+
const peers = await this.controlDatabase.queryCadrePeers();
|
|
3402
|
+
const connected = new Set(controlNode.getConnections().map((c) => c.remotePeer.toString()));
|
|
3403
|
+
return deriveCohortMembers(peers, controlNode.peerId.toString())
|
|
3404
|
+
.filter((id) => connected.has(id))
|
|
3405
|
+
.map((peerId) => ({ peerId }));
|
|
3406
|
+
}
|
|
3407
|
+
/**
|
|
3408
|
+
* The circuit relays this node's strand nodes would reserve through: the
|
|
3409
|
+
* union of the RESOLVED listen addrs' circuits — a hand-written
|
|
3410
|
+
* `network.listenAddrs` circuit entry or a `network.relayAddrs` entry, which
|
|
3411
|
+
* the strand node inherits verbatim either way — and the control node's own
|
|
3412
|
+
* live `/p2p-circuit` multiaddrs (a reservation this node discovered rather
|
|
3413
|
+
* than configured).
|
|
3414
|
+
*
|
|
3415
|
+
* NOTE: a relay the STRAND node discovers on its own (autorelay — in neither
|
|
3416
|
+
* source) gets no announcement, and a membership-gated one will deny it; fine
|
|
3417
|
+
* now, every realistic topology feeds one of the two sources.
|
|
3418
|
+
*/
|
|
3419
|
+
circuitRelayTargets() {
|
|
3420
|
+
return extractCircuitRelayTargets([
|
|
3421
|
+
...(resolveListenAddrs(this.config.network) ?? []),
|
|
3422
|
+
...(this.controlNode?.getMultiaddrs().map(String) ?? [])
|
|
3423
|
+
]);
|
|
3424
|
+
}
|
|
3425
|
+
/**
|
|
3426
|
+
* Record the announce timestamps {@link refreshDelegateGrants} throttles on,
|
|
3427
|
+
* for every relay a delegate-carrying announce pass dialed.
|
|
3428
|
+
*
|
|
3429
|
+
* Recorded OPTIMISTICALLY at announce time: `collectStrandAddrs` folds
|
|
3430
|
+
* per-peer failure to `[]` and reports no per-peer success, and threading
|
|
3431
|
+
* success out would change its API for little gain. A failed INITIAL announce
|
|
3432
|
+
* is fatal-at-start anyway (the relay denies the reservation,
|
|
3433
|
+
* `libp2p.start()` throws, and wake/check-in re-resume retries with a fresh
|
|
3434
|
+
* announce); a failed REFRESH retries within `DELEGATE_GRANT_TTL_MS / 2`
|
|
3435
|
+
* (15 min), still inside the 30 min TTL.
|
|
3436
|
+
*/
|
|
3437
|
+
recordDelegateAnnounces(relayPeerIds, strandId, now = Date.now()) {
|
|
3438
|
+
for (const relayPeerId of relayPeerIds) {
|
|
3439
|
+
this.delegateAnnounceAt.set(peerStrandKey(relayPeerId, strandId), now);
|
|
3440
|
+
}
|
|
3441
|
+
}
|
|
3442
|
+
/**
|
|
3443
|
+
* Re-announce every running strand's delegate peerId to this node's circuit
|
|
3444
|
+
* relays, throttled to once per `DELEGATE_GRANT_TTL_MS / 2` per
|
|
3445
|
+
* (relay, strand). A grant must outlive the reservation: a dropped relay
|
|
3446
|
+
* connection makes the strand's circuit-relay transport re-dial and face the
|
|
3447
|
+
* connection gate again, so the relay must still hold a live grant then.
|
|
3448
|
+
* RELAY targets only — siblings get their announce on every launch/resume
|
|
3449
|
+
* seed pass, and a grant only matters where a reservation can be re-dialed.
|
|
3450
|
+
*
|
|
3451
|
+
* Strands announce CONCURRENTLY: this runs ahead of the reconcile pass's
|
|
3452
|
+
* sibling enumeration, and one unreachable relay costs a dial timeout per
|
|
3453
|
+
* target, which must not stack up per strand.
|
|
3454
|
+
*/
|
|
3455
|
+
async refreshDelegateGrants(now = Date.now()) {
|
|
3456
|
+
if (!this.controlNode) {
|
|
3457
|
+
return;
|
|
3458
|
+
}
|
|
3459
|
+
// A running strand's delegate peerId is simply its live node's peerId — no
|
|
3460
|
+
// re-derivation here.
|
|
3461
|
+
const running = new Map();
|
|
3462
|
+
for (const [strandId, instance] of this.strandManager.getInstances()) {
|
|
3463
|
+
if (instance.libp2pNode) {
|
|
3464
|
+
running.set(strandId, instance.libp2pNode.peerId.toString());
|
|
3465
|
+
}
|
|
3466
|
+
}
|
|
3467
|
+
pruneStoppedStrandAnnounces(this.delegateAnnounceAt, new Set(running.keys()));
|
|
3468
|
+
if (running.size === 0) {
|
|
3469
|
+
return;
|
|
3470
|
+
}
|
|
3471
|
+
const relays = this.circuitRelayTargets();
|
|
3472
|
+
if (relays.length === 0) {
|
|
3473
|
+
return;
|
|
3474
|
+
}
|
|
3475
|
+
await Promise.all([...running].map(([strandId, delegatePeerId]) => this.announceDelegateToDueRelays(strandId, delegatePeerId, relays, now)));
|
|
3476
|
+
}
|
|
3477
|
+
/** One strand's share of {@link refreshDelegateGrants}: announce to the relays whose grant is due. */
|
|
3478
|
+
async announceDelegateToDueRelays(strandId, delegatePeerId, relays, now) {
|
|
3479
|
+
const controlNode = this.controlNode;
|
|
3480
|
+
if (!controlNode) {
|
|
3481
|
+
return;
|
|
3482
|
+
}
|
|
3483
|
+
const due = dueRelayAnnounces(this.delegateAnnounceAt, relays, strandId, now);
|
|
3484
|
+
if (due.length === 0) {
|
|
3485
|
+
return;
|
|
3486
|
+
}
|
|
3487
|
+
await collectStrandAddrs(controlNode, due.map(relayStrandAddrPeer), strandId, { delegatePeerId });
|
|
3488
|
+
this.recordDelegateAnnounces(due.map((relay) => relay.relayPeerId), strandId, now);
|
|
3489
|
+
}
|
|
3490
|
+
/**
|
|
3491
|
+
* Re-resolve each running strand's SIBLING addresses over the control mesh and
|
|
3492
|
+
* merge them into that strand's own libp2p address book, throttled to once per
|
|
3493
|
+
* {@link STRAND_PEER_ADDR_REFRESH_MS} per strand.
|
|
3494
|
+
*
|
|
3495
|
+
* Without this, a strand's address book is written exactly once — the
|
|
3496
|
+
* launch/resume seed — and everything below cadre-core that dials a strand peer
|
|
3497
|
+
* by bare peer id (Optimystic's cluster and repo clients, FRET ping/announce)
|
|
3498
|
+
* loses its address for any sibling it is not currently connected to: a sibling
|
|
3499
|
+
* that restarted its strand node or rotated its relay reservation is never
|
|
3500
|
+
* re-resolved, and even the original seed addresses fall off at the peerStore's
|
|
3501
|
+
* one-hour expiry (see `peer-addr-book.ts`).
|
|
3502
|
+
*
|
|
3503
|
+
* Distinct from {@link refreshDelegateGrants}, deliberately: that pass covers
|
|
3504
|
+
* RELAYS on a `DELEGATE_GRANT_TTL_MS / 2` throttle to keep circuit-relay
|
|
3505
|
+
* admission grants alive, this one covers SIBLINGS on an address-expiry
|
|
3506
|
+
* throttle to keep the address book warm. They overlap only in that both carry
|
|
3507
|
+
* `delegatePeerId`, so a sibling that also runs a relay gets its grant
|
|
3508
|
+
* refreshed here as a side effect. Merging them would tie an admission-grant
|
|
3509
|
+
* TTL to an address-expiry window that has nothing to do with it.
|
|
3510
|
+
*
|
|
3511
|
+
* Strands refresh CONCURRENTLY, like `refreshDelegateGrants`, so one
|
|
3512
|
+
* unreachable sibling's dial timeout does not stack up per strand.
|
|
3513
|
+
*/
|
|
3514
|
+
async refreshStrandPeerAddrs(now = Date.now()) {
|
|
3515
|
+
if (!this._running || !this.controlNode) {
|
|
3516
|
+
return;
|
|
3517
|
+
}
|
|
3518
|
+
// A hibernating / quiescing strand has no node to seed, so it is not running
|
|
3519
|
+
// for this pass's purposes even though its instance is still tracked.
|
|
3520
|
+
const running = new Set();
|
|
3521
|
+
for (const [strandId, instance] of this.strandManager.getInstances()) {
|
|
3522
|
+
if (instance.libp2pNode) {
|
|
3523
|
+
running.add(strandId);
|
|
3524
|
+
}
|
|
3525
|
+
}
|
|
3526
|
+
// Drop stamps for strands no longer running: the map must not grow for the
|
|
3527
|
+
// node's lifetime, and a resumed strand should refresh immediately rather
|
|
3528
|
+
// than inherit the stamp its previous incarnation left.
|
|
3529
|
+
for (const strandId of this.strandPeerAddrRefreshAt.keys()) {
|
|
3530
|
+
if (!running.has(strandId)) {
|
|
3531
|
+
this.strandPeerAddrRefreshAt.delete(strandId);
|
|
3532
|
+
}
|
|
3533
|
+
}
|
|
3534
|
+
const refreshMs = this.config.network?.controlCohort?.strandAddrRefreshMs ?? STRAND_PEER_ADDR_REFRESH_MS;
|
|
3535
|
+
const due = [...running].filter((strandId) => now - (this.strandPeerAddrRefreshAt.get(strandId) ?? 0) >= refreshMs);
|
|
3536
|
+
if (due.length === 0) {
|
|
3537
|
+
return;
|
|
3538
|
+
}
|
|
3539
|
+
// A due strand with nobody to ask leaves its throttle unstamped (see below),
|
|
3540
|
+
// so this pass re-enters on EVERY 15 s reconcile tick for as long as the node
|
|
3541
|
+
// is alone — which is the steady state of a solo node running a strand.
|
|
3542
|
+
// `connectedSiblingTargets`' membership read is unbounded, and with zero
|
|
3543
|
+
// connections its answer is empty whatever the table holds, so decide it here
|
|
3544
|
+
// from the connection list instead of paying for the read once a tick.
|
|
3545
|
+
if (this.controlNode.getConnections().length === 0) {
|
|
3546
|
+
return;
|
|
3547
|
+
}
|
|
3548
|
+
// NOTE: one strand-addr RPC per (due strand × connected sibling) per refresh
|
|
3549
|
+
// interval — each a tiny request/response on an already-open control
|
|
3550
|
+
// connection. If a node ever runs MANY strands at once, batch the RPC to
|
|
3551
|
+
// carry several strand ids per request rather than one fan-out per strand.
|
|
3552
|
+
const targets = await this.connectedSiblingTargets().catch((error) => {
|
|
3553
|
+
log('refreshStrandPeerAddrs: sibling enumeration failed (skipping pass): %o', error);
|
|
3554
|
+
return [];
|
|
3555
|
+
});
|
|
3556
|
+
// No sibling to ask (or a shutdown landed mid-enumeration): leave every stamp
|
|
3557
|
+
// untouched so the next reconcile tick retries, rather than sitting out the
|
|
3558
|
+
// whole refresh interval having asked nobody.
|
|
3559
|
+
if (targets.length === 0 || !this._running || !this.controlNode) {
|
|
3560
|
+
return;
|
|
3561
|
+
}
|
|
3562
|
+
await Promise.all(due.map((strandId) => this.refreshOneStrandPeerAddrs(strandId, targets, now)));
|
|
3563
|
+
}
|
|
3564
|
+
/**
|
|
3565
|
+
* One strand's share of {@link refreshStrandPeerAddrs}: RPC the siblings, then
|
|
3566
|
+
* merge their answers into this strand's address book. Errors are logged and
|
|
3567
|
+
* swallowed so one strand's failure never costs the others their refresh.
|
|
3568
|
+
*/
|
|
3569
|
+
async refreshOneStrandPeerAddrs(strandId, targets, now) {
|
|
3570
|
+
const controlNode = this.controlNode;
|
|
3571
|
+
const strandNode = this.strandManager.getInstance(strandId)?.libp2pNode;
|
|
3572
|
+
if (!controlNode || !strandNode) {
|
|
3573
|
+
return;
|
|
3574
|
+
}
|
|
3575
|
+
try {
|
|
3576
|
+
// The running strand node's own peerId is the delegate to announce — see
|
|
3577
|
+
// the relationship note on refreshStrandPeerAddrs.
|
|
3578
|
+
const addrs = await collectStrandAddrs(controlNode, [...targets], strandId, {
|
|
3579
|
+
delegatePeerId: strandNode.peerId.toString()
|
|
3580
|
+
});
|
|
3581
|
+
// Stamped on the RPC having happened, not on its answer: the fan-out is
|
|
3582
|
+
// what the throttle exists to bound, and an all-empty round is a normal
|
|
3583
|
+
// steady state for a strand no connected sibling currently runs.
|
|
3584
|
+
this.strandPeerAddrRefreshAt.set(strandId, now);
|
|
3585
|
+
// Re-read the instance after the await — a strand stopped mid-pass (or one
|
|
3586
|
+
// already restarted onto a new node) must never have its store written to.
|
|
3587
|
+
if (!this._running || this.strandManager.getInstance(strandId)?.libp2pNode !== strandNode) {
|
|
3588
|
+
return;
|
|
3589
|
+
}
|
|
3590
|
+
await this.mergeStrandPeerAddrs(strandNode, addrs, strandId);
|
|
3591
|
+
}
|
|
3592
|
+
catch (error) {
|
|
3593
|
+
log('refreshStrandPeerAddrs: strand %s refresh failed (continuing): %o', strandId, error);
|
|
3594
|
+
}
|
|
3595
|
+
}
|
|
3596
|
+
/**
|
|
3597
|
+
* Merge strand-network addresses into ONE strand node's libp2p address book,
|
|
3598
|
+
* attributed per peer. Best-effort throughout: an address-book write must never
|
|
3599
|
+
* fail a launch, a resume, or a reconcile pass.
|
|
3600
|
+
*
|
|
3601
|
+
* `addrs` is the peer-agnostic union the strand-addr RPC returns, so
|
|
3602
|
+
* {@link groupAddrsByPeerId} attributes each entry to the **strand transport**
|
|
3603
|
+
* peerId in its final `/p2p/` component — never the sibling's control peerId,
|
|
3604
|
+
* which names a different libp2p node entirely. Entries that name no peer are
|
|
3605
|
+
* dropped there and counted here, once per pass.
|
|
3606
|
+
*
|
|
3607
|
+
* NOTE: a member could answer with arbitrary multiaddrs bound to arbitrary peer
|
|
3608
|
+
* ids and poison this address book. That is the same exposure the launch-time
|
|
3609
|
+
* `bootstrapNodes` seeding already accepts, and the cost is bounded: an address
|
|
3610
|
+
* grants no authority, the dialed peer authenticates by peer id at the
|
|
3611
|
+
* handshake, and a bad entry costs one failed dial that ages out at the
|
|
3612
|
+
* peerStore's one-hour expiry. No new gating here — cross-party strand trust is
|
|
3613
|
+
* `backlog/strand-network-nat-relay-reachability`.
|
|
3614
|
+
*/
|
|
3615
|
+
async mergeStrandPeerAddrs(strandNode, addrs, strandId) {
|
|
3616
|
+
const counts = { merged: 0, restamped: 0, skipped: 0, failed: 0 };
|
|
3617
|
+
let grouped = 0;
|
|
3618
|
+
let peers = 0;
|
|
3619
|
+
try {
|
|
3620
|
+
const groups = groupAddrsByPeerId(addrs);
|
|
3621
|
+
peers = groups.size;
|
|
3622
|
+
const selfId = strandNode.peerId.toString();
|
|
3623
|
+
for (const [peerId, peerAddrs] of groups) {
|
|
3624
|
+
grouped += peerAddrs.length;
|
|
3625
|
+
// We never RPC ourselves, so self should never appear — belt and braces:
|
|
3626
|
+
// a node must not write its own addresses into its own address book.
|
|
3627
|
+
if (peerId === selfId) {
|
|
3628
|
+
continue;
|
|
3629
|
+
}
|
|
3630
|
+
counts[await mergePeerAddrs(strandNode, peerId, peerAddrs)]++;
|
|
3631
|
+
}
|
|
3632
|
+
}
|
|
3633
|
+
catch (error) {
|
|
3634
|
+
// `mergePeerAddrs` folds its own failures, so reaching here means the node
|
|
3635
|
+
// itself is unusable (torn down mid-merge). Never fail a launch/resume or a
|
|
3636
|
+
// reconcile pass over an address book.
|
|
3637
|
+
log('strand %s address book merge failed (continuing): %o', strandId, error);
|
|
3638
|
+
}
|
|
3639
|
+
log('strand %s address book merged (peers=%d, merged=%d, restamped=%d, skipped=%d, failed=%d, dropped=%d)', strandId, peers, counts.merged, counts.restamped, counts.skipped, counts.failed, addrs.length - grouped);
|
|
1967
3640
|
}
|
|
1968
3641
|
/**
|
|
1969
3642
|
* The local strand instance's dialable strand-network multiaddrs for the
|
|
@@ -1982,16 +3655,37 @@ export class CadreNode {
|
|
|
1982
3655
|
return orderSignalingFirst(node.getMultiaddrs().map((ma) => ma.toString()));
|
|
1983
3656
|
}
|
|
1984
3657
|
/**
|
|
1985
|
-
*
|
|
3658
|
+
* Stop a strand on THIS node only: untrack it from hibernation, drop its sApp config,
|
|
3659
|
+
* stop the local instance, and emit `strand:stopped`. The shared `Strand` row is left
|
|
3660
|
+
* intact, so on the next node restart (or watcher poll) the strand is rediscovered and
|
|
3661
|
+
* surfaces as `strand:discovered` again. Party-wide removal is {@link unpublishStrand}.
|
|
1986
3662
|
*/
|
|
1987
|
-
async
|
|
3663
|
+
async stopStrand(strandId) {
|
|
1988
3664
|
if (!this._running) {
|
|
1989
3665
|
throw new Error('CadreNode not running');
|
|
1990
3666
|
}
|
|
1991
|
-
|
|
3667
|
+
await this.detachStrand(strandId);
|
|
3668
|
+
}
|
|
3669
|
+
/**
|
|
3670
|
+
* Local teardown for one strand, shared by the caller-driven {@link stopStrand} and the
|
|
3671
|
+
* watcher-driven `handleStrandRemoved`: untrack hibernation, drop the sApp config, stop
|
|
3672
|
+
* the instance, emit `strand:stopped`. Touches no control-plane row — which side of the
|
|
3673
|
+
* removal the node is on is the caller's concern, not this method's.
|
|
3674
|
+
*
|
|
3675
|
+
* The stop + emit are skipped when the strand manager holds no instance for `strandId` —
|
|
3676
|
+
* e.g. a party owner that published a strand's row but never ran it locally, or an
|
|
3677
|
+
* explicit {@link stopStrand} for an id this node never started. `hibernationManager`
|
|
3678
|
+
* untrack and the `sAppConfigs` delete stay unconditional (both are no-ops when there is
|
|
3679
|
+
* nothing to remove), so a launch that failed before an instance was ever tracked still
|
|
3680
|
+
* gets its stray sApp config cleared.
|
|
3681
|
+
*/
|
|
3682
|
+
async detachStrand(strandId) {
|
|
1992
3683
|
this.hibernationManager.untrackStrand(strandId);
|
|
1993
|
-
// Remove sApp config
|
|
1994
3684
|
this.sAppConfigs.delete(strandId);
|
|
3685
|
+
if (!this.strandManager.hasStrand(strandId)) {
|
|
3686
|
+
log('detachStrand: strand %s not tracked locally — no-op (no stop, no strand:stopped)', strandId);
|
|
3687
|
+
return;
|
|
3688
|
+
}
|
|
1995
3689
|
await this.strandManager.stopStrand(strandId);
|
|
1996
3690
|
this.emit('strand:stopped', { strandId });
|
|
1997
3691
|
}
|
|
@@ -2175,12 +3869,86 @@ export class CadreNode {
|
|
|
2175
3869
|
getControlNode() {
|
|
2176
3870
|
return this.controlNode;
|
|
2177
3871
|
}
|
|
3872
|
+
/**
|
|
3873
|
+
* Start keeping a relay reservation: dial the given relay(s) from the control
|
|
3874
|
+
* node, ask the first one that answers for a reservation slot, wait until the
|
|
3875
|
+
* resulting `/p2p-circuit` address makes this node dialable — and keep a
|
|
3876
|
+
* supervisor running that re-drives whenever the reservation is later lost.
|
|
3877
|
+
*
|
|
3878
|
+
* For nodes that listen on the bare `/p2p-circuit` SEARCH addr (a browser tab)
|
|
3879
|
+
* rather than a configured `network.relayAddrs` circuit listener — see
|
|
3880
|
+
* `relay-reservation.ts` for why the two are alternatives and why configuring
|
|
3881
|
+
* both is a regression. Opt-in: nothing calls this for the CLI/host nodes.
|
|
3882
|
+
*
|
|
3883
|
+
* Resolves once the FIRST attempt has settled, exactly as it did when it drove
|
|
3884
|
+
* once; the retries continue in the background until {@link stop} or the next
|
|
3885
|
+
* `reserveRelays` call. Fail-soft — never throws. An unreachable relay, a
|
|
3886
|
+
* missing control node, or a timeout all resolve to a non-`reserved` status, so
|
|
3887
|
+
* a caller can await this during startup without a dead relay aborting the node.
|
|
3888
|
+
*
|
|
3889
|
+
* Passing an empty list stops the supervisor and resets the posture to `none`
|
|
3890
|
+
* without dialing or waiting.
|
|
3891
|
+
*/
|
|
3892
|
+
async reserveRelays(addrs, opts) {
|
|
3893
|
+
// Stop first, unconditionally: a second call with a different list must not
|
|
3894
|
+
// leave two loops fighting over one pending reservation slot.
|
|
3895
|
+
this.relayReserveSupervisor?.stop();
|
|
3896
|
+
this.relayReserveSupervisor = null;
|
|
3897
|
+
this.relayReserveAddrs = [...addrs];
|
|
3898
|
+
this.relayReserveError = null;
|
|
3899
|
+
if (addrs.length === 0) {
|
|
3900
|
+
return this.getRelayReservationState();
|
|
3901
|
+
}
|
|
3902
|
+
if (!this.controlNode) {
|
|
3903
|
+
// No node to supervise, so this is the one failure the node records itself.
|
|
3904
|
+
this.relayReserveError = 'control node unavailable';
|
|
3905
|
+
return this.getRelayReservationState();
|
|
3906
|
+
}
|
|
3907
|
+
const supervisor = superviseRelayReservation(this.controlNode, addrs, opts);
|
|
3908
|
+
this.relayReserveSupervisor = supervisor;
|
|
3909
|
+
await supervisor.firstAttempt;
|
|
3910
|
+
// Read through `this`, not `supervisor`: an overlapping call may have replaced
|
|
3911
|
+
// it while the first attempt was in flight, and the newest list is the truth.
|
|
3912
|
+
return this.getRelayReservationState();
|
|
3913
|
+
}
|
|
3914
|
+
/**
|
|
3915
|
+
* The node's CURRENT relay-reservation posture, recomputed from the control
|
|
3916
|
+
* node's live multiaddrs on every call.
|
|
3917
|
+
*
|
|
3918
|
+
* Deliberately not memoised: a reservation can be lost after
|
|
3919
|
+
* {@link reserveRelays} succeeds (the relay restarts, the connection drops) and
|
|
3920
|
+
* a cached `reserved` would let a caller mint invitations carrying circuit
|
|
3921
|
+
* addresses that no longer route.
|
|
3922
|
+
*
|
|
3923
|
+
* A lost reservation now recovers on its own — the supervisor {@link reserveRelays}
|
|
3924
|
+
* starts re-drives on a backoff, reporting `retrying` meanwhile. `error` here
|
|
3925
|
+
* means nothing is going to try again: no supervisor, or no control node.
|
|
3926
|
+
*/
|
|
3927
|
+
getRelayReservationState() {
|
|
3928
|
+
const supervisor = this.relayReserveSupervisor;
|
|
3929
|
+
return resolveRelayReservationState(this.controlNode, this.relayReserveAddrs, supervisor ? supervisor.lastError : this.relayReserveError, supervisor?.driving ?? false, supervisor?.retryAtMs ?? null);
|
|
3930
|
+
}
|
|
2178
3931
|
/**
|
|
2179
3932
|
* Get the control database (for advanced queries)
|
|
2180
3933
|
*/
|
|
2181
3934
|
getControlDatabase() {
|
|
2182
3935
|
return this.controlDatabase;
|
|
2183
3936
|
}
|
|
3937
|
+
/**
|
|
3938
|
+
* Open control-network connections right now. A lower-bound proxy for replication
|
|
3939
|
+
* reach: 0 connections ⇒ a control write commits local-only. The private
|
|
3940
|
+
* `committedAlone` write-while-alone seam is defined in terms of this.
|
|
3941
|
+
*
|
|
3942
|
+
* It approximates the precise signal (the block's cluster size), and a caller that
|
|
3943
|
+
* samples it AFTER a write samples a slightly wider window than `committedAlone`
|
|
3944
|
+
* does inside the write itself. It exists so an embedder can warn an owner before a
|
|
3945
|
+
* control write that this machine currently sees none of its siblings, and report
|
|
3946
|
+
* after one that the write may not have travelled — a warning shown unconditionally
|
|
3947
|
+
* is a warning people learn to ignore.
|
|
3948
|
+
*/
|
|
3949
|
+
getControlConnectionCount() {
|
|
3950
|
+
return this.controlNode?.getConnections().length ?? 0;
|
|
3951
|
+
}
|
|
2184
3952
|
/**
|
|
2185
3953
|
* Force a poll of the strand watcher (for testing)
|
|
2186
3954
|
*/
|
|
@@ -2206,17 +3974,36 @@ export class CadreNode {
|
|
|
2206
3974
|
if (!this.controlNode || !this.controlDatabase) {
|
|
2207
3975
|
throw new Error('CadreNode must be started before initializing seed bootstrap');
|
|
2208
3976
|
}
|
|
3977
|
+
// A node wiring seed-bootstrap with an owner PRIVATE key is declaring that
|
|
3978
|
+
// key an authority of its own party (founder genesis / self-signing owner)
|
|
3979
|
+
// — anchor it in the node-local store. The in-memory set updates
|
|
3980
|
+
// synchronously (see TrustedOwnerStore.trust); only the file persist is
|
|
3981
|
+
// fire-and-forget, logged on failure.
|
|
3982
|
+
// NOTE: non-founder members that also wire seed-bootstrap with their own
|
|
3983
|
+
// derived key (e.g. the phone joiner path in runOwnerGenesis, which needs
|
|
3984
|
+
// it to self-publish its CadrePeer row) self-anchor a key that is not a
|
|
3985
|
+
// party authority. Harmless while such a node never mints an invite: the
|
|
3986
|
+
// store does not replicate, so a node trusting itself grants nothing to
|
|
3987
|
+
// others. But `createInvite` now hands out the anchor's contents as the
|
|
3988
|
+
// invitee's pins, so the moment a non-founder member mints an invite it
|
|
3989
|
+
// exports its own non-authority key as a cadre owner key. If that becomes
|
|
3990
|
+
// reachable (today only cadre-cli/cadre-host owners mint invites), gate this
|
|
3991
|
+
// self-anchor on the actual OwnerKey genesis insert instead.
|
|
3992
|
+
if (this.trustedOwnerStore) {
|
|
3993
|
+
void this.trustedOwnerStore
|
|
3994
|
+
.trust(ed25519PublicKeyFromPrivate(ownerPrivateKey), 'genesis')
|
|
3995
|
+
.catch((error) => log('Trusted-owner genesis anchor persist failed: %o', error));
|
|
3996
|
+
}
|
|
2209
3997
|
this.seedBootstrapService = new SeedBootstrapService({
|
|
2210
3998
|
partyId: this.config.controlNetwork.partyId,
|
|
2211
3999
|
ownerPrivateKey,
|
|
2212
4000
|
inviteAddressResolver: () => this.resolveInviteAddresses(),
|
|
2213
4001
|
trustPolicy: this.config.seedTrustPolicy,
|
|
4002
|
+
// Seed trust anchors on the node-local store (seeded just above with this
|
|
4003
|
+
// node's own genesis key), never on the replicated OwnerKey table.
|
|
4004
|
+
...(this.trustedOwnerStore ? { trustedOwners: this.trustedOwnerStore } : {}),
|
|
2214
4005
|
});
|
|
2215
|
-
this.seedBootstrapService.setEventCallbacks(
|
|
2216
|
-
onSeedReceived: (partyId, peerId) => this.emit('seed:received', { partyId, peerId }),
|
|
2217
|
-
onSeedApplied: (partyId, peersAdded) => this.emit('seed:applied', { partyId, peersAdded }),
|
|
2218
|
-
onSeedError: (partyId, error) => this.emit('seed:error', { partyId, error }),
|
|
2219
|
-
});
|
|
4006
|
+
this.seedBootstrapService.setEventCallbacks(this.seedEventCallbacks());
|
|
2220
4007
|
this.seedBootstrapService.initialize(this.controlNode, this.controlDatabase);
|
|
2221
4008
|
log('Seed bootstrap service initialized');
|
|
2222
4009
|
}
|
|
@@ -2243,7 +4030,10 @@ export class CadreNode {
|
|
|
2243
4030
|
return this.getMultiaddrs();
|
|
2244
4031
|
}
|
|
2245
4032
|
/**
|
|
2246
|
-
* Enumerate the cadre's `CadrePeer` membership
|
|
4033
|
+
* Enumerate the cadre's `CadrePeer` membership — the ADDRESSABLE surface (see
|
|
4034
|
+
* {@link isMember}). Rows whose stamp is retired in `CadreControl.Revocation`
|
|
4035
|
+
* never reach here: {@link ControlDatabase.queryCadrePeers} excludes them, so a
|
|
4036
|
+
* revoked peer is not addressable either.
|
|
2247
4037
|
*/
|
|
2248
4038
|
async listMembers() {
|
|
2249
4039
|
if (!this.controlDatabase) {
|
|
@@ -2256,7 +4046,12 @@ export class CadreNode {
|
|
|
2256
4046
|
*
|
|
2257
4047
|
* This is the ADDRESSABLE surface ("do I have a dialable address record for this
|
|
2258
4048
|
* peer") — it includes this node's own self-published row. Address resolution and
|
|
2259
|
-
* push fan-out use this.
|
|
4049
|
+
* push fan-out use this. A peer whose stamp is retired in `CadreControl.Revocation`
|
|
4050
|
+
* is excluded here too (the filter lives in
|
|
4051
|
+
* {@link ControlDatabase.queryCadrePeers}): revocation removes a peer from the
|
|
4052
|
+
* addressable surface, not only the trust-facing one, so a revoked peer is no
|
|
4053
|
+
* longer dialed, RPC'd, or handed out as an address. The trust-facing gate is
|
|
4054
|
+
* {@link isAuthorizedMember}.
|
|
2260
4055
|
*/
|
|
2261
4056
|
async isMember(peerId) {
|
|
2262
4057
|
const members = await this.listMembers();
|
|
@@ -2264,26 +4059,84 @@ export class CadreNode {
|
|
|
2264
4059
|
}
|
|
2265
4060
|
/**
|
|
2266
4061
|
* Enumerate the party's AUTHORIZED members — the trust-facing set, distinct from
|
|
2267
|
-
* the addressable set ({@link listMembers}).
|
|
2268
|
-
*
|
|
2269
|
-
*
|
|
4062
|
+
* the addressable set ({@link listMembers}). A peer is authorized iff ALL hold:
|
|
4063
|
+
*
|
|
4064
|
+
* 1. it is not this node itself — a node publishes its own `CadrePeer` address
|
|
4065
|
+
* row so its dialable address rides in seeds, but "self" is not a peer this
|
|
4066
|
+
* node authorized;
|
|
4067
|
+
* 2. its `CadrePeer` row carries a complete voucher (`StampId`, `VouchOwner`,
|
|
4068
|
+
* `VouchSig` all non-null);
|
|
4069
|
+
* 3. `VouchOwner` is in the NODE-LOCAL trusted-owner anchor
|
|
4070
|
+
* ({@link getTrustedOwnerStore}) — never the replicated `OwnerKey` table,
|
|
4071
|
+
* which any stranger can genesis-pollute; and
|
|
4072
|
+
* 4. `VouchSig` verifies as that owner's signature over the row's voucher
|
|
4073
|
+
* digest ({@link verifyCadrePeerVoucher}), so the anchored owner really
|
|
4074
|
+
* vouched THIS peer id under THIS row's nonce; and
|
|
4075
|
+
* 5. the row's `StampId` is NOT retired in `CadreControl.Revocation` — enforced
|
|
4076
|
+
* upstream in {@link ControlDatabase.queryCadrePeers}, which drops retired rows
|
|
4077
|
+
* before ANY reader (this predicate included) sees them, so a row resurrected
|
|
4078
|
+
* by replaying the captured admission approval on a node that had not yet
|
|
4079
|
+
* converged on the tombstone (the write-time `NotRevoked` CHECK only sees
|
|
4080
|
+
* local rows) is still inert to every reader that has the tombstone.
|
|
4081
|
+
*
|
|
4082
|
+
* Fail-closed at every step: a missing anchor (pre-start), an empty anchor (a
|
|
4083
|
+
* not-yet-enrolled node authorizes no one), a null/partial voucher, an
|
|
4084
|
+
* unanchored `VouchOwner`, or a bad signature all yield "not authorized" —
|
|
4085
|
+
* having an address row is NOT membership. The control-network wake and
|
|
2270
4086
|
* strand-address gates consult this set, NOT the addressable one.
|
|
2271
4087
|
*
|
|
2272
|
-
* NOTE:
|
|
2273
|
-
*
|
|
2274
|
-
*
|
|
2275
|
-
*
|
|
2276
|
-
*
|
|
4088
|
+
* NOTE (rotation): if a party owner rotates keys and only the NEW key is pinned
|
|
4089
|
+
* in the anchor, rows the OLD key vouched fail check 3 until re-vouched — a
|
|
4090
|
+
* legit member goes un-authorized on readers that only pin the new key. Full
|
|
4091
|
+
* rotation handling (re-vouch on rotate) is the
|
|
4092
|
+
* `flip-strand-membership-rotation-known-gap` work, not this predicate's.
|
|
2277
4093
|
*/
|
|
2278
4094
|
async listAuthorizedMembers() {
|
|
4095
|
+
if (!this.controlDatabase) {
|
|
4096
|
+
throw new Error('CadreNode must be started before listing members');
|
|
4097
|
+
}
|
|
2279
4098
|
const selfPeerId = this.peerId?.toString();
|
|
2280
|
-
const
|
|
2281
|
-
|
|
4099
|
+
const rows = await this.controlDatabase.queryCadrePeers();
|
|
4100
|
+
const authorized = rows
|
|
4101
|
+
.filter(row => row.peerId !== selfPeerId && this.hasAnchoredVoucher(row))
|
|
4102
|
+
.map(({ peerId, multiaddr }) => ({ peerId, multiaddr }));
|
|
4103
|
+
// A node whose anchor was never seeded (no invite pin, no operator pin, not a
|
|
4104
|
+
// founder) refuses every wake and strand-addr request, which from the outside
|
|
4105
|
+
// looks like an unexplained "non-member" rejection. Say so once per call rather
|
|
4106
|
+
// than leaving the operator to infer it from silence.
|
|
4107
|
+
if (authorized.length === 0 && rows.length > 0 && (this.trustedOwnerStore?.all().size ?? 0) === 0) {
|
|
4108
|
+
log('listAuthorizedMembers: %d CadrePeer row(s) but the node-local trusted-owner anchor is EMPTY — authorizing no one (this node has no invite/operator owner-key pin)', rows.length);
|
|
4109
|
+
}
|
|
4110
|
+
return authorized;
|
|
4111
|
+
}
|
|
4112
|
+
/**
|
|
4113
|
+
* Checks 2–4 of the authorized-membership predicate (see
|
|
4114
|
+
* {@link listAuthorizedMembers}) for one `CadrePeer` row: complete voucher,
|
|
4115
|
+
* `VouchOwner` in the node-local anchor, signature valid over the row's
|
|
4116
|
+
* (PeerId, StampId) voucher digest.
|
|
4117
|
+
*
|
|
4118
|
+
* NOTE: verifies the ed25519 signature on every call (no memo of already-
|
|
4119
|
+
* verified (peerId, stampId, vouchSig) triples). Cadres are a handful of
|
|
4120
|
+
* devices and the gates run per inbound request, so this is cheap today; if
|
|
4121
|
+
* membership or gate traffic ever grows, cache verified triples keyed on the
|
|
4122
|
+
* row's `StampId` (which rotates with every re-vouch).
|
|
4123
|
+
*/
|
|
4124
|
+
hasAnchoredVoucher(row) {
|
|
4125
|
+
if (row.stampId === null || row.vouchOwner === null || row.vouchSig === null) {
|
|
4126
|
+
return false;
|
|
4127
|
+
}
|
|
4128
|
+
if (!this.trustedOwnerStore?.has(row.vouchOwner)) {
|
|
4129
|
+
return false;
|
|
4130
|
+
}
|
|
4131
|
+
return verifyCadrePeerVoucher(row.peerId, row.stampId, row.vouchOwner, row.vouchSig);
|
|
2282
4132
|
}
|
|
2283
4133
|
/**
|
|
2284
4134
|
* Probe whether a given peer is an AUTHORIZED party member (see
|
|
2285
4135
|
* {@link listAuthorizedMembers}) — the gate the control-network wake and
|
|
2286
|
-
* strand-address responders consult, NOT {@link isMember} (the addressable
|
|
4136
|
+
* strand-address responders consult, NOT {@link isMember} (the addressable
|
|
4137
|
+
* surface). Deliberately scans the full membership (one `CadrePeer` query)
|
|
4138
|
+
* rather than adding a single-row read path: cadres are small, and one code
|
|
4139
|
+
* path keeps the predicate impossible to drift from the list.
|
|
2287
4140
|
*/
|
|
2288
4141
|
async isAuthorizedMember(peerId) {
|
|
2289
4142
|
const members = await this.listAuthorizedMembers();
|
|
@@ -2334,12 +4187,12 @@ export class CadreNode {
|
|
|
2334
4187
|
// No owner key - this node only receives seeds
|
|
2335
4188
|
inviteAddressResolver: () => this.resolveInviteAddresses(),
|
|
2336
4189
|
trustPolicy: this.config.seedTrustPolicy,
|
|
4190
|
+
// A listener-only node accepts a wire-delivered seed solely against this
|
|
4191
|
+
// anchor (there is no per-call override on the inbound handler): with no
|
|
4192
|
+
// genesis/invite/operator pin it authorizes nobody, which is the point.
|
|
4193
|
+
...(this.trustedOwnerStore ? { trustedOwners: this.trustedOwnerStore } : {}),
|
|
2337
4194
|
});
|
|
2338
|
-
this.seedBootstrapService.setEventCallbacks(
|
|
2339
|
-
onSeedReceived: (partyId, peerId) => this.emit('seed:received', { partyId, peerId }),
|
|
2340
|
-
onSeedApplied: (partyId, peersAdded) => this.emit('seed:applied', { partyId, peersAdded }),
|
|
2341
|
-
onSeedError: (partyId, error) => this.emit('seed:error', { partyId, error }),
|
|
2342
|
-
});
|
|
4195
|
+
this.seedBootstrapService.setEventCallbacks(this.seedEventCallbacks());
|
|
2343
4196
|
this.seedBootstrapService.initialize(this.controlNode, this.controlDatabase);
|
|
2344
4197
|
log('Seed listener enabled');
|
|
2345
4198
|
}
|
|
@@ -2349,6 +4202,122 @@ export class CadreNode {
|
|
|
2349
4202
|
getSeedBootstrapService() {
|
|
2350
4203
|
return this.seedBootstrapService;
|
|
2351
4204
|
}
|
|
4205
|
+
/**
|
|
4206
|
+
* The event callbacks every {@link SeedBootstrapService} this node owns is
|
|
4207
|
+
* wired with — shared by the owner-capable service ({@link initializeSeedBootstrap})
|
|
4208
|
+
* and the listener-only one ({@link enableSeedListener}) so the two cannot drift.
|
|
4209
|
+
*
|
|
4210
|
+
* A seed applied by the INBOUND protocol handler writes no `CadrePeer` row of
|
|
4211
|
+
* its own (it merges the libp2p peer store and dials owners), so the automatic
|
|
4212
|
+
* write-driven refresh never fires for it — yet applying it can ANCHOR a new
|
|
4213
|
+
* owner key, which flips rows already present from unauthorized to authorized.
|
|
4214
|
+
* Hence the explicit refresh here; without it a peer the freshly anchored owner
|
|
4215
|
+
* vouched for stays denied until the next timed cohort reconcile.
|
|
4216
|
+
*/
|
|
4217
|
+
seedEventCallbacks() {
|
|
4218
|
+
return {
|
|
4219
|
+
onSeedReceived: (partyId, peerId) => this.emit('seed:received', { partyId, peerId }),
|
|
4220
|
+
onSeedApplied: (partyId, peersAdded, seed) => {
|
|
4221
|
+
this.emit('seed:applied', { partyId, peersAdded });
|
|
4222
|
+
// A wire-delivered seed never passes through the `applySeed` wrapper, so
|
|
4223
|
+
// this is where its owner peers become cold-start bootstrap targets.
|
|
4224
|
+
this.recordSeedBootstrapPeers(seed);
|
|
4225
|
+
void this.refreshMembershipGate('seed-applied');
|
|
4226
|
+
},
|
|
4227
|
+
onSeedError: (partyId, error) => this.emit('seed:error', { partyId, error }),
|
|
4228
|
+
};
|
|
4229
|
+
}
|
|
4230
|
+
/**
|
|
4231
|
+
* Re-materialize the authorized-peer snapshot that the fail-closed per-stream
|
|
4232
|
+
* control-DB gate ({@link authorizeInboundControlStream}) judges against.
|
|
4233
|
+
*
|
|
4234
|
+
* NO `CadrePeer` writer needs to call this: the control database notifies it
|
|
4235
|
+
* after every committed member-row write (`ControlDatabase.mutateCadrePeer`,
|
|
4236
|
+
* wired in {@link start}), which is exactly what makes the refresh automatic
|
|
4237
|
+
* rather than a caller obligation. It stays public for the changes that write
|
|
4238
|
+
* NO row locally and so raise no notification:
|
|
4239
|
+
*
|
|
4240
|
+
* - a membership row that arrived by REPLICATION (otherwise picked up on the
|
|
4241
|
+
* next timed cohort reconcile — bounded staleness by design), and
|
|
4242
|
+
* - a newly anchored trusted owner key ({@link applySeed}), which flips rows
|
|
4243
|
+
* ALREADY present from unauthorized to authorized without touching them.
|
|
4244
|
+
*
|
|
4245
|
+
* Coalescing: marks the snapshot stale and resolves once a refresh that began
|
|
4246
|
+
* after this call has completed, so a caller that awaits it always observes
|
|
4247
|
+
* its own change. Concurrent callers share one read. Idempotent and
|
|
4248
|
+
* best-effort (a failed read keeps the previous snapshot); never rejects.
|
|
4249
|
+
*/
|
|
4250
|
+
async refreshMembershipGate(reason = 'external-write') {
|
|
4251
|
+
this.membershipGateDirty = true;
|
|
4252
|
+
if (this.membershipGateDeferDepth > 0) {
|
|
4253
|
+
// Flushed once when the enclosing scope exits.
|
|
4254
|
+
return;
|
|
4255
|
+
}
|
|
4256
|
+
while (this.membershipGateDirty) {
|
|
4257
|
+
// Re-checked after the await: the drain may have taken its last look at
|
|
4258
|
+
// the flag before we set it, in which case we need a fresh one.
|
|
4259
|
+
this.membershipGateDrain ?? (this.membershipGateDrain = this.drainMembershipGate(reason));
|
|
4260
|
+
await this.membershipGateDrain;
|
|
4261
|
+
}
|
|
4262
|
+
}
|
|
4263
|
+
/**
|
|
4264
|
+
* The single in-flight refresh loop behind {@link refreshMembershipGate}: read
|
|
4265
|
+
* until the stale flag stays clear, then release the slot.
|
|
4266
|
+
*
|
|
4267
|
+
* NOTE: correct only because {@link membershipGateDirty} is guaranteed TRUE at
|
|
4268
|
+
* entry — its sole caller sets it synchronously, with no await in between. Were
|
|
4269
|
+
* this ever invoked with the flag already clear, the body would never suspend,
|
|
4270
|
+
* so the `finally` would null the slot BEFORE the caller's `??=` filled it, and
|
|
4271
|
+
* the slot would be left holding an already-settled drain that every later
|
|
4272
|
+
* refresh reuses — an unbreakable await/re-check spin. If a second caller is
|
|
4273
|
+
* ever added, have it mark the flag first (or assert it here).
|
|
4274
|
+
*/
|
|
4275
|
+
async drainMembershipGate(reason) {
|
|
4276
|
+
try {
|
|
4277
|
+
while (this.membershipGateDirty) {
|
|
4278
|
+
// Cleared BEFORE the read (not after) so a write landing mid-read marks
|
|
4279
|
+
// the snapshot stale again and earns another pass, and so a throw still
|
|
4280
|
+
// terminates the loop.
|
|
4281
|
+
this.membershipGateDirty = false;
|
|
4282
|
+
await this.refreshAuthorizedControlPeers(reason);
|
|
4283
|
+
}
|
|
4284
|
+
}
|
|
4285
|
+
catch (error) {
|
|
4286
|
+
// Upholds `refreshMembershipGate`'s never-rejects contract. The read helper
|
|
4287
|
+
// already swallows its own failures, so this is defensive — but a rejection
|
|
4288
|
+
// here would otherwise surface out of every awaiting writer.
|
|
4289
|
+
log('membership gate drain (%s) failed — keeping previous snapshot: %o', reason, error);
|
|
4290
|
+
}
|
|
4291
|
+
finally {
|
|
4292
|
+
this.membershipGateDrain = null;
|
|
4293
|
+
}
|
|
4294
|
+
}
|
|
4295
|
+
/**
|
|
4296
|
+
* Collapse a burst of `CadrePeer` writes into ONE gate refresh at scope exit.
|
|
4297
|
+
*
|
|
4298
|
+
* For loops that re-touch many rows (the write-while-alone drains), where a
|
|
4299
|
+
* per-row refresh would mean one full membership read per row for a snapshot
|
|
4300
|
+
* that only has to be correct once the loop settles.
|
|
4301
|
+
*
|
|
4302
|
+
* The depth counter is instance-level, so it also suppresses a genuinely
|
|
4303
|
+
* CONCURRENT external write's refresh for the life of the scope: that writer's
|
|
4304
|
+
* promise resolves before its peer is admitted, and admission lands at scope
|
|
4305
|
+
* exit instead. Acceptable — scope exit is milliseconds away (bounded by the
|
|
4306
|
+
* drain), versus the ~15 s reconcile interval that was the alternative before
|
|
4307
|
+
* any of this existed — but it is the reason these scopes stay short and rare.
|
|
4308
|
+
*/
|
|
4309
|
+
async deferMembershipGateRefresh(reason, body) {
|
|
4310
|
+
this.membershipGateDeferDepth++;
|
|
4311
|
+
try {
|
|
4312
|
+
return await body();
|
|
4313
|
+
}
|
|
4314
|
+
finally {
|
|
4315
|
+
this.membershipGateDeferDepth--;
|
|
4316
|
+
if (this.membershipGateDeferDepth === 0 && this.membershipGateDirty) {
|
|
4317
|
+
await this.refreshMembershipGate(reason);
|
|
4318
|
+
}
|
|
4319
|
+
}
|
|
4320
|
+
}
|
|
2352
4321
|
/**
|
|
2353
4322
|
* Authorize a new peer to join the cadre.
|
|
2354
4323
|
* Signs the peer ID with the owner key and inserts into CadrePeer table.
|
|
@@ -2360,6 +4329,8 @@ export class CadreNode {
|
|
|
2360
4329
|
if (!this.seedBootstrapService) {
|
|
2361
4330
|
throw new Error('Seed bootstrap service not initialized. Call initializeSeedBootstrap() first.');
|
|
2362
4331
|
}
|
|
4332
|
+
// The insert notifies the control DB's membership hub, so the per-stream gate
|
|
4333
|
+
// has already admitted the just-vouched peer by the time this resolves.
|
|
2363
4334
|
await this.seedBootstrapService.authorizePeer({ peerId, multiaddrs });
|
|
2364
4335
|
// Queue for re-replication if this committed local-only (no connected cohort).
|
|
2365
4336
|
this.noteControlWrite(peerId, 'authorize');
|
|
@@ -2374,6 +4345,8 @@ export class CadreNode {
|
|
|
2374
4345
|
if (!this.seedBootstrapService) {
|
|
2375
4346
|
throw new Error('Seed bootstrap service not initialized. Call initializeSeedBootstrap() first.');
|
|
2376
4347
|
}
|
|
4348
|
+
// The delete notifies the membership hub, so the removed peer is out of the
|
|
4349
|
+
// per-stream gate by the time this resolves.
|
|
2377
4350
|
await this.seedBootstrapService.removePeer(peerId);
|
|
2378
4351
|
// Track + loudly flag a delete that committed local-only (security-relevant).
|
|
2379
4352
|
this.noteControlWrite(peerId, 'remove');
|
|
@@ -2413,13 +4386,41 @@ export class CadreNode {
|
|
|
2413
4386
|
const tempService = new SeedBootstrapService({
|
|
2414
4387
|
partyId: seed.partyId,
|
|
2415
4388
|
trustPolicy: this.config.seedTrustPolicy,
|
|
4389
|
+
// The anchor is node-scoped, not service-scoped: a throwaway service
|
|
4390
|
+
// must consult (and persist an accepted signer into) the SAME store the
|
|
4391
|
+
// persistent one would, or a cold-start enrollment via this path would
|
|
4392
|
+
// anchor nothing and re-prompt for the pin on every later seed.
|
|
4393
|
+
...(this.trustedOwnerStore ? { trustedOwners: this.trustedOwnerStore } : {}),
|
|
2416
4394
|
});
|
|
2417
4395
|
if (this.controlNode && this.controlDatabase) {
|
|
2418
4396
|
tempService.initialize(this.controlNode, this.controlDatabase, { registerHandler: false });
|
|
2419
4397
|
}
|
|
2420
|
-
|
|
4398
|
+
const tempResult = await tempService.applySeed(seed, options);
|
|
4399
|
+
this.noteAppliedSeed(tempResult, seed);
|
|
4400
|
+
// Not a row write (so nothing notifies): applying a seed can anchor a new
|
|
4401
|
+
// owner key, which flips rows ALREADY present into the authorized set.
|
|
4402
|
+
await this.refreshMembershipGate('applySeed');
|
|
4403
|
+
return tempResult;
|
|
4404
|
+
}
|
|
4405
|
+
const result = await this.seedBootstrapService.applySeed(seed, options);
|
|
4406
|
+
this.noteAppliedSeed(result, seed);
|
|
4407
|
+
await this.refreshMembershipGate('applySeed');
|
|
4408
|
+
return result;
|
|
4409
|
+
}
|
|
4410
|
+
/**
|
|
4411
|
+
* Post-process an {@link applySeed} outcome: retain the seed's owner peers as
|
|
4412
|
+
* cold-start bootstrap targets, and surface a seed that was accepted but whose
|
|
4413
|
+
* every owner dial failed — the node is now seeded yet unconnected, and the
|
|
4414
|
+
* cold-start reconcile branch is what gets it out of that state.
|
|
4415
|
+
*/
|
|
4416
|
+
noteAppliedSeed(result, seed) {
|
|
4417
|
+
if (!result.success) {
|
|
4418
|
+
return;
|
|
4419
|
+
}
|
|
4420
|
+
this.recordSeedBootstrapPeers(seed);
|
|
4421
|
+
if (result.ownerDialsAttempted > 0 && result.ownerDialsFailed === result.ownerDialsAttempted) {
|
|
4422
|
+
log('applySeed: seeded but no owner reachable (%d/%d dial(s) failed) — cold-start reconcile will retry', result.ownerDialsFailed, result.ownerDialsAttempted);
|
|
2421
4423
|
}
|
|
2422
|
-
return await this.seedBootstrapService.applySeed(seed, options);
|
|
2423
4424
|
}
|
|
2424
4425
|
/**
|
|
2425
4426
|
* Deliver a seed directly to a peer via the /sereus/seed/1.0.0 protocol.
|
|
@@ -2470,6 +4471,8 @@ export class CadreNode {
|
|
|
2470
4471
|
if (!this.seedBootstrapService) {
|
|
2471
4472
|
throw new Error('Seed bootstrap service not initialized. Call initializeSeedBootstrap() first.');
|
|
2472
4473
|
}
|
|
4474
|
+
// The drone's `CadrePeer` insert notifies the membership hub on the way to the
|
|
4475
|
+
// seed, so the per-stream gate already admits it here.
|
|
2473
4476
|
return await this.seedBootstrapService.addDrone(options);
|
|
2474
4477
|
}
|
|
2475
4478
|
/**
|
|
@@ -2480,7 +4483,12 @@ export class CadreNode {
|
|
|
2480
4483
|
if (!this.seedBootstrapService) {
|
|
2481
4484
|
throw new Error('Seed bootstrap service not initialized. Call initializeSeedBootstrap() first.');
|
|
2482
4485
|
}
|
|
2483
|
-
|
|
4486
|
+
const result = await this.seedBootstrapService.createInvite(token, expiresIn);
|
|
4487
|
+
// The invitee dials this node BEFORE it is authorized (dialInvite →
|
|
4488
|
+
// acceptPhone), so hold the inbound connection gate open for strangers for
|
|
4489
|
+
// the invite's validity (or a bounded default when it never expires).
|
|
4490
|
+
this.openEnrollmentWindow(result.invite.expiresAt ?? Date.now() + DEFAULT_ENROLLMENT_WINDOW_MS);
|
|
4491
|
+
return result;
|
|
2484
4492
|
}
|
|
2485
4493
|
/**
|
|
2486
4494
|
* Accept a phone connection using an invite.
|
|
@@ -2490,6 +4498,8 @@ export class CadreNode {
|
|
|
2490
4498
|
if (!this.seedBootstrapService) {
|
|
2491
4499
|
throw new Error('Seed bootstrap service not initialized. Call initializeSeedBootstrap() first.');
|
|
2492
4500
|
}
|
|
4501
|
+
// The just-accepted phone opens control-DB streams next; its `CadrePeer` insert
|
|
4502
|
+
// notifies the membership hub, so it is already admitted when this resolves.
|
|
2493
4503
|
await this.seedBootstrapService.acceptPhone(options, issuedInvite);
|
|
2494
4504
|
}
|
|
2495
4505
|
/**
|
|
@@ -2500,6 +4510,10 @@ export class CadreNode {
|
|
|
2500
4510
|
if (!this.seedBootstrapService) {
|
|
2501
4511
|
throw new Error('Seed bootstrap service not initialized. Call initializeSeedBootstrap() first.');
|
|
2502
4512
|
}
|
|
4513
|
+
// The service authorizes the phone (a `CadrePeer` insert, which notifies the
|
|
4514
|
+
// membership hub) on the way to the seed, so the per-stream gate admits it
|
|
4515
|
+
// before this resolves — not on the next reconcile, by which time the phone's
|
|
4516
|
+
// own control-DB schema load may have died denied.
|
|
2503
4517
|
return await this.seedBootstrapService.addPhoneWithRelay(phonePeerId);
|
|
2504
4518
|
}
|
|
2505
4519
|
/**
|