@serfab/cadre-core 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/dist/bootstrap-peer-store-file.d.ts +13 -0
- package/dist/bootstrap-peer-store-file.js +42 -0
- package/dist/bootstrap-peer-store-file.js.map +1 -0
- package/dist/bootstrap-peer-store.d.ts +73 -0
- package/dist/bootstrap-peer-store.js +128 -0
- package/dist/bootstrap-peer-store.js.map +1 -0
- package/dist/cadre-node.d.ts +918 -53
- package/dist/cadre-node.js +1843 -190
- package/dist/cadre-node.js.map +1 -1
- package/dist/control-authorization.d.ts +93 -0
- package/dist/control-authorization.js +57 -0
- package/dist/control-authorization.js.map +1 -0
- package/dist/control-database.d.ts +990 -50
- package/dist/control-database.js +1459 -108
- package/dist/control-database.js.map +1 -1
- package/dist/control-formation-recorder.d.ts +89 -10
- package/dist/control-formation-recorder.js +129 -14
- package/dist/control-formation-recorder.js.map +1 -1
- package/dist/control-schema.d.ts +1 -1
- package/dist/control-schema.js +774 -210
- package/dist/control-schema.js.map +1 -1
- package/dist/control-stream.d.ts +38 -4
- package/dist/control-stream.js +70 -4
- package/dist/control-stream.js.map +1 -1
- package/dist/control-write-retry.d.ts +151 -0
- package/dist/control-write-retry.js +411 -0
- package/dist/control-write-retry.js.map +1 -0
- package/dist/delegate-admission.d.ts +134 -0
- package/dist/delegate-admission.js +244 -0
- package/dist/delegate-admission.js.map +1 -0
- package/dist/device-token.d.ts +11 -11
- package/dist/device-token.js +14 -13
- package/dist/device-token.js.map +1 -1
- package/dist/ed25519-key.d.ts +26 -0
- package/dist/ed25519-key.js +56 -1
- package/dist/ed25519-key.js.map +1 -1
- package/dist/file-durable-slot.d.ts +20 -0
- package/dist/file-durable-slot.js +53 -0
- package/dist/file-durable-slot.js.map +1 -0
- package/dist/formation-approval.d.ts +157 -0
- package/dist/formation-approval.js +375 -0
- package/dist/formation-approval.js.map +1 -0
- package/dist/fs-atomic.d.ts +38 -0
- package/dist/fs-atomic.js +97 -0
- package/dist/fs-atomic.js.map +1 -0
- package/dist/identity-key.d.ts +49 -0
- package/dist/identity-key.js +81 -0
- package/dist/identity-key.js.map +1 -0
- package/dist/index.d.ts +16 -6
- package/dist/index.js +39 -5
- package/dist/index.js.map +1 -1
- package/dist/key-store-file.d.ts +2 -24
- package/dist/key-store-file.js +10 -77
- package/dist/key-store-file.js.map +1 -1
- package/dist/membership-connection-gater.d.ts +146 -0
- package/dist/membership-connection-gater.js +180 -0
- package/dist/membership-connection-gater.js.map +1 -0
- package/dist/node-local-snapshot.d.ts +109 -0
- package/dist/node-local-snapshot.js +182 -0
- package/dist/node-local-snapshot.js.map +1 -0
- package/dist/peer-authorization.d.ts +135 -18
- package/dist/peer-authorization.js +152 -21
- package/dist/peer-authorization.js.map +1 -1
- package/dist/peer-record.d.ts +21 -8
- package/dist/peer-record.js +26 -10
- package/dist/peer-record.js.map +1 -1
- package/dist/relay-addrs.d.ts +45 -0
- package/dist/relay-addrs.js +103 -0
- package/dist/relay-addrs.js.map +1 -0
- package/dist/relay-reservation.d.ts +266 -0
- package/dist/relay-reservation.js +613 -0
- package/dist/relay-reservation.js.map +1 -0
- package/dist/seed-bootstrap.d.ts +221 -56
- package/dist/seed-bootstrap.js +328 -188
- package/dist/seed-bootstrap.js.map +1 -1
- package/dist/seed-trust-policy.d.ts +47 -21
- package/dist/seed-trust-policy.js +24 -15
- package/dist/seed-trust-policy.js.map +1 -1
- package/dist/strand-addr-protocol.d.ts +42 -13
- package/dist/strand-addr-protocol.js +58 -52
- package/dist/strand-addr-protocol.js.map +1 -1
- package/dist/strand-backfill.d.ts +109 -0
- package/dist/strand-backfill.js +279 -0
- package/dist/strand-backfill.js.map +1 -0
- package/dist/strand-formation-manager.d.ts +38 -9
- package/dist/strand-formation-manager.js +120 -17
- package/dist/strand-formation-manager.js.map +1 -1
- package/dist/strand-formation-protocol.d.ts +103 -4
- package/dist/strand-formation-protocol.js +232 -4
- package/dist/strand-formation-protocol.js.map +1 -1
- package/dist/strand-instance-manager.d.ts +39 -1
- package/dist/strand-instance-manager.js +92 -14
- package/dist/strand-instance-manager.js.map +1 -1
- package/dist/strand-member-registry.d.ts +5 -5
- package/dist/strand-member-registry.js +18 -10
- package/dist/strand-member-registry.js.map +1 -1
- package/dist/strand-membership-writer.d.ts +482 -88
- package/dist/strand-membership-writer.js +757 -136
- package/dist/strand-membership-writer.js.map +1 -1
- package/dist/strand-solicitation.d.ts +99 -33
- package/dist/strand-solicitation.js +100 -52
- package/dist/strand-solicitation.js.map +1 -1
- package/dist/strand-transport-key.d.ts +40 -0
- package/dist/strand-transport-key.js +56 -0
- package/dist/strand-transport-key.js.map +1 -0
- package/dist/strand-wake-protocol.d.ts +9 -7
- package/dist/strand-wake-protocol.js +22 -46
- package/dist/strand-wake-protocol.js.map +1 -1
- package/dist/strand-watcher.d.ts +17 -2
- package/dist/strand-watcher.js +47 -3
- package/dist/strand-watcher.js.map +1 -1
- package/dist/trusted-owner-store-file.d.ts +12 -0
- package/dist/trusted-owner-store-file.js +39 -0
- package/dist/trusted-owner-store-file.js.map +1 -0
- package/dist/trusted-owner-store.d.ts +64 -0
- package/dist/trusted-owner-store.js +126 -0
- package/dist/trusted-owner-store.js.map +1 -0
- package/dist/types.d.ts +275 -20
- package/dist/types.js +13 -0
- package/dist/types.js.map +1 -1
- package/package.json +19 -8
- package/src/bootstrap-peer-store-file.ts +44 -0
- package/src/bootstrap-peer-store.ts +172 -0
- package/src/cadre-node.ts +1992 -202
- package/src/control-authorization.ts +114 -0
- package/src/control-database.ts +1704 -146
- package/src/control-formation-recorder.ts +172 -19
- package/src/control-schema.ts +786 -222
- package/src/control-stream.ts +81 -4
- package/src/control-write-retry.ts +473 -0
- package/src/delegate-admission.ts +302 -0
- package/src/device-token.ts +18 -13
- package/src/ed25519-key.ts +134 -69
- package/src/file-durable-slot.ts +61 -0
- package/src/formation-approval.ts +604 -0
- package/src/fs-atomic.ts +107 -0
- package/src/identity-key.ts +109 -0
- package/src/index.ts +136 -5
- package/src/key-store-file.ts +10 -81
- package/src/membership-connection-gater.ts +204 -0
- package/src/node-local-snapshot.ts +260 -0
- package/src/peer-authorization.ts +199 -21
- package/src/peer-record.ts +28 -10
- package/src/relay-addrs.ts +111 -0
- package/src/relay-reservation.ts +823 -0
- package/src/seed-bootstrap.ts +397 -197
- package/src/seed-trust-policy.ts +58 -25
- package/src/strand-addr-protocol.ts +84 -53
- package/src/strand-backfill.ts +366 -0
- package/src/strand-formation-manager.ts +145 -19
- package/src/strand-formation-protocol.ts +309 -8
- package/src/strand-instance-manager.ts +111 -14
- package/src/strand-member-registry.ts +18 -13
- package/src/strand-membership-writer.ts +1430 -640
- package/src/strand-solicitation.ts +155 -90
- package/src/strand-transport-key.ts +58 -0
- package/src/strand-wake-protocol.ts +29 -46
- package/src/strand-watcher.ts +66 -3
- package/src/trusted-owner-store-file.ts +41 -0
- package/src/trusted-owner-store.ts +171 -0
- package/src/types.ts +292 -21
package/src/cadre-node.ts
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import debug from 'debug';
|
|
2
2
|
import { toString as uint8ArrayToString, fromString as uint8ArrayFromString } from 'uint8arrays';
|
|
3
3
|
import type { Libp2p, PeerId, PrivateKey, Connection } from '@libp2p/interface';
|
|
4
|
-
import {
|
|
5
|
-
import { peerIdFromString } from '@libp2p/peer-id';
|
|
4
|
+
import { peerIdFromString, peerIdFromPrivateKey } from '@libp2p/peer-id';
|
|
6
5
|
import { createLibp2pNode } from '@optimystic/db-p2p';
|
|
7
6
|
import { multiaddr } from '@multiformats/multiaddr';
|
|
8
7
|
import type { Multiaddr } from '@multiformats/multiaddr';
|
|
@@ -30,12 +29,20 @@ import type {
|
|
|
30
29
|
Libp2pNodeWithRepo,
|
|
31
30
|
PushPlatform,
|
|
32
31
|
DeviceTokenRecord,
|
|
32
|
+
CadrePeerVoucherFields,
|
|
33
|
+
CadrePeerRow,
|
|
34
|
+
PeerAddressRecord,
|
|
33
35
|
ResolveDeviceTokenOpts
|
|
34
36
|
} from './types.js';
|
|
35
|
-
import { DEFAULT_CHECKIN_WINDOW_MS } from './types.js';
|
|
37
|
+
import { CONTROL_CLUSTER_POLICY, CONTROL_REPLICATION_BREADTH, DEFAULT_CHECKIN_WINDOW_MS } from './types.js';
|
|
36
38
|
import { sign } from '@optimystic/quereus-plugin-crypto';
|
|
37
|
-
import { ed25519KeyPairFromLibp2p, type Ed25519KeyPair } from './ed25519-key.js';
|
|
39
|
+
import { ed25519KeyPairFromLibp2p, ed25519PublicKeyFromPrivate, requireEd25519PublicKeyB64, type Ed25519KeyPair } from './ed25519-key.js';
|
|
40
|
+
import { strandTransportKey } from './strand-transport-key.js';
|
|
38
41
|
import { DEFAULT_IDENTITY_KEY_ID } from './key-store.js';
|
|
42
|
+
import { loadOrCreateIdentityKey } from './identity-key.js';
|
|
43
|
+
import { MemoryTrustedOwnerStore, type TrustedOwnerStore, type TrustSource } from './trusted-owner-store.js';
|
|
44
|
+
import { MemoryBootstrapPeerStore, type BootstrapPeerStore } from './bootstrap-peer-store.js';
|
|
45
|
+
import { verifyCadrePeerVoucher } from './peer-authorization.js';
|
|
39
46
|
import { ed25519PublicKeyB64FromPeerId } from './seed-bootstrap.js';
|
|
40
47
|
import {
|
|
41
48
|
signPeerRecord,
|
|
@@ -43,6 +50,7 @@ import {
|
|
|
43
50
|
isPeerRecordFresh,
|
|
44
51
|
orderSignalingFirst,
|
|
45
52
|
isSignalingAddr,
|
|
53
|
+
trailingPeerId,
|
|
46
54
|
currentMemberTrustPolicy,
|
|
47
55
|
DEFAULT_PEER_RECORD_MAX_AGE_MS,
|
|
48
56
|
DEFAULT_PEER_RECORD_HEARTBEAT_MS
|
|
@@ -63,15 +71,32 @@ import {
|
|
|
63
71
|
} from './control-cohort.js';
|
|
64
72
|
import { EnrollmentService } from './enrollment.js';
|
|
65
73
|
import { HibernationManager, type HibernationCallbacks } from './hibernation-manager.js';
|
|
66
|
-
import { ControlDatabase } from './control-database.js';
|
|
67
|
-
import { SeedBootstrapService } from './seed-bootstrap.js';
|
|
74
|
+
import { ControlDatabase, type RevokedRowRef } from './control-database.js';
|
|
75
|
+
import { SeedBootstrapService, type SeedEventCallbacks } from './seed-bootstrap.js';
|
|
68
76
|
import type { SeedTrustPolicy } from './seed-trust-policy.js';
|
|
69
77
|
import {
|
|
70
78
|
StrandSolicitationService,
|
|
71
79
|
type StrandSolicitationServiceOptions
|
|
72
80
|
} from './strand-solicitation.js';
|
|
81
|
+
import { createMembershipConnectionGater, DEFAULT_ENROLLMENT_WINDOW_MS } from './membership-connection-gater.js';
|
|
73
82
|
import { StrandWakeService, dialWake } from './strand-wake-protocol.js';
|
|
74
|
-
import { StrandAddrService, collectStrandAddrs } from './strand-addr-protocol.js';
|
|
83
|
+
import { StrandAddrService, collectStrandAddrs, type StrandAddrPeer } from './strand-addr-protocol.js';
|
|
84
|
+
import {
|
|
85
|
+
DelegateAdmissionStore,
|
|
86
|
+
extractCircuitRelayTargets,
|
|
87
|
+
dueRelayAnnounces,
|
|
88
|
+
pruneStoppedStrandAnnounces,
|
|
89
|
+
peerStrandKey,
|
|
90
|
+
type CircuitRelayTarget
|
|
91
|
+
} from './delegate-admission.js';
|
|
92
|
+
import { resolveListenAddrs } from './relay-addrs.js';
|
|
93
|
+
import {
|
|
94
|
+
superviseRelayReservation,
|
|
95
|
+
resolveRelayReservationState,
|
|
96
|
+
type RelayReservationState,
|
|
97
|
+
type RelayReservationSupervisor,
|
|
98
|
+
type RelayReservationSupervisorOptions
|
|
99
|
+
} from './relay-reservation.js';
|
|
75
100
|
import { PushFanoutService } from './push-fanout.js';
|
|
76
101
|
import type { WakeAck, WakeRequest } from './types.js';
|
|
77
102
|
import {
|
|
@@ -94,6 +119,37 @@ const TURN_CONSUME_WINDOW_MS = 1000;
|
|
|
94
119
|
|
|
95
120
|
type EventHandler<T> = (data: T) => void;
|
|
96
121
|
|
|
122
|
+
/**
|
|
123
|
+
* A circuit relay as a strand-addr RPC target: dialed by peerId, with its
|
|
124
|
+
* direct addr riding along as the fallback for a relay we hold no connection to.
|
|
125
|
+
*/
|
|
126
|
+
function relayStrandAddrPeer(relay: CircuitRelayTarget): StrandAddrPeer {
|
|
127
|
+
return { peerId: relay.relayPeerId, addrs: [multiaddr(relay.relayAddr)] };
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* Build the signing callback the `ControlDatabase` writers expect: they hand it the
|
|
132
|
+
* canonical row-bound message BYTES (see `buildAuthorizationMessage`), and it ed25519-signs
|
|
133
|
+
* them directly (no pre-hash) with the owner private key, returning a base64url signature.
|
|
134
|
+
*/
|
|
135
|
+
function signMessageWith(privateKeyB64: string): (message: Uint8Array) => string {
|
|
136
|
+
return (message: Uint8Array): string =>
|
|
137
|
+
sign(message, privateKeyB64, 'ed25519', 'bytes', 'base64url', 'base64url') as string;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Reject a blank identifier before it reaches the control database, where it would fail an
|
|
142
|
+
* authorization CHECK as an opaque constraint error rather than an actionable message.
|
|
143
|
+
* Returns the trimmed value so callers write the same bytes they validated.
|
|
144
|
+
*/
|
|
145
|
+
function requireNonBlank(value: string, label: string): string {
|
|
146
|
+
const trimmed = value.trim();
|
|
147
|
+
if (trimmed.length === 0) {
|
|
148
|
+
throw new Error(`A ${label} is required (received an empty or whitespace-only value)`);
|
|
149
|
+
}
|
|
150
|
+
return trimmed;
|
|
151
|
+
}
|
|
152
|
+
|
|
97
153
|
/**
|
|
98
154
|
* CadreNode is the main entry point for a cadre member.
|
|
99
155
|
* It manages:
|
|
@@ -114,6 +170,15 @@ export class CadreNode implements SAppIdLookup {
|
|
|
114
170
|
* launch) reads this resolved field, never `config.privateKey` directly.
|
|
115
171
|
*/
|
|
116
172
|
private identityKey: PrivateKey | undefined;
|
|
173
|
+
/**
|
|
174
|
+
* Node-local, NON-replicated trusted-owner anchor (see
|
|
175
|
+
* `trusted-owner-store.ts`). Constructed (or adopted from
|
|
176
|
+
* `config.trustedOwners.store`) by {@link initializeTrustedOwnerStore} during
|
|
177
|
+
* {@link start}; deliberately NOT cleared by {@link cleanup}, so an in-memory
|
|
178
|
+
* anchor survives a stop()→start() cycle of the same node instance. Never
|
|
179
|
+
* sourced from replicated control state.
|
|
180
|
+
*/
|
|
181
|
+
private trustedOwnerStore: TrustedOwnerStore | null = null;
|
|
117
182
|
private controlNode: Libp2p | null = null;
|
|
118
183
|
private controlDatabase: ControlDatabase | null = null;
|
|
119
184
|
private strandWatcher: StrandWatcher | null = null;
|
|
@@ -154,6 +219,26 @@ export class CadreNode implements SAppIdLookup {
|
|
|
154
219
|
private windowWaiters: Set<{ timer: ReturnType<typeof setTimeout>; resolve: () => void }> = new Set();
|
|
155
220
|
private eventHandlers: Map<keyof CadreNodeEvents, Set<EventHandler<never>>> = new Map();
|
|
156
221
|
|
|
222
|
+
/**
|
|
223
|
+
* Relay multiaddrs the caller asked {@link reserveRelays} to reserve through.
|
|
224
|
+
* Empty (the default) means nobody asked, so {@link getRelayReservationState}
|
|
225
|
+
* reports `none` — this whole surface is opt-in and dormant for the CLI/host
|
|
226
|
+
* nodes, which use the configured `network.relayAddrs` route instead.
|
|
227
|
+
*/
|
|
228
|
+
private relayReserveAddrs: string[] = [];
|
|
229
|
+
/**
|
|
230
|
+
* The running retry loop for {@link relayReserveAddrs}, or `null` when nobody
|
|
231
|
+
* asked for a reservation / there was no control node to supervise. It owns the
|
|
232
|
+
* in-flight flag, the last failure and the next-attempt time; this class only
|
|
233
|
+
* starts it, stops it and reads it.
|
|
234
|
+
*/
|
|
235
|
+
private relayReserveSupervisor: RelayReservationSupervisor | null = null;
|
|
236
|
+
/**
|
|
237
|
+
* Reason {@link reserveRelays} produced no reservation when there is no
|
|
238
|
+
* supervisor to hold one — i.e. the pre-start `control node unavailable` case.
|
|
239
|
+
*/
|
|
240
|
+
private relayReserveError: string | null = null;
|
|
241
|
+
|
|
157
242
|
/** Map of strandId -> sAppConfig for sAppId filtering and management */
|
|
158
243
|
private sAppConfigs: Map<string, SAppConfig> = new Map();
|
|
159
244
|
|
|
@@ -165,6 +250,113 @@ export class CadreNode implements SAppIdLookup {
|
|
|
165
250
|
*/
|
|
166
251
|
private latestInviteAddresses: string[] | null = null;
|
|
167
252
|
|
|
253
|
+
/**
|
|
254
|
+
* Epoch ms until which the control-network inbound gate admits
|
|
255
|
+
* not-yet-authorized peers (see {@link admitInboundControlConnection} /
|
|
256
|
+
* {@link openEnrollmentWindow}). 0 = no window open. Opened automatically by
|
|
257
|
+
* {@link createInvite}; deliberately NOT reset on stop() — a stop/start cycle
|
|
258
|
+
* inside an outstanding invite's validity must not strand the invitee.
|
|
259
|
+
*
|
|
260
|
+
* NOTE: in-memory only, so a PROCESS restart mid-invite closes the door until
|
|
261
|
+
* the owner re-mints or the invitee is authorized. Fine while invites are
|
|
262
|
+
* short-lived and redeemed promptly; if long-lived invites or restart-prone
|
|
263
|
+
* hosts become normal, persist the window (or derive it from the issued-invite
|
|
264
|
+
* records `SeedBootstrapService` already keeps).
|
|
265
|
+
*/
|
|
266
|
+
private enrollmentWindowUntil = 0;
|
|
267
|
+
|
|
268
|
+
/**
|
|
269
|
+
* Lazily-parsed PeerIds of `controlNetwork.bootstrapNodes` (see
|
|
270
|
+
* {@link getBootstrapPeerIds}) — infrastructure the inbound gate always admits.
|
|
271
|
+
*/
|
|
272
|
+
private bootstrapPeerIds: Set<string> | null = null;
|
|
273
|
+
|
|
274
|
+
/**
|
|
275
|
+
* Materialized in-memory snapshot of the AUTHORIZED member peer ids (the
|
|
276
|
+
* {@link listAuthorizedMembers} result), consulted by the per-stream
|
|
277
|
+
* control-DB gate ({@link authorizeInboundControlStream}). That predicate
|
|
278
|
+
* runs on EVERY inbound Optimystic control-DB stream and must never await a
|
|
279
|
+
* control-DB read: those reads pull blocks over the very protocols the
|
|
280
|
+
* predicate gates, which closes a circular wait that ends in mutual denial
|
|
281
|
+
* (the upstream gate is fail-closed on timeout, unlike the fail-open
|
|
282
|
+
* connection gater). So the set is refreshed OUT OF BAND instead — after
|
|
283
|
+
* start, on every {@link reconcileControlCohort} pass (15s cadence), and
|
|
284
|
+
* immediately after every LOCAL membership mutation so a just-vouched peer
|
|
285
|
+
* is admitted without waiting for the timer. That last refresh is AUTOMATIC:
|
|
286
|
+
* the control database notifies {@link refreshMembershipGate} after every
|
|
287
|
+
* committed `CadrePeer` write (see `ControlDatabase.mutateCadrePeer`), so no
|
|
288
|
+
* writer has to remember. A change that arrives by REPLICATION is picked up
|
|
289
|
+
* on the next timed refresh — bounded staleness, acceptable because this gate
|
|
290
|
+
* is defense in depth: rows an unadmitted peer manages to write are still
|
|
291
|
+
* disbelieved at read time by the voucher-anchored predicate.
|
|
292
|
+
*/
|
|
293
|
+
private authorizedControlPeers: Set<string> = new Set();
|
|
294
|
+
|
|
295
|
+
/**
|
|
296
|
+
* Coalescing state for {@link refreshMembershipGate}: a pending "the snapshot
|
|
297
|
+
* is stale" flag, the single in-flight drain that consumes it, and the depth
|
|
298
|
+
* of open {@link deferMembershipGateRefresh} scopes (a burst of writes inside
|
|
299
|
+
* one scope collapses to a single refresh at scope exit).
|
|
300
|
+
*/
|
|
301
|
+
private membershipGateDirty = false;
|
|
302
|
+
private membershipGateDrain: Promise<void> | null = null;
|
|
303
|
+
private membershipGateDeferDepth = 0;
|
|
304
|
+
|
|
305
|
+
/**
|
|
306
|
+
* In-memory delegate admission grants (see `delegate-admission.ts`): the
|
|
307
|
+
* strand-node transport peerIds this party's members have announced over the
|
|
308
|
+
* strand-addr RPC, admitted at the CONNECTION level so a member's NAT'd
|
|
309
|
+
* strand node can hold a circuit-relay reservation on this node. Consulted
|
|
310
|
+
* ONLY by {@link admitInboundControlConnection}; the fail-closed per-stream
|
|
311
|
+
* gate ({@link authorizeInboundControlStream}) never honors it.
|
|
312
|
+
*/
|
|
313
|
+
private readonly delegateAdmission = new DelegateAdmissionStore();
|
|
314
|
+
|
|
315
|
+
/**
|
|
316
|
+
* When this node last ANNOUNCED a delegate, keyed `<targetPeerId>\n<strandId>`
|
|
317
|
+
* (the peer announced TO, not the delegate). Throttles
|
|
318
|
+
* {@link refreshDelegateGrants} to once per `DELEGATE_GRANT_TTL_MS / 2` per
|
|
319
|
+
* (relay, strand) so the 15 s reconcile tick never becomes per-tick RPC
|
|
320
|
+
* chatter; the launch/resume announce passes record here too. Keys whose
|
|
321
|
+
* strand is no longer running are pruned on each reconcile pass.
|
|
322
|
+
*/
|
|
323
|
+
private readonly delegateAnnounceAt = new Map<string, number>();
|
|
324
|
+
|
|
325
|
+
/**
|
|
326
|
+
* Cold-start bootstrap dial targets: the owner-flagged peers of every seed
|
|
327
|
+
* this node has applied, keyed by peer id, valued by the seed's multiaddr
|
|
328
|
+
* strings (parsed lazily, at dial time). Written by
|
|
329
|
+
* {@link recordSeedBootstrapPeers}; read only by
|
|
330
|
+
* {@link dialColdStartBootstrap} while the control database still has no
|
|
331
|
+
* siblings to dial.
|
|
332
|
+
*
|
|
333
|
+
* A STORE rather than a plain map (see `bootstrap-peer-store.ts`), because the
|
|
334
|
+
* targets must outlive the process: a node seeded into a party it could not
|
|
335
|
+
* reach has nothing else on disk naming that party's addresses (`applySeed`
|
|
336
|
+
* writes no control row, and `CadrePeer` fills in only after a connection
|
|
337
|
+
* succeeds), so an in-memory-only set left it stranded permanently across a
|
|
338
|
+
* restart. Constructed (or adopted from `config.bootstrapPeers.store`) by
|
|
339
|
+
* {@link initializeBootstrapPeerStore} during {@link start}; deliberately NOT
|
|
340
|
+
* cleared by {@link cleanup}, so it survives a stop()→start() cycle of the same
|
|
341
|
+
* node instance — same lifecycle as {@link trustedOwnerStore}. Durability
|
|
342
|
+
* depends on the injected backend: Node gets a file-backed store from the CLI,
|
|
343
|
+
* RN/browser stay ephemeral for now.
|
|
344
|
+
*
|
|
345
|
+
* Deliberately NOT the libp2p peer store, which {@link peerStoreAddrs} already
|
|
346
|
+
* reads for the steady-state path. The peer store is shared with everything
|
|
347
|
+
* libp2p discovers, so "dial every entry" would grow into dialing arbitrary
|
|
348
|
+
* discovered peers as the node lives longer; this store holds exactly the peers
|
|
349
|
+
* an owner-signed, trust-anchored seed nominated as owners. `applySeed` merges
|
|
350
|
+
* the same addresses into the peer store as well, so the two never disagree —
|
|
351
|
+
* this one is just the precisely-scoped subset.
|
|
352
|
+
*
|
|
353
|
+
* A later seed OVERWRITES an entry rather than merging, so a re-seed after an
|
|
354
|
+
* owner's address changes replaces the stale address instead of accumulating
|
|
355
|
+
* dead ones. Entries are never evicted: they are the node's only way back into
|
|
356
|
+
* the party if it is ever stranded again.
|
|
357
|
+
*/
|
|
358
|
+
private bootstrapPeerStore: BootstrapPeerStore | null = null;
|
|
359
|
+
|
|
168
360
|
/** Initial self-registration timer (see {@link scheduleSelfRegistration}). */
|
|
169
361
|
private selfRegistrationTimer: ReturnType<typeof setTimeout> | null = null;
|
|
170
362
|
/** TTL heartbeat that re-publishes the self record before it goes stale. */
|
|
@@ -195,15 +387,38 @@ export class CadreNode implements SAppIdLookup {
|
|
|
195
387
|
|
|
196
388
|
// ── Write-while-alone re-replication (control-write-ensure-replicated) ──────
|
|
197
389
|
/**
|
|
198
|
-
* Re-replication queue for owner
|
|
199
|
-
* node was alone — `controlNode.getConnections().length === 0` at write
|
|
200
|
-
* means the Optimystic commit was local-only (the block's cluster was ≤1)
|
|
201
|
-
* never broadcast. Maps the affected subject peerId →
|
|
202
|
-
* the cohort grows.
|
|
203
|
-
*
|
|
204
|
-
*
|
|
390
|
+
* Re-replication queue for owner `authorizePeer` writes that committed while
|
|
391
|
+
* this node was alone — `controlNode.getConnections().length === 0` at write
|
|
392
|
+
* time means the Optimystic commit was local-only (the block's cluster was ≤1)
|
|
393
|
+
* and never broadcast. Maps the affected subject peerId → 'authorize', to
|
|
394
|
+
* re-issue once the cohort grows. Removals are NOT tracked here — a delete
|
|
395
|
+
* leaves a `Revocation` tombstone, tracked by {@link pendingRevocations}; a
|
|
396
|
+
* removePeer also clears any queued authorize for the same subject (the row is
|
|
397
|
+
* gone, so re-issuing the insert would be wrong). Drained on the 0→≥1
|
|
398
|
+
* control-connection transition by {@link drainPendingControlReplication}.
|
|
399
|
+
*/
|
|
400
|
+
private pendingPeerWrites: Map<string, 'authorize'> = new Map();
|
|
401
|
+
/**
|
|
402
|
+
* Re-replication queue for `Revocation` tombstones that committed while this
|
|
403
|
+
* node was alone, keyed on the retired `StampId`. Fed by the control DB's
|
|
404
|
+
* committed-delete seam ({@link noteGuardedDelete}), so one queue covers all
|
|
405
|
+
* four guarded tables (`CadrePeer` / `DeviceToken` / `Strand` /
|
|
406
|
+
* `ValidationKey`). Drained (owner-signed `ReissuedAt` bump, which re-broadcasts
|
|
407
|
+
* the tombstone) by {@link drainPendingRevocations}. Cleared on stop alongside
|
|
408
|
+
* {@link pendingPeerWrites} — the tombstone rows are durable, so the next
|
|
409
|
+
* lifetime's first-growth sweep re-covers anything dropped here.
|
|
410
|
+
*/
|
|
411
|
+
private pendingRevocations: Map<string, RevokedRowRef> = new Map();
|
|
412
|
+
/**
|
|
413
|
+
* Guards the once-per-lifetime first-growth revocation sweep (re-issue EVERY
|
|
414
|
+
* locally-held tombstone, covering removals from before this node started).
|
|
415
|
+
* Deliberately separate from {@link reconstructedLocalOnlyWrites} and set only
|
|
416
|
+
* after a sweep pass SUCCEEDS (including the nothing-held case) — a throwing
|
|
417
|
+
* sweep retries on the next growth edge instead of being lost for the lifetime.
|
|
418
|
+
* Reset on stop, so a stop()→start() cycle sweeps again (that second start is
|
|
419
|
+
* exactly a "removals from before this lifetime" case).
|
|
205
420
|
*/
|
|
206
|
-
private
|
|
421
|
+
private reissuedHeldRevocations = false;
|
|
207
422
|
/** This node's own `CadrePeer` self-write committed local-only (re-touched on growth). */
|
|
208
423
|
private pendingSelfPeerWrite = false;
|
|
209
424
|
/** This node's own `DeviceToken` self-write committed local-only (re-touched on growth). */
|
|
@@ -380,6 +595,17 @@ export class CadreNode implements SAppIdLookup {
|
|
|
380
595
|
|
|
381
596
|
log('Starting CadreNode for party: %s', this.config.controlNetwork.partyId);
|
|
382
597
|
|
|
598
|
+
// NOTE: the only direct `console.*` in this library — a boot-time operator warning, not a
|
|
599
|
+
// diagnostic trace (those use `debug`, which an operator never sees without `DEBUG=`). If a
|
|
600
|
+
// second such warning ever appears here, route both through a `CadreNodeEvents` entry the
|
|
601
|
+
// embedder surfaces instead of growing a console surface inside a library.
|
|
602
|
+
if (this.config.network?.announceAddrs && this.config.network.announceAddrs.length > 0) {
|
|
603
|
+
console.warn(
|
|
604
|
+
'network.announceAddrs is set but not yet supported (no upstream db-p2p option to apply it) ' +
|
|
605
|
+
'— this node will keep advertising its listen/relay addresses instead.'
|
|
606
|
+
);
|
|
607
|
+
}
|
|
608
|
+
|
|
383
609
|
try {
|
|
384
610
|
const tTotal = performance.now();
|
|
385
611
|
|
|
@@ -393,6 +619,18 @@ export class CadreNode implements SAppIdLookup {
|
|
|
393
619
|
// store fails closed before a node is created.
|
|
394
620
|
await this.resolveIdentityKey();
|
|
395
621
|
|
|
622
|
+
// Bring up the node-local trusted-owner anchor (and seed config pins)
|
|
623
|
+
// before any network bring-up: a mis-scoped injected store fails closed
|
|
624
|
+
// here, and out-of-band pins are anchored before the first seed/peer
|
|
625
|
+
// interaction could consult them.
|
|
626
|
+
await this.initializeTrustedOwnerStore();
|
|
627
|
+
|
|
628
|
+
// Bring up the node-local cold-start bootstrap-peer store alongside the
|
|
629
|
+
// anchor, and for the same two reasons: a mis-scoped injected store fails
|
|
630
|
+
// closed before any network bring-up, and the retained dial targets are
|
|
631
|
+
// loaded before the first reconcile pass could consult them.
|
|
632
|
+
this.initializeBootstrapPeerStore();
|
|
633
|
+
|
|
396
634
|
// Create the control network libp2p node
|
|
397
635
|
let t0 = performance.now();
|
|
398
636
|
this.controlNode = await this.createControlNode();
|
|
@@ -417,6 +655,16 @@ export class CadreNode implements SAppIdLookup {
|
|
|
417
655
|
timing('[start] controlDatabase.initialize: %dms', Math.round(performance.now() - t0));
|
|
418
656
|
log('Control database initialized');
|
|
419
657
|
|
|
658
|
+
// Attach the per-stream gate to the control DB's membership hub, so every
|
|
659
|
+
// committed `CadrePeer` write re-materializes the authorized snapshot on its
|
|
660
|
+
// own. Wired before anything can write. `_running` is still false for the
|
|
661
|
+
// rest of start() and the refresh early-returns while it is — a write in that
|
|
662
|
+
// window is covered by start's own pass below.
|
|
663
|
+
this.controlDatabase.setMembershipChangeListener((reason) => this.refreshMembershipGate(reason));
|
|
664
|
+
// Committed-delete seam → write-while-alone revocation queue. Same wiring
|
|
665
|
+
// window argument as the membership listener above.
|
|
666
|
+
this.controlDatabase.setGuardedDeleteListener((revocation) => this.noteGuardedDelete(revocation));
|
|
667
|
+
|
|
420
668
|
// Create strand queryable using the control database
|
|
421
669
|
const queryable = this.createStrandQueryable();
|
|
422
670
|
|
|
@@ -454,9 +702,15 @@ export class CadreNode implements SAppIdLookup {
|
|
|
454
702
|
// resolving a strand's bootstrap seed asks us for our live strand-network
|
|
455
703
|
// multiaddrs (its CadrePeer row only knows our *control* address). Gated on
|
|
456
704
|
// AUTHORIZED membership; answers only for strands we are actively meshing.
|
|
705
|
+
// The same RPC doubles as the delegate-announce channel: a member's request
|
|
706
|
+
// may carry the derived peerId its strand node runs as, and we record an
|
|
707
|
+
// admission grant so our connection gate (and thus our relay server, when
|
|
708
|
+
// enabled) admits it — see delegate-admission.ts.
|
|
457
709
|
this.strandAddrService = new StrandAddrService({
|
|
458
710
|
isMember: (peerId) => this.isAuthorizedMember(peerId),
|
|
459
711
|
getStrandMultiaddrs: (strandId) => this.getStrandMultiaddrs(strandId),
|
|
712
|
+
onDelegateAnnounce: (announcer, strandId, delegate) =>
|
|
713
|
+
this.grantDelegateAdmission(announcer, strandId, delegate),
|
|
460
714
|
});
|
|
461
715
|
this.strandAddrService.initialize(this.controlNode);
|
|
462
716
|
|
|
@@ -476,6 +730,12 @@ export class CadreNode implements SAppIdLookup {
|
|
|
476
730
|
// write-while-alone re-replication drain must fire on the first 0→≥1 edge.
|
|
477
731
|
this.wireControlConnectionListeners();
|
|
478
732
|
|
|
733
|
+
// Seed the per-stream gate's materialized authorized set now that the
|
|
734
|
+
// control DB is up (non-blocking; the helper never rejects). Until it
|
|
735
|
+
// lands the snapshot is empty, which the stream gate treats as cold
|
|
736
|
+
// start and admits.
|
|
737
|
+
void this.refreshMembershipGate('start');
|
|
738
|
+
|
|
479
739
|
// Schedule self-registration in background
|
|
480
740
|
this.scheduleSelfRegistration();
|
|
481
741
|
|
|
@@ -522,7 +782,18 @@ export class CadreNode implements SAppIdLookup {
|
|
|
522
782
|
}
|
|
523
783
|
|
|
524
784
|
log('Stopping CadreNode');
|
|
785
|
+
// Stop the retry loop BEFORE tearing the node down, not after: `cleanup()`
|
|
786
|
+
// stops the control node partway through, and a tick that fires during it
|
|
787
|
+
// would dial and then poll a half-torn-down node for the whole reserve
|
|
788
|
+
// timeout — logging spurious dial failures and holding the process open.
|
|
789
|
+
this.relayReserveSupervisor?.stop();
|
|
790
|
+
this.relayReserveSupervisor = null;
|
|
525
791
|
await this.cleanup();
|
|
792
|
+
// Drop the rest of the relay-reservation posture: a stopped node holds no
|
|
793
|
+
// reservation, so a restarted instance must not report the previous run's
|
|
794
|
+
// `reserved`.
|
|
795
|
+
this.relayReserveAddrs = [];
|
|
796
|
+
this.relayReserveError = null;
|
|
526
797
|
this._running = false;
|
|
527
798
|
this.emit('control:disconnected', undefined);
|
|
528
799
|
log('CadreNode stopped');
|
|
@@ -567,11 +838,11 @@ export class CadreNode implements SAppIdLookup {
|
|
|
567
838
|
* before any network bring-up. Resolution order:
|
|
568
839
|
*
|
|
569
840
|
* 1. Both `keyStore` and `privateKey` set ⇒ configuration error (throws).
|
|
570
|
-
* 2. `keyStore` set ⇒
|
|
571
|
-
* {@link DEFAULT_IDENTITY_KEY_ID})
|
|
572
|
-
*
|
|
573
|
-
*
|
|
574
|
-
*
|
|
841
|
+
* 2. `keyStore` set ⇒ {@link loadOrCreateIdentityKey} against `identityKeyId`
|
|
842
|
+
* (default {@link DEFAULT_IDENTITY_KEY_ID}): load when the slot is present,
|
|
843
|
+
* generate + persist when it is empty. A rejected `get` (access denied /
|
|
844
|
+
* backend failure) PROPAGATES — we never generate a new key on a read error,
|
|
845
|
+
* which would silently orphan the real identity.
|
|
575
846
|
* 3. `privateKey` set ⇒ use it directly.
|
|
576
847
|
* 4. Neither ⇒ leave undefined; libp2p generates an ephemeral key.
|
|
577
848
|
*
|
|
@@ -592,21 +863,11 @@ export class CadreNode implements SAppIdLookup {
|
|
|
592
863
|
}
|
|
593
864
|
|
|
594
865
|
if (keyStore) {
|
|
595
|
-
|
|
596
|
-
//
|
|
597
|
-
//
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
// Corrupt/garbage bytes throw here; surface loudly rather than
|
|
601
|
-
// regenerating (which would orphan the real identity).
|
|
602
|
-
this.identityKey = privateKeyFromProtobuf(bytes);
|
|
603
|
-
log('Identity key loaded from key store (slot present)');
|
|
604
|
-
return;
|
|
605
|
-
}
|
|
606
|
-
const generated = await generateKeyPair('Ed25519');
|
|
607
|
-
await keyStore.set(keyId, privateKeyToProtobuf(generated));
|
|
608
|
-
this.identityKey = generated;
|
|
609
|
-
log('Identity key generated and persisted to key store (first run)');
|
|
866
|
+
// Shared with the embedding app: `reference-app-rn` resolves the very same
|
|
867
|
+
// key before constructing this node so it can sign its ICE-manifest request
|
|
868
|
+
// with it (identity-key.ts). One copy of the rule, so neither side can
|
|
869
|
+
// drift into generating a second identity.
|
|
870
|
+
this.identityKey = await loadOrCreateIdentityKey(keyStore, identityKeyId ?? DEFAULT_IDENTITY_KEY_ID);
|
|
610
871
|
return;
|
|
611
872
|
}
|
|
612
873
|
|
|
@@ -617,9 +878,129 @@ export class CadreNode implements SAppIdLookup {
|
|
|
617
878
|
// Neither configured: libp2p generates an ephemeral key internally.
|
|
618
879
|
}
|
|
619
880
|
|
|
881
|
+
/**
|
|
882
|
+
* Construct (or adopt) the node-local trusted-owner anchor and seed the
|
|
883
|
+
* out-of-band pinned keys from `config.trustedOwners`. The store is NEVER
|
|
884
|
+
* sourced from the replicated control DB — its entries come only from
|
|
885
|
+
* genesis self-trust ({@link initializeSeedBootstrap}), config pins (here),
|
|
886
|
+
* or runtime enrollment pins ({@link trustOwnerKeys}).
|
|
887
|
+
*
|
|
888
|
+
* Idempotent across stop()→start(): the store instance is kept, and
|
|
889
|
+
* re-seeding config pins is a no-op ({@link TrustedOwnerStore.trust} is
|
|
890
|
+
* idempotent). An injected store scoped to a different party is a
|
|
891
|
+
* configuration error (fail closed before any network bring-up).
|
|
892
|
+
*/
|
|
893
|
+
private async initializeTrustedOwnerStore(): Promise<void> {
|
|
894
|
+
const { trustedOwners } = this.config;
|
|
895
|
+
const partyId = this.config.controlNetwork.partyId;
|
|
896
|
+
if (!this.trustedOwnerStore) {
|
|
897
|
+
const store = trustedOwners?.store ?? new MemoryTrustedOwnerStore(partyId);
|
|
898
|
+
if (store.partyId !== partyId) {
|
|
899
|
+
throw new Error(
|
|
900
|
+
`CadreNodeConfig: trustedOwners.store is scoped to party ${store.partyId}, ` +
|
|
901
|
+
`but this node serves party ${partyId} — refusing to mix trust anchors`
|
|
902
|
+
);
|
|
903
|
+
}
|
|
904
|
+
this.trustedOwnerStore = store;
|
|
905
|
+
}
|
|
906
|
+
// Validate every pin before trusting any: a malformed entry must not leave
|
|
907
|
+
// earlier, valid pins anchored while start() then fails on a later one —
|
|
908
|
+
// config pins land all-or-nothing, same as the runtime seam below.
|
|
909
|
+
const pinnedKeys = (trustedOwners?.pinnedKeys ?? []).map(key => requireEd25519PublicKeyB64(key, 'pinned owner key'));
|
|
910
|
+
for (const key of pinnedKeys) {
|
|
911
|
+
await this.trustedOwnerStore.trust(key, trustedOwners?.pinnedSource ?? 'operator');
|
|
912
|
+
}
|
|
913
|
+
}
|
|
914
|
+
|
|
915
|
+
/**
|
|
916
|
+
* Construct (or adopt) the node-local cold-start bootstrap-peer store (see
|
|
917
|
+
* {@link bootstrapPeerStore}). Synchronous: an injected store has already
|
|
918
|
+
* loaded its persisted targets by the time it is handed in (its `open` is the
|
|
919
|
+
* async part), and the in-memory fallback has nothing to load.
|
|
920
|
+
*
|
|
921
|
+
* Idempotent across stop()→start(): the store instance is kept, so retained
|
|
922
|
+
* targets survive a restart of the same node instance even with the ephemeral
|
|
923
|
+
* default. An injected store scoped to a different party is a configuration
|
|
924
|
+
* error (fail closed before any network bring-up) — a foreign party's addresses
|
|
925
|
+
* must never enter this node's dial loop.
|
|
926
|
+
*/
|
|
927
|
+
private initializeBootstrapPeerStore(): void {
|
|
928
|
+
const partyId = this.config.controlNetwork.partyId;
|
|
929
|
+
if (this.bootstrapPeerStore) {
|
|
930
|
+
return;
|
|
931
|
+
}
|
|
932
|
+
const store = this.config.bootstrapPeers?.store ?? new MemoryBootstrapPeerStore(partyId);
|
|
933
|
+
if (store.partyId !== partyId) {
|
|
934
|
+
throw new Error(
|
|
935
|
+
`CadreNodeConfig: bootstrapPeers.store is scoped to party ${store.partyId}, ` +
|
|
936
|
+
`but this node serves party ${partyId} — refusing to mix cold-start dial targets`
|
|
937
|
+
);
|
|
938
|
+
}
|
|
939
|
+
this.bootstrapPeerStore = store;
|
|
940
|
+
}
|
|
941
|
+
|
|
942
|
+
/**
|
|
943
|
+
* The node-local cold-start bootstrap-peer store (null before {@link start}) —
|
|
944
|
+
* the retained owner addresses {@link reconcileControlCohort}'s cold-start
|
|
945
|
+
* branch re-dials. Exposed for diagnostics and for a host that wants to show
|
|
946
|
+
* "what would this node dial if it is stranded?".
|
|
947
|
+
*/
|
|
948
|
+
getBootstrapPeerStore(): BootstrapPeerStore | null {
|
|
949
|
+
return this.bootstrapPeerStore;
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
/**
|
|
953
|
+
* The node-local trusted-owner anchor (null before {@link start}). This is
|
|
954
|
+
* the set the authorized-membership predicate and seed-trust anchor consult —
|
|
955
|
+
* never the replicated `OwnerKey` table, which any stranger can pollute.
|
|
956
|
+
*/
|
|
957
|
+
getTrustedOwnerStore(): TrustedOwnerStore | null {
|
|
958
|
+
return this.trustedOwnerStore;
|
|
959
|
+
}
|
|
960
|
+
|
|
961
|
+
/**
|
|
962
|
+
* Persist out-of-band-established owner keys into the node-local anchor —
|
|
963
|
+
* the runtime enrollment seam: call with `CadreInvite.ownerKeys` when
|
|
964
|
+
* redeeming an invite (BEFORE the first `applySeed`, so the anchor already
|
|
965
|
+
* holds the pins when seed trust consults it), or with an operator-supplied
|
|
966
|
+
* pin. Idempotent. ('genesis' provenance is reserved for the node's own
|
|
967
|
+
* founding key, seeded internally by {@link initializeSeedBootstrap}.)
|
|
968
|
+
*
|
|
969
|
+
* Validates every key's shape before trusting any of them (all-or-nothing):
|
|
970
|
+
* a malformed entry anywhere in `keys` rejects the whole call before a
|
|
971
|
+
* single key is anchored. For the invite route this means a `CadreInvite`
|
|
972
|
+
* carrying one malformed `ownerKeys` entry fails the redemption outright —
|
|
973
|
+
* consistent with the anchor's existing whole-record-or-nothing policy for
|
|
974
|
+
* a corrupt persisted entry (see `trusted-owner-store.ts`'s
|
|
975
|
+
* `unusableEntry: 'discard-all'`) — rather than silently anchoring a subset
|
|
976
|
+
* and leaving the caller to notice a key went missing.
|
|
977
|
+
*/
|
|
978
|
+
async trustOwnerKeys(keys: Iterable<string>, source: Exclude<TrustSource, 'genesis'>): Promise<void> {
|
|
979
|
+
if (!this.trustedOwnerStore) {
|
|
980
|
+
throw new Error('CadreNode must be started before trusting owner keys');
|
|
981
|
+
}
|
|
982
|
+
const validated = Array.from(keys, key => requireEd25519PublicKeyB64(key, 'pinned owner key'));
|
|
983
|
+
for (const key of validated) {
|
|
984
|
+
await this.trustedOwnerStore.trust(key, source);
|
|
985
|
+
}
|
|
986
|
+
}
|
|
987
|
+
|
|
620
988
|
private async createControlNode(): Promise<Libp2p> {
|
|
989
|
+
return await createLibp2pNode(this.buildControlNodeOptions());
|
|
990
|
+
}
|
|
991
|
+
|
|
992
|
+
/**
|
|
993
|
+
* Map this node's config onto the control network's libp2p node options.
|
|
994
|
+
*
|
|
995
|
+
* Split out of {@link createControlNode} purely so the mapping is assertable without
|
|
996
|
+
* standing up a real libp2p node — `packages/cadre-core/test/cadre-node-control-node-options.spec.ts`
|
|
997
|
+
* calls it on a bare `new CadreNode(config)`. Read nothing else into the split; the
|
|
998
|
+
* only caller in production is `createControlNode`.
|
|
999
|
+
*/
|
|
1000
|
+
private buildControlNodeOptions(): Parameters<typeof createLibp2pNode>[0] {
|
|
621
1001
|
const { controlNetwork, network, storage, profile } = this.config;
|
|
622
1002
|
const identityKey = this.identityKey;
|
|
1003
|
+
const listenAddrs = resolveListenAddrs(network);
|
|
623
1004
|
|
|
624
1005
|
// Determine relay mode: if explicitly set in config, use that;
|
|
625
1006
|
// otherwise default to true for storage profile nodes (better connectivity/uptime)
|
|
@@ -640,16 +1021,290 @@ export class CadreNode implements SAppIdLookup {
|
|
|
640
1021
|
storage: controlStorageProvider,
|
|
641
1022
|
fretProfile: profile === 'storage' ? 'core' : 'edge',
|
|
642
1023
|
relay: enableRelay,
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
1024
|
+
// Fixed, and deliberately above any party's node count: every member reads the
|
|
1025
|
+
// whole control database, so a cohort that excludes a member leaves it dependent
|
|
1026
|
+
// on read repair — which cannot converge at a two-member cohort. Not a knob;
|
|
1027
|
+
// see CONTROL_REPLICATION_BREADTH. `assumedClusterSize: 2` is declared explicitly in
|
|
1028
|
+
// CONTROL_CLUSTER_POLICY: only the admission gate defaults to 2, while the read-repair
|
|
1029
|
+
// corroboration floor would otherwise fall back to this clusterSize of 16.
|
|
1030
|
+
clusterSize: CONTROL_REPLICATION_BREADTH,
|
|
1031
|
+
clusterPolicy: CONTROL_CLUSTER_POLICY,
|
|
1032
|
+
arachnode: { enableRingZulu: profile === 'storage' },
|
|
646
1033
|
...(identityKey && { privateKey: identityKey }),
|
|
647
1034
|
...(network?.transports && { transports: network.transports }),
|
|
648
|
-
|
|
649
|
-
|
|
1035
|
+
// Configured `listenAddrs`, plus a `/p2p-circuit` listener for every
|
|
1036
|
+
// `network.relayAddrs` entry — that listener is what reserves the relay slot.
|
|
1037
|
+
...(listenAddrs && { listenAddrs }),
|
|
1038
|
+
// The CONTROL node composes the membership admission gate onto any
|
|
1039
|
+
// caller-supplied gater (deny from either wins on inbound; all other
|
|
1040
|
+
// hooks pass through). Strand cohort nodes keep the raw configured gater
|
|
1041
|
+
// (see strand-instance-manager.ts) — their peers are legitimately
|
|
1042
|
+
// cross-party, so cadre membership must not gate them.
|
|
1043
|
+
connectionGater: createMembershipConnectionGater(
|
|
1044
|
+
{ admitInbound: (remotePeerId) => this.admitInboundControlConnection(remotePeerId) },
|
|
1045
|
+
network?.connectionGater
|
|
1046
|
+
),
|
|
1047
|
+
// Fail-closed per-stream authorization for the four Optimystic control-DB
|
|
1048
|
+
// protocols — the members-only layer the connection gater's stranger
|
|
1049
|
+
// carve-outs cannot express (see authorizeInboundControlStream). Control
|
|
1050
|
+
// node only: strand cohort nodes serve cross-party peers.
|
|
1051
|
+
authorizeInboundStream: (remotePeerId, protocol) =>
|
|
1052
|
+
this.authorizeInboundControlStream(remotePeerId, protocol)
|
|
650
1053
|
};
|
|
651
1054
|
|
|
652
|
-
return
|
|
1055
|
+
return nodeOptions;
|
|
1056
|
+
}
|
|
1057
|
+
|
|
1058
|
+
/**
|
|
1059
|
+
* Decide whether an inbound CONTROL-network connection from `remotePeerId`
|
|
1060
|
+
* should be admitted — the policy behind the connection gater wired in
|
|
1061
|
+
* {@link createControlNode}. Deny only on a positive "unauthorized outsider
|
|
1062
|
+
* while no stranger path is open" determination; everything ambiguous admits
|
|
1063
|
+
* and defers to the fail-closed per-stream gates (see
|
|
1064
|
+
* `membership-connection-gater.ts` for the layer's rationale and the
|
|
1065
|
+
* stranger-open protocol allowlist).
|
|
1066
|
+
*
|
|
1067
|
+
* Admits when ANY of:
|
|
1068
|
+
* 1. a shared-baseline check admits ({@link admitControlPeerUnconditionally}
|
|
1069
|
+
* — not running / DB torn down, absent-or-empty trusted-owner anchor, or
|
|
1070
|
+
* configured bootstrap/relay infrastructure);
|
|
1071
|
+
* 2. an enrollment window is open ({@link openEnrollmentWindow}, opened by
|
|
1072
|
+
* {@link createInvite}) — the invitee dials in before it is authorized;
|
|
1073
|
+
* 3. the peer holds a live DELEGATE ADMISSION GRANT — an authorized member
|
|
1074
|
+
* announced it (over the strand-addr RPC) as the transport peerId of its
|
|
1075
|
+
* own strand node, so a NAT'd member's strand node can hold a
|
|
1076
|
+
* circuit-relay reservation here (see `delegate-admission.ts`).
|
|
1077
|
+
* Connection only: the per-stream gate below never honors a grant;
|
|
1078
|
+
* 4. the authorized-member set is empty — cold start: the rows that would
|
|
1079
|
+
* authorize anyone arrive by replication over these very connections;
|
|
1080
|
+
* 5. the peer IS an authorized member; or
|
|
1081
|
+
* 6. an open invitation is OUTSTANDING — at least one unexpired,
|
|
1082
|
+
* not-fully-consumed invitation this node minted or persisted (see
|
|
1083
|
+
* `StrandSolicitationService.hasOutstandingInvitation`). A formation
|
|
1084
|
+
* initiator is another party's peer by design and its token is only
|
|
1085
|
+
* checkable inside the protocol, so the gate asks the coarser question
|
|
1086
|
+
* "does this node expect a stranger at all?". Merely REGISTERING the
|
|
1087
|
+
* responder ({@link initializeStrandSolicitation}) no longer suspends
|
|
1088
|
+
* stranger denial — eager registration (as `reference-app-rn` does at
|
|
1089
|
+
* bring-up) and `formStrand`'s lazy initialization both leave the gate
|
|
1090
|
+
* armed, because neither mints an invitation.
|
|
1091
|
+
*
|
|
1092
|
+
* Ordering is semantically free (the checks are OR'd) but decides who pays:
|
|
1093
|
+
* checks 1-3 are in-memory, 4/5 share one control-DB read, and only a peer
|
|
1094
|
+
* already on the deny path reaches check 6's invitation lookup.
|
|
1095
|
+
*
|
|
1096
|
+
* Caveats of check 6, both self-healing:
|
|
1097
|
+
* - the in-memory mint registry dies with the process, so after a restart
|
|
1098
|
+
* only invitations persisted as `FormationInvite` rows still hold the
|
|
1099
|
+
* exemption open (re-mint otherwise — same story as the enrollment window);
|
|
1100
|
+
* - a peer holding a token whose `FormationInvite` row has not replicated to
|
|
1101
|
+
* this node yet is denied even though the formation handler would have
|
|
1102
|
+
* accepted it, exactly like the unreplicated-membership-row case below.
|
|
1103
|
+
*
|
|
1104
|
+
* NOTE: check 4/5 runs a control-DB read per inbound connection
|
|
1105
|
+
* (`listAuthorizedMembers`); connections are rare and cadres small, and this
|
|
1106
|
+
* layer is fail-open behind `ADMISSION_DECISION_TIMEOUT_MS`, so the live
|
|
1107
|
+
* read is safe here — unlike the per-stream gate, which must consult the
|
|
1108
|
+
* materialized {@link authorizedControlPeers} snapshot instead.
|
|
1109
|
+
* NOTE: a sibling whose membership row has not yet replicated to
|
|
1110
|
+
* this node is denied until the row converges (typically via the owner);
|
|
1111
|
+
* either side's next outbound reconcile dial (outbound is never gated)
|
|
1112
|
+
* re-establishes the link — self-healing, but visible as a transient deny.
|
|
1113
|
+
*/
|
|
1114
|
+
private async admitInboundControlConnection(remotePeerId: string): Promise<boolean> {
|
|
1115
|
+
if (this.admitControlPeerUnconditionally(remotePeerId)) {
|
|
1116
|
+
return true;
|
|
1117
|
+
}
|
|
1118
|
+
if (Date.now() < this.enrollmentWindowUntil) {
|
|
1119
|
+
return true;
|
|
1120
|
+
}
|
|
1121
|
+
if (this.delegateAdmission.has(remotePeerId)) {
|
|
1122
|
+
return true;
|
|
1123
|
+
}
|
|
1124
|
+
const authorized = await this.listAuthorizedMembers();
|
|
1125
|
+
if (authorized.length === 0) {
|
|
1126
|
+
return true;
|
|
1127
|
+
}
|
|
1128
|
+
if (authorized.some((m) => m.peerId === remotePeerId)) {
|
|
1129
|
+
return true;
|
|
1130
|
+
}
|
|
1131
|
+
try {
|
|
1132
|
+
if (await this.strandSolicitationService?.hasOutstandingInvitation()) {
|
|
1133
|
+
return true;
|
|
1134
|
+
}
|
|
1135
|
+
} catch (error) {
|
|
1136
|
+
log('admitInboundControlConnection: outstanding-invitation check threw for %s — admitting (fail-open): %o', remotePeerId, error);
|
|
1137
|
+
return true;
|
|
1138
|
+
}
|
|
1139
|
+
log('admitInboundControlConnection: DENYING inbound from %s — not an authorized member and no enrollment path open', remotePeerId);
|
|
1140
|
+
return false;
|
|
1141
|
+
}
|
|
1142
|
+
|
|
1143
|
+
/**
|
|
1144
|
+
* Hold the control-network inbound gate open for not-yet-authorized peers
|
|
1145
|
+
* until `untilEpochMs` (extends, never shrinks, an already-open window).
|
|
1146
|
+
* {@link createInvite} calls this automatically; a host running an
|
|
1147
|
+
* out-of-band enrollment flow (e.g. accepting a phone whose invite this node
|
|
1148
|
+
* never minted) can open it explicitly before the stranger dials in.
|
|
1149
|
+
*/
|
|
1150
|
+
openEnrollmentWindow(untilEpochMs: number): void {
|
|
1151
|
+
this.enrollmentWindowUntil = Math.max(this.enrollmentWindowUntil, untilEpochMs);
|
|
1152
|
+
log('Enrollment window open until %d', this.enrollmentWindowUntil);
|
|
1153
|
+
}
|
|
1154
|
+
|
|
1155
|
+
/**
|
|
1156
|
+
* Record (or refresh) a delegate admission grant: `delegatePeerId` is the
|
|
1157
|
+
* transport peerId of `announcerPeerId`'s strand-`strandId` node, admitted
|
|
1158
|
+
* at the CONNECTION level (all a circuit-relay reservation needs) for
|
|
1159
|
+
* `DELEGATE_GRANT_TTL_MS`. Called by the strand-addr responder after its
|
|
1160
|
+
* authorized-membership gate has passed; public so tests can drive the
|
|
1161
|
+
* admission policy without a full strand launch. A re-announce for the same
|
|
1162
|
+
* (announcer, strand) REPLACES the previous delegate rather than
|
|
1163
|
+
* accumulating. Never honored by {@link authorizeInboundControlStream}.
|
|
1164
|
+
*/
|
|
1165
|
+
grantDelegateAdmission(announcerPeerId: string, strandId: string, delegatePeerId: string): void {
|
|
1166
|
+
this.delegateAdmission.grant(announcerPeerId, strandId, delegatePeerId);
|
|
1167
|
+
}
|
|
1168
|
+
|
|
1169
|
+
/** Is `remotePeerId` covered by a live delegate admission grant? */
|
|
1170
|
+
hasDelegateAdmission(remotePeerId: string): boolean {
|
|
1171
|
+
return this.delegateAdmission.has(remotePeerId);
|
|
1172
|
+
}
|
|
1173
|
+
|
|
1174
|
+
/**
|
|
1175
|
+
* The admission checks SHARED by the connection gate
|
|
1176
|
+
* ({@link admitInboundControlConnection}) and the per-stream control-DB gate
|
|
1177
|
+
* ({@link authorizeInboundControlStream}) — factored so the two layers cannot
|
|
1178
|
+
* drift. All in-memory and cheap (the stream gate runs this per inbound
|
|
1179
|
+
* stream). Returns true when the peer must be admitted BEFORE any membership
|
|
1180
|
+
* source is consulted:
|
|
1181
|
+
* - the node is not fully up (`start()` in progress / DB torn down) — both
|
|
1182
|
+
* gates exist from libp2p bring-up, before the control DB does;
|
|
1183
|
+
* - the trusted-owner anchor is absent or empty — an un-enrolled node has
|
|
1184
|
+
* no basis to judge anyone and MUST accept its enrollment seed;
|
|
1185
|
+
* - the peer is one of the configured control bootstrap/relay nodes —
|
|
1186
|
+
* operator-configured infrastructure, not cadre members.
|
|
1187
|
+
* False only means "no unconditional admit": the caller then judges the peer
|
|
1188
|
+
* against its own membership source (a live DB read for the fail-open
|
|
1189
|
+
* connection gate; the materialized snapshot for the fail-closed stream gate).
|
|
1190
|
+
*/
|
|
1191
|
+
private admitControlPeerUnconditionally(remotePeerId: string): boolean {
|
|
1192
|
+
if (!this._running || !this.controlDatabase) {
|
|
1193
|
+
return true;
|
|
1194
|
+
}
|
|
1195
|
+
if (!this.trustedOwnerStore || this.trustedOwnerStore.all().size === 0) {
|
|
1196
|
+
return true;
|
|
1197
|
+
}
|
|
1198
|
+
return this.getBootstrapPeerIds().has(remotePeerId);
|
|
1199
|
+
}
|
|
1200
|
+
|
|
1201
|
+
/**
|
|
1202
|
+
* Per-stream authorization for the four Optimystic control-DB protocols
|
|
1203
|
+
* (`/optimystic/control-<party>/{repo,cluster,sync,block-transfer}/…`),
|
|
1204
|
+
* wired as `authorizeInboundStream` in {@link createControlNode} — the
|
|
1205
|
+
* fail-closed layer behind the fail-open connection gater. Control node
|
|
1206
|
+
* ONLY: strand cohort nodes legitimately serve cross-party peers.
|
|
1207
|
+
*
|
|
1208
|
+
* The STRICT SUBSET of {@link admitInboundControlConnection}: the same
|
|
1209
|
+
* "no basis to judge" admissions (shared via
|
|
1210
|
+
* {@link admitControlPeerUnconditionally}), minus the stranger carve-outs
|
|
1211
|
+
* (enrollment window, outstanding open invitation, delegate admission
|
|
1212
|
+
* grant). The first two exist so a stranger can reach `/sereus/seed/1.0.0`
|
|
1213
|
+
* and `/sereus/formation/1.0.0` — neither is gated here, and admitting a
|
|
1214
|
+
* stranger to `repo` during an enrollment window is exactly the hole this
|
|
1215
|
+
* gate closes. A DELEGATE-admitted connection (a member's strand node
|
|
1216
|
+
* holding a circuit-relay reservation, see `delegate-admission.ts`) is
|
|
1217
|
+
* likewise exactly a case this gate must still refuse: the delegate gets
|
|
1218
|
+
* the connection and never the control DB.
|
|
1219
|
+
*
|
|
1220
|
+
* Admits when ANY of:
|
|
1221
|
+
* 1. a shared-baseline check admits (not running / no DB, empty anchor,
|
|
1222
|
+
* configured bootstrap infra);
|
|
1223
|
+
* 2. the materialized authorized set is empty — cold start, before the
|
|
1224
|
+
* rows that would authorize anyone have replicated in;
|
|
1225
|
+
* 3. the peer IS in the materialized authorized set.
|
|
1226
|
+
*
|
|
1227
|
+
* Deliberately SYNCHRONOUS and pure in-memory — see
|
|
1228
|
+
* {@link authorizedControlPeers} for why a control-DB read here would
|
|
1229
|
+
* deadlock into mutual denial. With a sync predicate the upstream deadline
|
|
1230
|
+
* (`authorizeInboundStreamTimeoutMs`) never trips, so it stays at its
|
|
1231
|
+
* default.
|
|
1232
|
+
*
|
|
1233
|
+
* NOTE: the snapshot keys on PEER ID, so a node with no persistent identity
|
|
1234
|
+
* key never lands a `CadrePeer` row ({@link registerSelf} skips) and its
|
|
1235
|
+
* siblings deny its control-DB streams once their own snapshot is non-empty —
|
|
1236
|
+
* owner status is a KEY, not a peer id, so being the owner does not exempt it.
|
|
1237
|
+
* Harmless today (a real deployment persists its identity; the ephemeral-owner
|
|
1238
|
+
* case appears only in tests, which configure the owner as bootstrap infra —
|
|
1239
|
+
* `push-wake-e2e.integration.ts` design note 3). If an ephemeral-identity node
|
|
1240
|
+
* ever becomes a supported deployment, this gate needs a key-based admission.
|
|
1241
|
+
*/
|
|
1242
|
+
private authorizeInboundControlStream(remotePeerId: string, protocol: string): boolean {
|
|
1243
|
+
if (this.admitControlPeerUnconditionally(remotePeerId)) {
|
|
1244
|
+
return true;
|
|
1245
|
+
}
|
|
1246
|
+
if (this.authorizedControlPeers.size === 0) {
|
|
1247
|
+
return true;
|
|
1248
|
+
}
|
|
1249
|
+
if (this.authorizedControlPeers.has(remotePeerId)) {
|
|
1250
|
+
return true;
|
|
1251
|
+
}
|
|
1252
|
+
// Upstream logs its own denial line; this one carries the WHY.
|
|
1253
|
+
log('authorizeInboundControlStream: DENYING %s on %s — not in the materialized authorized set (%d member(s))',
|
|
1254
|
+
remotePeerId, protocol, this.authorizedControlPeers.size);
|
|
1255
|
+
return false;
|
|
1256
|
+
}
|
|
1257
|
+
|
|
1258
|
+
/**
|
|
1259
|
+
* Refresh {@link authorizedControlPeers} from the control DB, best-effort: a
|
|
1260
|
+
* failed read keeps the previous snapshot (never clears it), so a transient
|
|
1261
|
+
* DB error can neither flip the stream gate's cold-start carve-out back open
|
|
1262
|
+
* nor drop a legitimate member mid-flight. Never rejects.
|
|
1263
|
+
*
|
|
1264
|
+
* The SOLE caller is {@link drainMembershipGate}, which serializes refreshes —
|
|
1265
|
+
* so two reads can no longer settle out of order.
|
|
1266
|
+
*/
|
|
1267
|
+
private async refreshAuthorizedControlPeers(reason: string): Promise<void> {
|
|
1268
|
+
if (!this._running || !this.controlDatabase) {
|
|
1269
|
+
return;
|
|
1270
|
+
}
|
|
1271
|
+
try {
|
|
1272
|
+
const members = await this.listAuthorizedMembers();
|
|
1273
|
+
this.authorizedControlPeers = new Set(members.map((m) => m.peerId));
|
|
1274
|
+
log('refreshAuthorizedControlPeers(%s): %d authorized peer(s)', reason, this.authorizedControlPeers.size);
|
|
1275
|
+
} catch (error) {
|
|
1276
|
+
log('refreshAuthorizedControlPeers(%s) failed — keeping previous snapshot: %o', reason, error);
|
|
1277
|
+
}
|
|
1278
|
+
}
|
|
1279
|
+
|
|
1280
|
+
/**
|
|
1281
|
+
* PeerIds of the configured control bootstrap nodes (relay/bootstrap
|
|
1282
|
+
* infrastructure — always admitted inbound). Parsed once, lazily; an
|
|
1283
|
+
* address without a `/p2p/<id>` component contributes nothing.
|
|
1284
|
+
*/
|
|
1285
|
+
private getBootstrapPeerIds(): Set<string> {
|
|
1286
|
+
if (!this.bootstrapPeerIds) {
|
|
1287
|
+
const ids = new Set<string>();
|
|
1288
|
+
for (const addr of this.config.controlNetwork.bootstrapNodes) {
|
|
1289
|
+
try {
|
|
1290
|
+
// Last `/p2p/<id>` component: on a plain bootstrap addr the node's own
|
|
1291
|
+
// id; on a circuit addr the dial target (which is what gets admitted).
|
|
1292
|
+
let id: string | undefined;
|
|
1293
|
+
for (const component of multiaddr(addr).getComponents()) {
|
|
1294
|
+
if (component.name === 'p2p' && component.value) {
|
|
1295
|
+
id = component.value;
|
|
1296
|
+
}
|
|
1297
|
+
}
|
|
1298
|
+
if (id) {
|
|
1299
|
+
ids.add(id);
|
|
1300
|
+
}
|
|
1301
|
+
} catch (error) {
|
|
1302
|
+
log('getBootstrapPeerIds: skipping unparsable bootstrap addr %s: %o', addr, error);
|
|
1303
|
+
}
|
|
1304
|
+
}
|
|
1305
|
+
this.bootstrapPeerIds = ids;
|
|
1306
|
+
}
|
|
1307
|
+
return this.bootstrapPeerIds;
|
|
653
1308
|
}
|
|
654
1309
|
|
|
655
1310
|
private createStrandQueryable(): StrandQueryable {
|
|
@@ -702,7 +1357,11 @@ export class CadreNode implements SAppIdLookup {
|
|
|
702
1357
|
* PeerId (the resolved node identity from `keyStore`/`config.privateKey`).
|
|
703
1358
|
* - If the row already exists: a self-signed UPDATE bumping `UpdatedAt`.
|
|
704
1359
|
* - If not, and the node is its own owner: an owner-signed INSERT that
|
|
705
|
-
* also carries the self-signature.
|
|
1360
|
+
* also carries the self-signature. That INSERT is idempotent, so if an owner
|
|
1361
|
+
* {@link authorizePeer} of this node's own id seated the row inside the
|
|
1362
|
+
* read-then-insert window it no-ops — the publish then falls through to the
|
|
1363
|
+
* UPDATE path (re-reading and re-signing against the row that landed) rather
|
|
1364
|
+
* than leaving the authorize's null `Sig` in place, and reports `refreshed`.
|
|
706
1365
|
* - Otherwise: logs and returns (a non-owner node with no row yet must
|
|
707
1366
|
* wait for an owner to insert it; it can then self-refresh).
|
|
708
1367
|
*
|
|
@@ -746,29 +1405,67 @@ export class CadreNode implements SAppIdLookup {
|
|
|
746
1405
|
const addrs = await this.collectSelfAddrs();
|
|
747
1406
|
const existing = await this.controlDatabase.queryPeerRecord(peerId);
|
|
748
1407
|
|
|
749
|
-
|
|
1408
|
+
if (!existing) {
|
|
1409
|
+
if (!this.seedBootstrapService) {
|
|
1410
|
+
log('registerSelf: not yet a CadrePeer member and no owner service to self-insert; skipping (an owner must add this peer first)');
|
|
1411
|
+
return 'skipped';
|
|
1412
|
+
}
|
|
1413
|
+
// First-time row: requires an owner signature (the node is its own
|
|
1414
|
+
// owner). insertSelfPeerRecord throws if no owner key is present.
|
|
1415
|
+
const record = this.signSelfRecord(peerId, signingKey, addrs, null);
|
|
1416
|
+
if (await this.seedBootstrapService.insertSelfPeerRecord(record)) {
|
|
1417
|
+
if (this.committedAlone()) this.pendingSelfPeerWrite = true;
|
|
1418
|
+
log('registerSelf: inserted own CadrePeer record (owner-signed, updatedAt=%d, %d addrs, sig=%s…)', record.updatedAt, addrs.length, record.sig.slice(0, 16));
|
|
1419
|
+
return 'inserted';
|
|
1420
|
+
}
|
|
1421
|
+
// An owner authorize of this node's OWN peer id seated the row (null `Sig`)
|
|
1422
|
+
// inside the read-then-insert window, so the idempotent insert no-op'd. Fall
|
|
1423
|
+
// through to the self-update path, which re-signs against the row that landed.
|
|
1424
|
+
//
|
|
1425
|
+
// NOTE: this path reports 'refreshed' — honest about the write, which really was
|
|
1426
|
+
// an UPDATE, though the caller's intent was a first publish. No caller branches on
|
|
1427
|
+
// 'inserted' today (the CLI only logs it); if one ever needs "this was the row's
|
|
1428
|
+
// first publish", add a fourth SelfRegistrationOutcome rather than re-labelling.
|
|
1429
|
+
log('registerSelf: own CadrePeer row appeared mid-publish (concurrent authorize); self-updating to carry the signature');
|
|
1430
|
+
}
|
|
1431
|
+
|
|
1432
|
+
// Null only on the fall-through above, where the pre-race read missed the row a
|
|
1433
|
+
// concurrent authorize then seated — so re-read to get that row's `UpdatedAt`, which
|
|
1434
|
+
// the strictly-greater self-update rule is measured against.
|
|
1435
|
+
//
|
|
1436
|
+
// NOTE: when `existing` is set it predates this publish, so a removePeer(self) racing
|
|
1437
|
+
// in would leave the UPDATE below matching no rows while still reporting 'refreshed'.
|
|
1438
|
+
// Closing that needs updateSelfPeerRecord to report rows-affected — only worth doing
|
|
1439
|
+
// if removing self ever becomes something that happens concurrently in practice.
|
|
1440
|
+
const current = existing ?? await this.controlDatabase.queryPeerRecord(peerId);
|
|
1441
|
+
if (!current) {
|
|
1442
|
+
log('registerSelf: own CadrePeer row vanished mid-publish; skipping (next refresh re-inserts)');
|
|
1443
|
+
return 'skipped';
|
|
1444
|
+
}
|
|
1445
|
+
const record = this.signSelfRecord(peerId, signingKey, addrs, current);
|
|
1446
|
+
await this.controlDatabase.updateSelfPeerRecord(record);
|
|
1447
|
+
if (this.committedAlone()) this.pendingSelfPeerWrite = true;
|
|
1448
|
+
log('registerSelf: refreshed own CadrePeer record (updatedAt=%d, %d addrs, sig=%s…)', record.updatedAt, addrs.length, record.sig.slice(0, 16));
|
|
1449
|
+
return 'refreshed';
|
|
1450
|
+
}
|
|
1451
|
+
|
|
1452
|
+
/**
|
|
1453
|
+
* Sign this node's address record for publication, stamped strictly later than
|
|
1454
|
+
* `existing` (the row it is about to replace, or null for a first insert) so a
|
|
1455
|
+
* same-millisecond re-publish still satisfies the monotonic `UpdatedAt` rule the
|
|
1456
|
+
* `CadrePeer.AuthorizedUpdate` self-branch enforces.
|
|
1457
|
+
*/
|
|
1458
|
+
private signSelfRecord(
|
|
1459
|
+
peerId: string,
|
|
1460
|
+
signingKey: { privateKeyB64: string; publicKeyB64: string },
|
|
1461
|
+
addrs: string[],
|
|
1462
|
+
existing: PeerAddressRecord | null
|
|
1463
|
+
): PeerAddressRecord {
|
|
750
1464
|
const updatedAt = Math.max(Date.now(), (existing?.updatedAt ?? 0) + 1);
|
|
751
|
-
|
|
1465
|
+
return signPeerRecord(
|
|
752
1466
|
{ peerId, publicKey: signingKey.publicKeyB64, addrs, updatedAt },
|
|
753
1467
|
signingKey.privateKeyB64
|
|
754
1468
|
);
|
|
755
|
-
|
|
756
|
-
if (existing) {
|
|
757
|
-
await this.controlDatabase.updateSelfPeerRecord(record);
|
|
758
|
-
if (this.committedAlone()) this.pendingSelfPeerWrite = true;
|
|
759
|
-
log('registerSelf: refreshed own CadrePeer record (updatedAt=%d, %d addrs)', updatedAt, addrs.length);
|
|
760
|
-
return 'refreshed';
|
|
761
|
-
}
|
|
762
|
-
if (this.seedBootstrapService) {
|
|
763
|
-
// First-time row: requires an owner signature (the node is its own
|
|
764
|
-
// owner). insertSelfPeerRecord throws if no owner key is present.
|
|
765
|
-
await this.seedBootstrapService.insertSelfPeerRecord(record);
|
|
766
|
-
if (this.committedAlone()) this.pendingSelfPeerWrite = true;
|
|
767
|
-
log('registerSelf: inserted own CadrePeer record (owner-signed, updatedAt=%d, %d addrs)', updatedAt, addrs.length);
|
|
768
|
-
return 'inserted';
|
|
769
|
-
}
|
|
770
|
-
log('registerSelf: not yet a CadrePeer member and no owner service to self-insert; skipping (an owner must add this peer first)');
|
|
771
|
-
return 'skipped';
|
|
772
1469
|
}
|
|
773
1470
|
|
|
774
1471
|
/**
|
|
@@ -914,13 +1611,17 @@ export class CadreNode implements SAppIdLookup {
|
|
|
914
1611
|
this.turnConnectionCloseHandler = null;
|
|
915
1612
|
|
|
916
1613
|
// Reset the write-while-alone re-replication state so a stop()→start() cycle
|
|
917
|
-
// re-arms the growth edge and re-runs
|
|
918
|
-
//
|
|
919
|
-
//
|
|
920
|
-
//
|
|
1614
|
+
// re-arms the growth edge and re-runs BOTH one-shot passes. Any queued
|
|
1615
|
+
// local-only writes are moot once the node is torn down: owner inserts are
|
|
1616
|
+
// re-covered by the next start's reconstruction, and delete tombstones by its
|
|
1617
|
+
// first-growth revocation sweep — which is why the sweep flag must reset too,
|
|
1618
|
+
// or the second lifetime would skip the sweep and strand exactly the
|
|
1619
|
+
// committed-alone tombstones it exists to carry.
|
|
921
1620
|
this.hasControlConnection = false;
|
|
922
1621
|
this.reconstructedLocalOnlyWrites = false;
|
|
1622
|
+
this.reissuedHeldRevocations = false;
|
|
923
1623
|
this.pendingPeerWrites.clear();
|
|
1624
|
+
this.pendingRevocations.clear();
|
|
924
1625
|
this.pendingSelfPeerWrite = false;
|
|
925
1626
|
this.pendingSelfDeviceWrite = false;
|
|
926
1627
|
}
|
|
@@ -962,7 +1663,12 @@ export class CadreNode implements SAppIdLookup {
|
|
|
962
1663
|
|
|
963
1664
|
// Self-signature over (peerId, addrs, updatedAt).
|
|
964
1665
|
if (!verifyPeerRecordSignature(record)) {
|
|
965
|
-
|
|
1666
|
+
// Field detail distinguishes "holding the owner-vouch revision (Sig null,
|
|
1667
|
+
// no addrs — a not-yet-self-published row)" from a genuinely corrupt or
|
|
1668
|
+
// mixed-revision row. Sig/addrs are public replicated data.
|
|
1669
|
+
log('resolvePeerAddrs: signature verification failed for %s (updatedAt=%d, addrs=%o, sig=%s)',
|
|
1670
|
+
peerId, record.updatedAt, record.addrs,
|
|
1671
|
+
record.sig ? `${record.sig.slice(0, 16)}…` : '(empty)');
|
|
966
1672
|
return [];
|
|
967
1673
|
}
|
|
968
1674
|
|
|
@@ -1056,16 +1762,81 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1056
1762
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
1057
1763
|
return;
|
|
1058
1764
|
}
|
|
1765
|
+
// Ride this pass's cadence to refresh the per-stream gate's materialized
|
|
1766
|
+
// authorized set — membership changes that ARRIVED BY REPLICATION (rather
|
|
1767
|
+
// than a local write) are picked up here, bounding the snapshot's staleness
|
|
1768
|
+
// to the reconcile interval.
|
|
1769
|
+
// NOTE: this refresh and the sibling enumeration below each run their own
|
|
1770
|
+
// CadrePeer query (two reads per pass); if those reads ever get costly,
|
|
1771
|
+
// share one row-set across both.
|
|
1772
|
+
await this.refreshMembershipGate('reconcile');
|
|
1773
|
+
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
1774
|
+
return;
|
|
1775
|
+
}
|
|
1776
|
+
// Refresh relay delegate grants BEFORE the sibling enumeration below: a
|
|
1777
|
+
// solo cadre with a party relay has no siblings to dial but must still keep
|
|
1778
|
+
// its running strands' grants alive (a dropped relay connection re-dials
|
|
1779
|
+
// the reservation and faces the connection gate again).
|
|
1780
|
+
await this.refreshDelegateGrants();
|
|
1781
|
+
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
1782
|
+
return;
|
|
1783
|
+
}
|
|
1059
1784
|
const selfPeerId = this.controlNode.peerId.toString();
|
|
1060
1785
|
|
|
1786
|
+
// Reap guarded rows this node still holds for incarnations an already-committed
|
|
1787
|
+
// Revocation tombstone retires — the half of a removal replication cannot carry
|
|
1788
|
+
// (a delete of an already-absent row replays as nothing). Connected-only: a reap
|
|
1789
|
+
// is a write, and a write committed alone is local-only and forks this node's own
|
|
1790
|
+
// history, which is the condition this work exists to stop creating
|
|
1791
|
+
// (tickets/blocked/forked-control-collection-sync-livelocks.md). Nothing is urgent
|
|
1792
|
+
// here — the stale row already reads as absent everywhere — so waiting for
|
|
1793
|
+
// connectivity costs nothing and keeps the reap fork-CLOSING rather than
|
|
1794
|
+
// fork-creating.
|
|
1795
|
+
//
|
|
1796
|
+
// BEFORE step 1, not after: the pass early-returns when it finds no siblings, and a
|
|
1797
|
+
// cadre whose only sibling row is the tombstoned one lands in exactly that branch
|
|
1798
|
+
// (listMembers() filters retired stamps, so the row needing the reap is invisible to
|
|
1799
|
+
// the sibling count). Placed after step 1 it would never run for the smallest and
|
|
1800
|
+
// most likely case.
|
|
1801
|
+
//
|
|
1802
|
+
// Shares no mutable state with drainPendingRevocations, which re-issues tombstones
|
|
1803
|
+
// while this consumes them: different tables (Revocation update vs guarded-row
|
|
1804
|
+
// delete) and both take the write lock per statement. The two can overlap freely —
|
|
1805
|
+
// do not add a mutex.
|
|
1806
|
+
//
|
|
1807
|
+
// NOTE: the gate is sampled ONCE for the whole sweep, so a disconnect landing
|
|
1808
|
+
// mid-sweep lets the remaining rows commit alone — the fork this gate exists to
|
|
1809
|
+
// avoid. Bounded by how long a sweep runs, which is bounded by the backlog of
|
|
1810
|
+
// unreaped rows (0 in steady state). If that backlog can ever be large, move the
|
|
1811
|
+
// check inside reapRevokedRows' loop via an injected predicate.
|
|
1812
|
+
if (this.getControlConnectionCount() > 0) {
|
|
1813
|
+
try {
|
|
1814
|
+
const reaped = await this.controlDatabase.reapRevokedRows(selfPeerId);
|
|
1815
|
+
if (reaped > 0) {
|
|
1816
|
+
log('reconcileControlCohort: reaped %d revoked control row(s)', reaped);
|
|
1817
|
+
}
|
|
1818
|
+
} catch (error) {
|
|
1819
|
+
// Best-effort like every other step in this pass: reconnecting siblings
|
|
1820
|
+
// outranks garbage collection, so a reap failure never aborts the reconcile.
|
|
1821
|
+
log('reconcileControlCohort: reap pass failed (continuing): %o', error);
|
|
1822
|
+
}
|
|
1823
|
+
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
1824
|
+
return;
|
|
1825
|
+
}
|
|
1826
|
+
}
|
|
1827
|
+
|
|
1061
1828
|
// 1. Enumerate known siblings. This membership read is itself a pull-on-read
|
|
1062
1829
|
// that helps the CadrePeer table converge — a reader-only node converges
|
|
1063
1830
|
// purely by these reads.
|
|
1064
1831
|
const members = await this.listMembers();
|
|
1065
1832
|
const siblings = members.filter((m) => m.peerId !== selfPeerId);
|
|
1066
1833
|
if (siblings.length === 0) {
|
|
1067
|
-
//
|
|
1068
|
-
// dial
|
|
1834
|
+
// No rows to dial from. Either a solo cadre (nothing to do) or a COLD
|
|
1835
|
+
// START whose seed dial never landed — and those look identical from here,
|
|
1836
|
+
// because filling this table needs a connection and getting a connection
|
|
1837
|
+
// needs this table. Fall back to the seed's bootstrap addresses; a solo
|
|
1838
|
+
// cadre has none and this is a no-op. Must not throw or busy-loop.
|
|
1839
|
+
await this.dialColdStartBootstrap();
|
|
1069
1840
|
return;
|
|
1070
1841
|
}
|
|
1071
1842
|
// Re-guard after the await: a stop() may have raced the membership read.
|
|
@@ -1074,6 +1845,11 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1074
1845
|
}
|
|
1075
1846
|
|
|
1076
1847
|
// 2. Classify backbone (owner) members and select a bounded dial set.
|
|
1848
|
+
// NOTE: deliberately the REPLICATED table, not the node-local anchor — this
|
|
1849
|
+
// only *prefers* owner peers as dial targets, so pollution costs a wasted
|
|
1850
|
+
// dial while anchoring would drop legitimate co-owners this node never
|
|
1851
|
+
// pinned. Same call as `SeedPeer.isOwner` in seed-bootstrap.ts. Move it to
|
|
1852
|
+
// the anchor if owner status here ever gates something trusted.
|
|
1077
1853
|
const ownerKeys = await this.controlDatabase.getOwnerKeys();
|
|
1078
1854
|
if (!this._running || !this.controlNode) {
|
|
1079
1855
|
return;
|
|
@@ -1173,6 +1949,154 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1173
1949
|
}
|
|
1174
1950
|
}
|
|
1175
1951
|
|
|
1952
|
+
/**
|
|
1953
|
+
* Retain a just-applied seed's owner-flagged peers as cold-start bootstrap
|
|
1954
|
+
* dial targets (see {@link bootstrapPeerStore}).
|
|
1955
|
+
*
|
|
1956
|
+
* Called for every seed this node accepts, on BOTH intake paths — the
|
|
1957
|
+
* {@link applySeed} wrapper (which may run on a throwaway service, so the
|
|
1958
|
+
* service itself is the wrong place to keep this) and the inbound
|
|
1959
|
+
* `/sereus/seed/1.0.0` handler via `onSeedApplied`. Peers with no address are
|
|
1960
|
+
* skipped: there is nothing to dial.
|
|
1961
|
+
*
|
|
1962
|
+
* `isOwner` is the seed's own claim about its peers, exactly as
|
|
1963
|
+
* `SeedBootstrapService.applySeed` uses it to choose its dial targets. That is
|
|
1964
|
+
* sound here for the same reason it is sound there — the whole seed is
|
|
1965
|
+
* signature-checked against a trust-anchored signer before this runs, and the
|
|
1966
|
+
* flag only *selects a dial target*; a dial grants no authority.
|
|
1967
|
+
*
|
|
1968
|
+
* Self is excluded. `createSeed` projects EVERY `CadrePeer` row, so an owner
|
|
1969
|
+
* that applies a seed minted after it joined finds itself in the owner list;
|
|
1970
|
+
* retaining that would make the cold-start pass dial this node forever (the
|
|
1971
|
+
* steady-state pass filters self out of its sibling list for the same reason).
|
|
1972
|
+
*/
|
|
1973
|
+
private recordSeedBootstrapPeers(seed: ControlNetworkSeed): void {
|
|
1974
|
+
const store = this.bootstrapPeerStore;
|
|
1975
|
+
if (!store) {
|
|
1976
|
+
// Unreachable in production: start() builds the store before any network
|
|
1977
|
+
// bring-up, and both intake paths need a started node (an uninitialized
|
|
1978
|
+
// SeedBootstrapService rejects the seed, so noteAppliedSeed returns early).
|
|
1979
|
+
log('recordSeedBootstrapPeers: no bootstrap-peer store yet; retaining nothing');
|
|
1980
|
+
return;
|
|
1981
|
+
}
|
|
1982
|
+
const selfPeerId = this.controlNode?.peerId.toString();
|
|
1983
|
+
for (const peer of seed.peers) {
|
|
1984
|
+
if (!peer.isOwner || peer.multiaddrs.length === 0 || peer.peerId === selfPeerId) {
|
|
1985
|
+
continue;
|
|
1986
|
+
}
|
|
1987
|
+
// Sync-visible by contract; the promise tracks durability only. A persist
|
|
1988
|
+
// failure costs restart survival, never this session's retry set — the same
|
|
1989
|
+
// trade `SeedBootstrapService.anchorAcceptedSigner` makes — so log and carry
|
|
1990
|
+
// on rather than failing the seed that was already accepted.
|
|
1991
|
+
void store.record(peer.peerId, peer.multiaddrs).catch((error: unknown) => {
|
|
1992
|
+
log('recordSeedBootstrapPeers: persisting bootstrap peer %s failed (retained in memory): %o',
|
|
1993
|
+
peer.peerId, error);
|
|
1994
|
+
});
|
|
1995
|
+
}
|
|
1996
|
+
}
|
|
1997
|
+
|
|
1998
|
+
/**
|
|
1999
|
+
* Cold-start branch of the reconcile pass: re-dial the seed's owner peers
|
|
2000
|
+
* while the control database still has no siblings to dial.
|
|
2001
|
+
*
|
|
2002
|
+
* `SeedBootstrapService.applySeed` dials those owners exactly ONCE, best-effort.
|
|
2003
|
+
* When that single dial fails — owner momentarily down, relay reservation not
|
|
2004
|
+
* up yet, NAT traversal lost the race — the joining node has an empty
|
|
2005
|
+
* `CadrePeer` table and no connection, and the steady-state pass above cannot
|
|
2006
|
+
* help: it dials only siblings enumerated from that very table. Retrying here
|
|
2007
|
+
* is the only way back in.
|
|
2008
|
+
*
|
|
2009
|
+
* Unbounded, at the reconcile cadence, with no backoff: the branch is already
|
|
2010
|
+
* gated by "the control database has no siblings", so it stops the moment the
|
|
2011
|
+
* node is actually in the party, and a node that is stranded MUST keep trying —
|
|
2012
|
+
* a give-up rule would turn a transient outage into a permanent one. The steady
|
|
2013
|
+
* cost is one dial per bootstrap peer per pass (seeds nominate one or a few
|
|
2014
|
+
* owners), each failing fast against an unreachable address.
|
|
2015
|
+
* NOTE: if seeds ever carry many owner peers, or the reconcile cadence tightens
|
|
2016
|
+
* well below its 15 s default, add per-peer backoff here.
|
|
2017
|
+
*
|
|
2018
|
+
* Best-effort per peer, exactly like {@link dialControlSibling}: one dead
|
|
2019
|
+
* address never aborts the pass.
|
|
2020
|
+
*/
|
|
2021
|
+
private async dialColdStartBootstrap(): Promise<void> {
|
|
2022
|
+
const controlNode = this.controlNode;
|
|
2023
|
+
// Snapshot up front: the store hands back a copy, so a seed applied mid-pass
|
|
2024
|
+
// cannot mutate what we are iterating.
|
|
2025
|
+
const targets = this.bootstrapPeerStore?.all();
|
|
2026
|
+
if (!controlNode || !targets || targets.size === 0) {
|
|
2027
|
+
return;
|
|
2028
|
+
}
|
|
2029
|
+
// Same skip rule as step 3 of the steady-state pass: no churn on live links.
|
|
2030
|
+
const connected = new Set(controlNode.getConnections().map((c) => c.remotePeer.toString()));
|
|
2031
|
+
let dialed = 0;
|
|
2032
|
+
for (const [peerId, entry] of targets) {
|
|
2033
|
+
if (!this._running || !this.controlNode) {
|
|
2034
|
+
return;
|
|
2035
|
+
}
|
|
2036
|
+
if (connected.has(peerId)) {
|
|
2037
|
+
continue;
|
|
2038
|
+
}
|
|
2039
|
+
if (await this.dialBootstrapPeer(peerId, entry.addrs)) {
|
|
2040
|
+
dialed++;
|
|
2041
|
+
}
|
|
2042
|
+
}
|
|
2043
|
+
log('reconcileControlCohort: cold-start pass complete (bootstrap=%d, connected=%d, dialed=%d)',
|
|
2044
|
+
targets.size, connected.size, dialed);
|
|
2045
|
+
}
|
|
2046
|
+
|
|
2047
|
+
/**
|
|
2048
|
+
* Bind a bootstrap peer's seed addresses to its peer id, so the dial
|
|
2049
|
+
* authenticates the peer it is aiming at rather than trusting whoever answers.
|
|
2050
|
+
*
|
|
2051
|
+
* An address that already carries `/p2p/<id>` must carry THIS peer's id or it
|
|
2052
|
+
* is dropped (a seed that disagrees with itself is not a dial target); one
|
|
2053
|
+
* that carries none — an `addressOverrides`-published record, say — gets the
|
|
2054
|
+
* id encapsulated. Unencapsulatable addresses are dropped rather than dialed
|
|
2055
|
+
* bare.
|
|
2056
|
+
*/
|
|
2057
|
+
private bootstrapDialAddrs(peerId: string, addrs: string[]): Multiaddr[] {
|
|
2058
|
+
const out: Multiaddr[] = [];
|
|
2059
|
+
for (const addr of this.parseMultiaddrs(addrs)) {
|
|
2060
|
+
const embedded = trailingPeerId(addr);
|
|
2061
|
+
if (embedded === peerId) {
|
|
2062
|
+
out.push(addr);
|
|
2063
|
+
continue;
|
|
2064
|
+
}
|
|
2065
|
+
if (embedded !== null) {
|
|
2066
|
+
log('reconcileControlCohort(cold-start): addr %s names peer %s, not %s; dropping',
|
|
2067
|
+
addr.toString(), embedded, peerId);
|
|
2068
|
+
continue;
|
|
2069
|
+
}
|
|
2070
|
+
try {
|
|
2071
|
+
out.push(addr.encapsulate(`/p2p/${peerId}`));
|
|
2072
|
+
} catch (error) {
|
|
2073
|
+
log('reconcileControlCohort(cold-start): cannot bind %s to %s: %o', addr.toString(), peerId, error);
|
|
2074
|
+
}
|
|
2075
|
+
}
|
|
2076
|
+
return out;
|
|
2077
|
+
}
|
|
2078
|
+
|
|
2079
|
+
/** Dial one cold-start bootstrap peer, best-effort. Returns whether it connected. */
|
|
2080
|
+
private async dialBootstrapPeer(peerId: string, addrs: string[]): Promise<boolean> {
|
|
2081
|
+
const controlNode = this.controlNode;
|
|
2082
|
+
if (!controlNode) {
|
|
2083
|
+
return false;
|
|
2084
|
+
}
|
|
2085
|
+
const parsed = this.bootstrapDialAddrs(peerId, addrs);
|
|
2086
|
+
if (parsed.length === 0) {
|
|
2087
|
+
log('reconcileControlCohort(cold-start): no dialable address for bootstrap peer %s; skipping', peerId);
|
|
2088
|
+
return false;
|
|
2089
|
+
}
|
|
2090
|
+
try {
|
|
2091
|
+
log('reconcileControlCohort(cold-start): dialing bootstrap peer %s (%d addr(s))', peerId, parsed.length);
|
|
2092
|
+
await controlNode.dial(parsed);
|
|
2093
|
+
return true;
|
|
2094
|
+
} catch (error) {
|
|
2095
|
+
log('reconcileControlCohort(cold-start): dial of bootstrap peer %s failed (continuing): %o', peerId, error);
|
|
2096
|
+
return false;
|
|
2097
|
+
}
|
|
2098
|
+
}
|
|
2099
|
+
|
|
1176
2100
|
// ============================================================================
|
|
1177
2101
|
// Write-while-alone re-replication (control-write-ensure-replicated)
|
|
1178
2102
|
//
|
|
@@ -1198,9 +2122,12 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1198
2122
|
* is caught by the cohort's periodic pull-on-read instead. Re-issuing an already-
|
|
1199
2123
|
* replicated row is harmless (an idempotent monotonic bump), so the coarse proxy
|
|
1200
2124
|
* is acceptable as the agreed first cut.
|
|
2125
|
+
*
|
|
2126
|
+
* Reads {@link getControlConnectionCount} — the public, embedder-facing form of
|
|
2127
|
+
* the same sample — so the two cannot drift.
|
|
1201
2128
|
*/
|
|
1202
2129
|
private committedAlone(): boolean {
|
|
1203
|
-
return
|
|
2130
|
+
return this.getControlConnectionCount() === 0;
|
|
1204
2131
|
}
|
|
1205
2132
|
|
|
1206
2133
|
/**
|
|
@@ -1210,26 +2137,48 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1210
2137
|
* cohort grows; otherwise the write replicated and any stale queue entry for this
|
|
1211
2138
|
* subject is dropped.
|
|
1212
2139
|
*
|
|
1213
|
-
* A `remove`
|
|
1214
|
-
*
|
|
1215
|
-
*
|
|
1216
|
-
*
|
|
2140
|
+
* A `remove` never queues here — a delete's re-replicable half is its
|
|
2141
|
+
* `Revocation` tombstone, which the committed-delete seam routes into
|
|
2142
|
+
* {@link pendingRevocations} ({@link noteGuardedDelete}). The `remove` arm only
|
|
2143
|
+
* drops any queued authorize for the subject (the row is gone; re-issuing the
|
|
2144
|
+
* insert would resurrect it) and, when the commit was alone, logs the
|
|
2145
|
+
* security-relevant window loudly.
|
|
1217
2146
|
*/
|
|
1218
2147
|
private noteControlWrite(peerId: string, kind: 'authorize' | 'remove'): void {
|
|
2148
|
+
if (kind === 'remove') {
|
|
2149
|
+
this.pendingPeerWrites.delete(peerId);
|
|
2150
|
+
if (this.committedAlone()) {
|
|
2151
|
+
log('removePeer(%s) committed while ALONE (0 control connections): the deletion is ' +
|
|
2152
|
+
'local-only, so a revoked peer may persist in the cohort until the queued ' +
|
|
2153
|
+
'Revocation tombstone is re-issued on cohort growth (drainPendingRevocations).', peerId);
|
|
2154
|
+
}
|
|
2155
|
+
return;
|
|
2156
|
+
}
|
|
1219
2157
|
if (!this.committedAlone()) {
|
|
1220
2158
|
this.pendingPeerWrites.delete(peerId);
|
|
1221
2159
|
return;
|
|
1222
2160
|
}
|
|
1223
|
-
this.pendingPeerWrites.set(peerId,
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
2161
|
+
this.pendingPeerWrites.set(peerId, 'authorize');
|
|
2162
|
+
log('authorizePeer(%s) committed while alone (0 control connections); queued for ' +
|
|
2163
|
+
're-replication on cohort growth', peerId);
|
|
2164
|
+
}
|
|
2165
|
+
|
|
2166
|
+
/**
|
|
2167
|
+
* Committed-delete seam handler ({@link GuardedDeleteListener}, wired in
|
|
2168
|
+
* {@link start}): a guarded-table delete and its `Revocation` tombstone
|
|
2169
|
+
* committed. If the node was alone the tombstone is local-only — queue it for
|
|
2170
|
+
* an owner-signed re-issue on cohort growth; otherwise it broadcast, so drop
|
|
2171
|
+
* any stale queue entry for the stamp. Synchronous by contract — bookkeeping
|
|
2172
|
+
* only, never throws into the delete path.
|
|
2173
|
+
*/
|
|
2174
|
+
private noteGuardedDelete(revocation: RevokedRowRef): void {
|
|
2175
|
+
if (!this.committedAlone()) {
|
|
2176
|
+
this.pendingRevocations.delete(revocation.stampId);
|
|
2177
|
+
return;
|
|
1232
2178
|
}
|
|
2179
|
+
this.pendingRevocations.set(revocation.stampId, revocation);
|
|
2180
|
+
log('%s delete of %s committed while alone (0 control connections); Revocation tombstone ' +
|
|
2181
|
+
'(stamp %s) queued for re-issue on cohort growth', revocation.tableName, revocation.rowKey, revocation.stampId);
|
|
1233
2182
|
}
|
|
1234
2183
|
|
|
1235
2184
|
/**
|
|
@@ -1261,10 +2210,23 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1261
2210
|
return;
|
|
1262
2211
|
}
|
|
1263
2212
|
const firstGrowth = !this.reconstructedLocalOnlyWrites;
|
|
1264
|
-
log('Draining control re-replication (reason=%s, firstGrowth=%s, pendingPeers=%d, self=%s, deviceToken=%s)',
|
|
1265
|
-
reason, firstGrowth, this.pendingPeerWrites.size, this.
|
|
1266
|
-
|
|
1267
|
-
|
|
2213
|
+
log('Draining control re-replication (reason=%s, firstGrowth=%s, pendingPeers=%d, pendingRevocations=%d, self=%s, deviceToken=%s)',
|
|
2214
|
+
reason, firstGrowth, this.pendingPeerWrites.size, this.pendingRevocations.size,
|
|
2215
|
+
this.pendingSelfPeerWrite, this.pendingSelfDeviceWrite);
|
|
2216
|
+
|
|
2217
|
+
// 1. Revocation tombstones FIRST: a removal that never reached the cohort
|
|
2218
|
+
// leaves a revoked peer live elsewhere, which outranks every re-touch
|
|
2219
|
+
// below. Ordering is not cosmetic — every `CadrePeer` write here (self row
|
|
2220
|
+
// included) can burn tens of seconds in upstream retry livelock after a
|
|
2221
|
+
// collection fork, while the tombstone lives in a different collection
|
|
2222
|
+
// that is merely behind, not forked. No membership-gate refresh around
|
|
2223
|
+
// this step: a re-issue only bumps `ReissuedAt` — LOCAL membership
|
|
2224
|
+
// visibility changed at delete time (deleteCadrePeer already notified),
|
|
2225
|
+
// and receiving-node staleness is the pre-existing pull-on-read /
|
|
2226
|
+
// periodic-refresh property.
|
|
2227
|
+
await this.drainPendingRevocations();
|
|
2228
|
+
|
|
2229
|
+
// 2. Self rows. On the first growth always re-touch (covers a self row carried
|
|
1268
2230
|
// over from a prior process that committed local-only before this start);
|
|
1269
2231
|
// afterwards only when a self-write-while-alone was recorded this session.
|
|
1270
2232
|
// registerSelf is idempotent + single-flight, so this never races a
|
|
@@ -1278,16 +2240,16 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1278
2240
|
this.pendingSelfDeviceWrite = false;
|
|
1279
2241
|
}
|
|
1280
2242
|
|
|
1281
|
-
//
|
|
2243
|
+
// 3. First growth: an owner reconstructs membership rows it may have
|
|
1282
2244
|
// authored that never replicated (covers writes from before this process
|
|
1283
2245
|
// started). Rows already tracked in the in-memory queue are handled by
|
|
1284
|
-
// step
|
|
2246
|
+
// step 4 — skipped here to avoid a double re-touch.
|
|
1285
2247
|
if (firstGrowth) {
|
|
1286
2248
|
this.reconstructedLocalOnlyWrites = true;
|
|
1287
2249
|
await this.reconstructAuthoredMembership();
|
|
1288
2250
|
}
|
|
1289
2251
|
|
|
1290
|
-
//
|
|
2252
|
+
// 4. In-session pending owner authorize writes.
|
|
1291
2253
|
await this.drainPendingPeerWrites();
|
|
1292
2254
|
}
|
|
1293
2255
|
|
|
@@ -1314,6 +2276,69 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1314
2276
|
}
|
|
1315
2277
|
}
|
|
1316
2278
|
|
|
2279
|
+
/**
|
|
2280
|
+
* Re-issue `Revocation` tombstones so a removal that committed while alone
|
|
2281
|
+
* still reaches the cohort. Two regimes, one method:
|
|
2282
|
+
*
|
|
2283
|
+
* - **First successful pass per process** ({@link reissuedHeldRevocations}
|
|
2284
|
+
* false): sweep EVERY locally-held tombstone from `queryRevocations()` — the
|
|
2285
|
+
* only cover for a removal made before this process started, since the
|
|
2286
|
+
* in-memory queue does not survive a restart. Queued stamps are part of that
|
|
2287
|
+
* same batch (exactly once — one transaction, no double bump).
|
|
2288
|
+
* - **Later passes**: only the tombstones queued in-session by
|
|
2289
|
+
* {@link noteGuardedDelete}.
|
|
2290
|
+
*
|
|
2291
|
+
* Owner-gated: a node with no owner key cannot sign a re-issue, so stray queue
|
|
2292
|
+
* entries are dropped (mirrors {@link drainPendingPeerWrites}). Best-effort — a
|
|
2293
|
+
* failure is logged and leaves the queue (and the sweep flag) untouched for the
|
|
2294
|
+
* next growth edge. A successful `reissueRevocations` exec is NOT proof of
|
|
2295
|
+
* broadcast (the connection that fired the growth edge may not be in the
|
|
2296
|
+
* affected block's cluster — see
|
|
2297
|
+
* tickets/backlog/control-rereplication-broadcast-confirmation); the drain
|
|
2298
|
+
* inherits that known gap, and a full disconnect→reconnect (or the next
|
|
2299
|
+
* process's sweep) re-covers it.
|
|
2300
|
+
*
|
|
2301
|
+
* NOTE: the sweep is O(all tombstones ever) row-updates — plus one owner
|
|
2302
|
+
* signature each — in one transaction, once per lifetime. `Revocation` is
|
|
2303
|
+
* append-only and unbounded growth is declared acceptable for a cadre-sized
|
|
2304
|
+
* party. If the tombstone table ever gets large, bound the sweep (e.g. persist
|
|
2305
|
+
* a node-local high-water mark of what has been re-issued while connected)
|
|
2306
|
+
* instead of re-touching everything; note the `Math.max(...)` spread below also
|
|
2307
|
+
* caps out around 10^5 rows (V8 argument limit) before the size becomes merely
|
|
2308
|
+
* a latency problem.
|
|
2309
|
+
*/
|
|
2310
|
+
private async drainPendingRevocations(): Promise<void> {
|
|
2311
|
+
const sweep = !this.reissuedHeldRevocations;
|
|
2312
|
+
if (!sweep && this.pendingRevocations.size === 0) {
|
|
2313
|
+
return;
|
|
2314
|
+
}
|
|
2315
|
+
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
2316
|
+
return;
|
|
2317
|
+
}
|
|
2318
|
+
if (!this.seedBootstrapService?.canAuthorize()) {
|
|
2319
|
+
// A non-owner node cannot sign a re-issue; drop any stray entries.
|
|
2320
|
+
this.pendingRevocations.clear();
|
|
2321
|
+
return;
|
|
2322
|
+
}
|
|
2323
|
+
try {
|
|
2324
|
+
const held = await this.controlDatabase.queryRevocations();
|
|
2325
|
+
const rows = sweep ? held : held.filter((r) => this.pendingRevocations.has(r.stampId));
|
|
2326
|
+
if (rows.length > 0) {
|
|
2327
|
+
const reissuedAt = Math.max(Date.now(), Math.max(...rows.map((r) => r.reissuedAt)) + 1);
|
|
2328
|
+
await this.seedBootstrapService.reissueRevocations(rows, reissuedAt);
|
|
2329
|
+
log('drain: re-issued %d revocation tombstone(s)%s', rows.length, sweep ? ' (first-growth sweep)' : '');
|
|
2330
|
+
}
|
|
2331
|
+
this.reissuedHeldRevocations = true;
|
|
2332
|
+
// Per-stamp clear (not clear()) so an entry queued concurrently mid-drain
|
|
2333
|
+
// survives for the next growth edge.
|
|
2334
|
+
for (const row of rows) {
|
|
2335
|
+
this.pendingRevocations.delete(row.stampId);
|
|
2336
|
+
}
|
|
2337
|
+
} catch (error) {
|
|
2338
|
+
log('drain: revocation re-issue failed; leaving queued for next growth: %o', error);
|
|
2339
|
+
}
|
|
2340
|
+
}
|
|
2341
|
+
|
|
1317
2342
|
/**
|
|
1318
2343
|
* One-shot, first-cohort-growth reconstruction of the write-while-alone queue for
|
|
1319
2344
|
* an OWNER node: re-touch every membership row that may be an unreplicated
|
|
@@ -1326,9 +2351,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1326
2351
|
* row is a no-op-equivalent).
|
|
1327
2352
|
*
|
|
1328
2353
|
* A node with no owner private key skips this entirely — it cannot re-sign
|
|
1329
|
-
* rows for other peers, and rows it merely holds are not its to re-issue.
|
|
1330
|
-
*
|
|
1331
|
-
*
|
|
2354
|
+
* rows for other peers, and rows it merely holds are not its to re-issue.
|
|
2355
|
+
* DELETEs are not reconstructed here — the `Revocation` tombstone sweep in
|
|
2356
|
+
* {@link drainPendingRevocations} covers them (a removed row leaves no
|
|
2357
|
+
* `CadrePeer` trace, but its tombstone is re-issuable).
|
|
1332
2358
|
*/
|
|
1333
2359
|
private async reconstructAuthoredMembership(): Promise<void> {
|
|
1334
2360
|
if (!this.controlDatabase || !this.controlNode) {
|
|
@@ -1339,10 +2365,29 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1339
2365
|
}
|
|
1340
2366
|
const selfPeerId = this.controlNode.peerId.toString();
|
|
1341
2367
|
const members = await this.controlDatabase.queryCadrePeers();
|
|
2368
|
+
// One gate refresh for the whole sweep: every re-touch is a notifying
|
|
2369
|
+
// `CadrePeer` write, and these rows are already in the local snapshot (they
|
|
2370
|
+
// committed here, just local-only), so per-row refreshes would be O(rows)
|
|
2371
|
+
// membership reads for a snapshot that does not change.
|
|
2372
|
+
const touched = await this.deferMembershipGateRefresh(
|
|
2373
|
+
'drain-reconstruct',
|
|
2374
|
+
() => this.reissueAuthoredMembershipRows(selfPeerId, members)
|
|
2375
|
+
);
|
|
2376
|
+
if (touched > 0) {
|
|
2377
|
+
log('drain: reconstructed %d owner-authored membership row(s) on first cohort growth', touched);
|
|
2378
|
+
}
|
|
2379
|
+
}
|
|
2380
|
+
|
|
2381
|
+
/**
|
|
2382
|
+
* Body of {@link reconstructAuthoredMembership}'s sweep: re-issue each candidate
|
|
2383
|
+
* owner-authored row, returning how many were re-touched. A per-row failure is
|
|
2384
|
+
* logged and skipped; a teardown mid-sweep stops it.
|
|
2385
|
+
*/
|
|
2386
|
+
private async reissueAuthoredMembershipRows(selfPeerId: string, members: CadrePeerRow[]): Promise<number> {
|
|
1342
2387
|
let touched = 0;
|
|
1343
2388
|
for (const member of members) {
|
|
1344
2389
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
1345
|
-
return;
|
|
2390
|
+
return touched;
|
|
1346
2391
|
}
|
|
1347
2392
|
if (member.peerId === selfPeerId || this.pendingPeerWrites.has(member.peerId)) {
|
|
1348
2393
|
continue;
|
|
@@ -1359,27 +2404,19 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1359
2404
|
log('drain: reconstruction re-touch of %s failed (continuing): %o', member.peerId, error);
|
|
1360
2405
|
}
|
|
1361
2406
|
}
|
|
1362
|
-
|
|
1363
|
-
log('drain: reconstructed %d owner-authored membership row(s) on first cohort growth', touched);
|
|
1364
|
-
}
|
|
2407
|
+
return touched;
|
|
1365
2408
|
}
|
|
1366
2409
|
|
|
1367
2410
|
/**
|
|
1368
2411
|
* Drain the in-session write-while-alone queue: re-issue each pending owner
|
|
1369
|
-
*
|
|
1370
|
-
*
|
|
1371
|
-
*
|
|
1372
|
-
*
|
|
1373
|
-
*
|
|
1374
|
-
*
|
|
1375
|
-
*
|
|
1376
|
-
*
|
|
1377
|
-
* - `remove` (security-relevant): a removePeer-while-alone physically removed the
|
|
1378
|
-
* row locally, so a re-issued DELETE matches nothing and does NOT propagate —
|
|
1379
|
-
* full delete durability requires a schema tombstone (deferred:
|
|
1380
|
-
* `control-delete-while-alone-tombstone`). We still attempt the re-issue
|
|
1381
|
-
* (harmless, and it propagates in the row-still-present case); the commit-alone
|
|
1382
|
-
* was already logged loudly in {@link noteControlWrite}.
|
|
2412
|
+
* authorize as an idempotent monotonic owner UPDATE
|
|
2413
|
+
* ({@link reissuePeerAuthorize}) now that the cohort can broadcast it.
|
|
2414
|
+
* Sequential (no fan-out) to avoid a thundering re-touch; an entry is cleared
|
|
2415
|
+
* only on success, so a failure (or a still-alone re-commit) leaves it queued
|
|
2416
|
+
* for the next growth. An entry is skipped (and cleared) if the row vanished
|
|
2417
|
+
* (raced a delete) or now carries a self-`Sig` (the peer self-published and owns
|
|
2418
|
+
* its republish). Removals are drained by {@link drainPendingRevocations}, not
|
|
2419
|
+
* here.
|
|
1383
2420
|
*/
|
|
1384
2421
|
private async drainPendingPeerWrites(): Promise<void> {
|
|
1385
2422
|
if (this.pendingPeerWrites.size === 0) {
|
|
@@ -1390,27 +2427,33 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1390
2427
|
this.pendingPeerWrites.clear();
|
|
1391
2428
|
return;
|
|
1392
2429
|
}
|
|
1393
|
-
for
|
|
1394
|
-
|
|
2430
|
+
// One gate refresh for the whole drain, not one per entry: these re-issues
|
|
2431
|
+
// replay writes this node already applied locally (and already refreshed
|
|
2432
|
+
// for), so the snapshot only has to be correct once the drain settles.
|
|
2433
|
+
await this.deferMembershipGateRefresh('drain-reissue', () => this.reissuePendingPeerWrites());
|
|
2434
|
+
}
|
|
2435
|
+
|
|
2436
|
+
/**
|
|
2437
|
+
* Body of {@link drainPendingPeerWrites}' loop. An entry is cleared only on
|
|
2438
|
+
* success (or when there is nothing left to re-issue); a failure is logged and
|
|
2439
|
+
* the entry stays queued for the next cohort growth.
|
|
2440
|
+
*/
|
|
2441
|
+
private async reissuePendingPeerWrites(): Promise<void> {
|
|
2442
|
+
for (const peerId of [...this.pendingPeerWrites.keys()]) {
|
|
2443
|
+
if (!this._running || !this.controlNode || !this.controlDatabase || !this.seedBootstrapService) {
|
|
1395
2444
|
return;
|
|
1396
2445
|
}
|
|
1397
2446
|
try {
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
continue;
|
|
1404
|
-
}
|
|
1405
|
-
await this.reissuePeerAuthorize(peerId, record.updatedAt);
|
|
1406
|
-
} else {
|
|
1407
|
-
// Best-effort delete re-issue (see method doc): propagates only if the row
|
|
1408
|
-
// is somehow still present locally.
|
|
1409
|
-
await this.seedBootstrapService.removePeer(peerId);
|
|
2447
|
+
const record = await this.controlDatabase.queryPeerRecord(peerId);
|
|
2448
|
+
if (!record || record.sig) {
|
|
2449
|
+
// Removed since, or peer self-published — nothing for the owner to re-issue.
|
|
2450
|
+
this.pendingPeerWrites.delete(peerId);
|
|
2451
|
+
continue;
|
|
1410
2452
|
}
|
|
2453
|
+
await this.reissuePeerAuthorize(peerId, record.updatedAt);
|
|
1411
2454
|
this.pendingPeerWrites.delete(peerId);
|
|
1412
2455
|
} catch (error) {
|
|
1413
|
-
log('drain: re-issue of %s (
|
|
2456
|
+
log('drain: re-issue of %s (authorize) failed; leaving queued for next growth: %o', peerId, error);
|
|
1414
2457
|
}
|
|
1415
2458
|
}
|
|
1416
2459
|
}
|
|
@@ -1582,6 +2625,13 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1582
2625
|
return;
|
|
1583
2626
|
}
|
|
1584
2627
|
if (this.seedBootstrapService) {
|
|
2628
|
+
// NOTE: the `CadrePeer` twin of this read-then-insert needed race recovery because an
|
|
2629
|
+
// owner `authorizePeer` can seat a peer's row for it. Nothing seats a `DeviceToken`
|
|
2630
|
+
// row on a peer's behalf, and this insert has no in-lock existence check, so a lost
|
|
2631
|
+
// race would THROW rather than silently drop the record. If an owner-driven
|
|
2632
|
+
// device-token seeding path is ever added, or two first-publishes can run
|
|
2633
|
+
// concurrently (this method has no single-flight guard, unlike `registerSelf`),
|
|
2634
|
+
// revisit — see `publishSelfRecord`'s fall-through.
|
|
1585
2635
|
await this.seedBootstrapService.insertSelfDeviceToken(record);
|
|
1586
2636
|
if (this.committedAlone()) this.pendingSelfDeviceWrite = true;
|
|
1587
2637
|
log('registerDeviceToken: inserted own DeviceToken (owner-signed, platform=%s, updatedAt=%d)', platform, updatedAt);
|
|
@@ -1604,8 +2654,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1604
2654
|
* 2. `publicKey <-> peerId` binding — the stored key's libp2p identity is the
|
|
1605
2655
|
* requested peerId,
|
|
1606
2656
|
* 3. a `DeviceToken` row exists with a known {@link PushPlatform},
|
|
1607
|
-
* 4.
|
|
1608
|
-
*
|
|
2657
|
+
* 4. the row's `StampId` is NOT retired in `CadreControl.Revocation` — the
|
|
2658
|
+
* read-side half of the clear (see below),
|
|
2659
|
+
* 5. self-signature verifies against the bound `CadrePeer.PublicKey`,
|
|
2660
|
+
* 6. freshness — `updatedAt` is positive and within `opts.maxAgeMs` (default:
|
|
1609
2661
|
* no ceiling, since a push token is valid until it rotates).
|
|
1610
2662
|
*
|
|
1611
2663
|
* A peer that is not a current member, or whose token has no backing `CadrePeer`
|
|
@@ -1639,6 +2691,23 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1639
2691
|
return null;
|
|
1640
2692
|
}
|
|
1641
2693
|
|
|
2694
|
+
// Retired stamp: the read-side mitigation for the write-time race, mirroring what
|
|
2695
|
+
// listAuthorizedMembers does for CadrePeer. Clearing a token retires its StampId
|
|
2696
|
+
// into Revocation, but the schema's NotRevoked CHECK only sees LOCALLY visible
|
|
2697
|
+
// tombstones — a node that converged on a replayed insert before the tombstone can
|
|
2698
|
+
// hold both rows, so the reader must drop the resurrected one.
|
|
2699
|
+
//
|
|
2700
|
+
// NOTE: the freshness ceiling below defaults to INFINITE by design (a suspended
|
|
2701
|
+
// phone must stay push-reachable long after it published), so unlike a peer address
|
|
2702
|
+
// record a stale token never ages out. Stamp retirement is the ONLY thing that
|
|
2703
|
+
// retires a cleared token — do not weaken this check on the assumption that
|
|
2704
|
+
// staleness will eventually cover it.
|
|
2705
|
+
const revokedStamps = await this.controlDatabase.queryRevokedStamps('DeviceToken');
|
|
2706
|
+
if (revokedStamps.has(record.stampId)) {
|
|
2707
|
+
log('resolveDeviceToken: StampId for %s is retired (cleared token, resurrected row)', peerId);
|
|
2708
|
+
return null;
|
|
2709
|
+
}
|
|
2710
|
+
|
|
1642
2711
|
// Self-signature over (peerId, platform, token, updatedAt), verified against the
|
|
1643
2712
|
// CadrePeer.PublicKey bound to this peerId.
|
|
1644
2713
|
if (!verifyDeviceTokenSignature(record, peerRecord.publicKey)) {
|
|
@@ -1741,6 +2810,13 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1741
2810
|
strandId: strand.Id,
|
|
1742
2811
|
error: error instanceof Error ? error : new Error(String(error))
|
|
1743
2812
|
});
|
|
2813
|
+
// Rethrow after the event: the only caller is the StrandWatcher's
|
|
2814
|
+
// onStrandAdded callback, which uses the rejection to decide whether the
|
|
2815
|
+
// strand was really added. Swallowing it here would have the watcher mark
|
|
2816
|
+
// the strand as known-and-added, so no later poll would ever retry it.
|
|
2817
|
+
// The watcher catches and logs, so this never escapes as an unhandled
|
|
2818
|
+
// rejection.
|
|
2819
|
+
throw error;
|
|
1744
2820
|
}
|
|
1745
2821
|
}
|
|
1746
2822
|
|
|
@@ -1748,14 +2824,9 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1748
2824
|
log('Handling strand removed: %s', strandId);
|
|
1749
2825
|
|
|
1750
2826
|
try {
|
|
1751
|
-
//
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
// Remove sApp config
|
|
1755
|
-
this.sAppConfigs.delete(strandId);
|
|
1756
|
-
|
|
1757
|
-
await this.strandManager.stopStrand(strandId);
|
|
1758
|
-
this.emit('strand:stopped', { strandId });
|
|
2827
|
+
// Same local teardown the explicit stop performs — deliberately NOT stopStrand
|
|
2828
|
+
// itself, whose `_running` guard would throw on a poll that lands during shutdown.
|
|
2829
|
+
await this.detachStrand(strandId);
|
|
1759
2830
|
} catch (error) {
|
|
1760
2831
|
log('Error stopping strand %s: %o', strandId, error);
|
|
1761
2832
|
this.emit('strand:error', {
|
|
@@ -1825,8 +2896,19 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1825
2896
|
// Clear sApp configs
|
|
1826
2897
|
this.sAppConfigs.clear();
|
|
1827
2898
|
|
|
1828
|
-
//
|
|
2899
|
+
// Drop delegate-admission state: the grants are scoped to the session that
|
|
2900
|
+
// recorded them, so a stop()/start() cycle on this object must not keep
|
|
2901
|
+
// admitting delegates announced under the previous one, nor carry stale
|
|
2902
|
+
// announce timestamps that would suppress the next session's refresh.
|
|
2903
|
+
this.delegateAdmission.clear();
|
|
2904
|
+
this.delegateAnnounceAt.clear();
|
|
2905
|
+
|
|
2906
|
+
// Close control database (this also shuts down the collection factory).
|
|
2907
|
+
// Detach the membership hub first: nothing this teardown does should drive a
|
|
2908
|
+
// gate refresh, and a notification arriving after this point is a no-op.
|
|
1829
2909
|
if (this.controlDatabase) {
|
|
2910
|
+
this.controlDatabase.setMembershipChangeListener(null);
|
|
2911
|
+
this.controlDatabase.setGuardedDeleteListener(null);
|
|
1830
2912
|
await this.controlDatabase.close();
|
|
1831
2913
|
this.controlDatabase = null;
|
|
1832
2914
|
}
|
|
@@ -1919,7 +3001,13 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1919
3001
|
* (returns the live instance unchanged) as a backstop against double-resume.
|
|
1920
3002
|
*/
|
|
1921
3003
|
private async resumeStrandRuntime(strandId: string): Promise<void> {
|
|
1922
|
-
|
|
3004
|
+
// Re-derive the transport peerId (deterministic and cheap — a quiesced
|
|
3005
|
+
// instance has no `libp2pNode` to read it from) so the seed pass announces
|
|
3006
|
+
// the delegate before `resumeStrand` runs `libp2p.start()`.
|
|
3007
|
+
const delegatePeerId = this.identityKey
|
|
3008
|
+
? peerIdFromPrivateKey(await strandTransportKey(this.identityKey, strandId)).toString()
|
|
3009
|
+
: undefined;
|
|
3010
|
+
const seed = await this.resolveCohortSeed(strandId, delegatePeerId);
|
|
1923
3011
|
const mode = selectStrandMode(undefined, seed.hasOtherPeers);
|
|
1924
3012
|
await this.strandManager.resumeStrand(strandId, {
|
|
1925
3013
|
bootstrapNodes: seed.bootstrapNodes,
|
|
@@ -2059,6 +3147,14 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2059
3147
|
/**
|
|
2060
3148
|
* Add a strand with its sApp configuration.
|
|
2061
3149
|
* The hosting application must provide the sApp schema when creating a strand.
|
|
3150
|
+
*
|
|
3151
|
+
* A rejected call leaves nothing running but DOES leave the sApp config
|
|
3152
|
+
* registered, deliberately: both an explicit retry and the {@link StrandWatcher}'s
|
|
3153
|
+
* automatic relaunch need it. So once the strand's row is visible on the control
|
|
3154
|
+
* network (via {@link publishStrand} or another member), a failed launch keeps
|
|
3155
|
+
* being re-attempted in the background — each failure re-emitting `strand:error`
|
|
3156
|
+
* — until it succeeds. {@link detachStrand} (reached via {@link stopStrand}) is
|
|
3157
|
+
* what abandons a strand for good.
|
|
2062
3158
|
*/
|
|
2063
3159
|
async addStrand(config: StrandConfig): Promise<StrandInstance> {
|
|
2064
3160
|
if (!this._running) {
|
|
@@ -2094,7 +3190,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2094
3190
|
* {@link ed25519KeyPairFromLibp2p} also exposes as the node's owner keypair,
|
|
2095
3191
|
* so peer identity and owner key are one and the same. That key must be
|
|
2096
3192
|
* enrolled in `OwnerKey` (e.g. via {@link ControlDatabase.ensureOwnerKey}
|
|
2097
|
-
* at genesis) or the schema's `Strand.
|
|
3193
|
+
* at genesis) or the schema's `Strand.AuthorizedInsert` constraint rejects the write.
|
|
2098
3194
|
* Failing loudly here is intentional: a silently-unpublished strand would run
|
|
2099
3195
|
* as a local-only island that no peer could ever discover or join.
|
|
2100
3196
|
*
|
|
@@ -2102,34 +3198,111 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2102
3198
|
* {@link addStrand}).
|
|
2103
3199
|
* @param type - `'o'` for open (default) or `'c'` for closed.
|
|
2104
3200
|
* @param memberPrivateKey - Optional membership key for a closed strand.
|
|
2105
|
-
* @throws if the node is not started, exposes no owner signing key, or
|
|
2106
|
-
* control DB rejects the (unauthorized) insert.
|
|
3201
|
+
* @throws if the node is not started, exposes no owner signing key, the id is blank, or
|
|
3202
|
+
* the control DB rejects the (unauthorized) insert.
|
|
2107
3203
|
*/
|
|
2108
3204
|
async publishStrand(strandId: string, type: 'o' | 'c' = 'o', memberPrivateKey?: string): Promise<void> {
|
|
2109
|
-
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
const
|
|
2113
|
-
|
|
2114
|
-
|
|
2115
|
-
|
|
2116
|
-
|
|
2117
|
-
|
|
2118
|
-
|
|
3205
|
+
const signingKey = this.requireOwnerSigningKey(`publish strand ${strandId}`);
|
|
3206
|
+
// Trim/reject here so the id that lands matches the one unpublishStrand looks up: it
|
|
3207
|
+
// trims too, and an untrimmed row would be unreachable by the same string.
|
|
3208
|
+
const trimmed = requireNonBlank(strandId, 'strand id');
|
|
3209
|
+
await this.controlDatabase!.insertStrand(
|
|
3210
|
+
trimmed,
|
|
3211
|
+
type,
|
|
3212
|
+
signingKey.publicKeyB64,
|
|
3213
|
+
signMessageWith(signingKey.privateKeyB64),
|
|
3214
|
+
memberPrivateKey
|
|
3215
|
+
);
|
|
3216
|
+
log('Published strand %s (type %s) to control DB under owner %s', trimmed, type, signingKey.publicKeyB64);
|
|
3217
|
+
}
|
|
3218
|
+
|
|
3219
|
+
/**
|
|
3220
|
+
* Remove this party's `Strand` row from the shared control database — the owner-signed
|
|
3221
|
+
* inverse of {@link publishStrand}.
|
|
3222
|
+
*
|
|
3223
|
+
* Party-wide, unlike {@link stopStrand}, which only stops the strand on THIS node: every
|
|
3224
|
+
* cadre node watching the table sees the row vanish on its next poll (default 5 s) and
|
|
3225
|
+
* stops its own instance. This node converges immediately — the method forces a watcher
|
|
3226
|
+
* poll and stops any still-running local instance before resolving. Removing the row
|
|
3227
|
+
* removes OUR party's participation only: other parties in the strand keep their own
|
|
3228
|
+
* rows and the strand network carries on, so no cross-party sign-off is involved — this
|
|
3229
|
+
* is an owner-signed control-plane write like every other, not "destroy the network".
|
|
3230
|
+
*
|
|
3231
|
+
* Irreversible for a closed strand (`Type='c'`): the row carries `MemberPrivateKey`,
|
|
3232
|
+
* this party's membership secret for that network, and it is stored nowhere else.
|
|
3233
|
+
* With it gone the party can never again admit a member to that closed strand, and a
|
|
3234
|
+
* re-published row would carry a DIFFERENT key that does not match the membership
|
|
3235
|
+
* already written into the strand's RBAC layer. The strand id itself is NOT
|
|
3236
|
+
* blacklisted: a fresh owner-signed {@link publishStrand} re-seats it under a new
|
|
3237
|
+
* stamp — only the unsigned consent re-seat is permanently foreclosed (the removal's
|
|
3238
|
+
* `Revocation` tombstone names the id, which the consent branch of
|
|
3239
|
+
* `Strand.AuthorizedInsert` refuses ever after). Outstanding `FormationInvite` rows
|
|
3240
|
+
* bound to the removed strand are unredeemable while the row is absent (the formation
|
|
3241
|
+
* recorder resolves the strand as missing and rejects cleanly).
|
|
3242
|
+
*
|
|
3243
|
+
* Convergence caveats, same class as {@link enrollValidationKey}'s: a sibling that has
|
|
3244
|
+
* not yet synced keeps running the strand until its own watcher observes the missing
|
|
3245
|
+
* row; a sibling whose `strandFilter` never admitted this strand never observes the
|
|
3246
|
+
* removal AT ALL and keeps running its instance indefinitely — opting out of watching a
|
|
3247
|
+
* strand is also opting out of its party-wide removal, so such a node's only stop is its
|
|
3248
|
+
* own local {@link stopStrand}/`unpublishStrand` call; and a removal committed while
|
|
3249
|
+
* ALONE (0 control connections) deletes the row local-only. The accompanying
|
|
3250
|
+
* `Revocation` tombstone IS queued and re-issued on the next cohort-growth edge
|
|
3251
|
+
* ({@link noteGuardedDelete}/{@link drainPendingRevocations}), so the stamp retirement
|
|
3252
|
+
* — and with it the consent re-seat foreclosure — does propagate; but the physical row
|
|
3253
|
+
* deletion cannot be replayed, and `queryStrands` reads raw (no retired-stamp filter),
|
|
3254
|
+
* so siblings that already hold the row keep running the strand until the collection
|
|
3255
|
+
* itself converges (logged loudly; see the delete-while-alone durability note in
|
|
3256
|
+
* docs/architecture.md).
|
|
3257
|
+
*
|
|
3258
|
+
* A no-op (no throw, no tombstone) when the row is already absent — but a
|
|
3259
|
+
* locally-running instance of that id is still stopped.
|
|
3260
|
+
*
|
|
3261
|
+
* NOTE: control-plane only — the strand's local durable storage is retained. Stopping the
|
|
3262
|
+
* instance closes the `StrandDatabase` and its libp2p node but purges no blocks, so a
|
|
3263
|
+
* closed strand's content stays readable on disk to anyone with the data directory. If a
|
|
3264
|
+
* caller ever needs removal to mean "and erase the local copy", that is a separate purge
|
|
3265
|
+
* step, not a widening of this method.
|
|
3266
|
+
*
|
|
3267
|
+
* @param strandId - The `Strand` row id to remove (as passed to {@link publishStrand}).
|
|
3268
|
+
* @throws if the node is not started, exposes no owner signing key, the id is blank, or
|
|
3269
|
+
* the signer is not an enrolled owner (the schema's `Strand.AuthorizedDelete` rejects
|
|
3270
|
+
* the write and the row survives). A rejection does NOT imply the row survived: the
|
|
3271
|
+
* local stop runs after the control-plane delete has already committed, so a failure
|
|
3272
|
+
* there throws over a completed removal.
|
|
3273
|
+
*/
|
|
3274
|
+
async unpublishStrand(strandId: string): Promise<void> {
|
|
3275
|
+
const signingKey = this.requireOwnerSigningKey(`unpublish strand ${strandId}`);
|
|
3276
|
+
const trimmed = requireNonBlank(strandId, 'strand id');
|
|
3277
|
+
const removed = await this.controlDatabase!.deleteStrand(
|
|
3278
|
+
trimmed,
|
|
3279
|
+
signingKey.publicKeyB64,
|
|
3280
|
+
signMessageWith(signingKey.privateKeyB64)
|
|
3281
|
+
);
|
|
3282
|
+
// Gate on `removed`: the absent-row no-op wrote nothing, so warning about an
|
|
3283
|
+
// unreplicated deletion there would send an operator chasing a phantom.
|
|
3284
|
+
if (removed && this.committedAlone()) {
|
|
3285
|
+
log('unpublishStrand(%s) committed while ALONE (0 control connections): the deletion is ' +
|
|
3286
|
+
'local-only. Its Revocation tombstone is queued for re-issue on cohort growth, but the ' +
|
|
3287
|
+
'row deletion itself cannot be replayed, so other nodes may keep running the strand ' +
|
|
3288
|
+
'until the collection converges.', trimmed);
|
|
3289
|
+
}
|
|
3290
|
+
// Converge locally now rather than waiting up to a poll interval. The watcher fires
|
|
3291
|
+
// onStrandRemoved for a strand it tracked; the explicit stop below covers a node whose
|
|
3292
|
+
// strandFilter never admitted this strand (the watcher never knew it, so it will never
|
|
3293
|
+
// fire). StrandInstanceManager.stopStrand no-ops on an unknown id, so the two paths
|
|
3294
|
+
// cannot double-stop into an error.
|
|
3295
|
+
await this.strandWatcher?.forcePoll();
|
|
3296
|
+
if (this.strandManager.getInstance(trimmed)) {
|
|
3297
|
+
await this.stopStrand(trimmed);
|
|
2119
3298
|
}
|
|
2120
|
-
|
|
2121
|
-
// buildAuthorizationMessage); ed25519-sign them directly (no pre-hash) with
|
|
2122
|
-
// the owner private key, returning a base64url signature.
|
|
2123
|
-
const signMessage = (message: Uint8Array): string =>
|
|
2124
|
-
sign(message, signingKey.privateKeyB64, 'ed25519', 'bytes', 'base64url', 'base64url') as string;
|
|
2125
|
-
await this.controlDatabase.insertStrand(strandId, type, signingKey.publicKeyB64, signMessage, memberPrivateKey);
|
|
2126
|
-
log('Published strand %s (type %s) to control DB under owner %s', strandId, type, signingKey.publicKeyB64);
|
|
3299
|
+
log('Unpublished strand %s from control DB under owner %s', trimmed, signingKey.publicKeyB64);
|
|
2127
3300
|
}
|
|
2128
3301
|
|
|
2129
3302
|
/**
|
|
2130
3303
|
* Publish an owner-signed `FormationInvite` (open-invitation token) to the
|
|
2131
3304
|
* shared control database, so a later {@link formStrand} redemption can be
|
|
2132
|
-
* validated against it (the consent branch of `Strand.
|
|
3305
|
+
* validated against it (the consent branch of `Strand.AuthorizedInsert`).
|
|
2133
3306
|
*
|
|
2134
3307
|
* Counterpart to {@link createOpenInvitation}, which only mints the
|
|
2135
3308
|
* out-of-band {@link OpenInvitation} envelope: persisting the matching
|
|
@@ -2153,21 +3326,124 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2153
3326
|
sAppId: string,
|
|
2154
3327
|
options: { expiresAtMs?: number; totalUses?: number; validationUrl?: string; strandId?: string } = {}
|
|
2155
3328
|
): Promise<void> {
|
|
3329
|
+
const signingKey = this.requireOwnerSigningKey(`publish formation invite ${token}`);
|
|
3330
|
+
await this.controlDatabase!.insertFormationInvite(
|
|
3331
|
+
token,
|
|
3332
|
+
sAppId,
|
|
3333
|
+
signingKey.publicKeyB64,
|
|
3334
|
+
signMessageWith(signingKey.privateKeyB64),
|
|
3335
|
+
options
|
|
3336
|
+
);
|
|
3337
|
+
// A host may publish an invite whose token was minted elsewhere; registering
|
|
3338
|
+
// it locally opens the connection gate's formation exemption immediately
|
|
3339
|
+
// rather than waiting for the durable row to become readable.
|
|
3340
|
+
this.strandSolicitationService?.registerMintedInvitation(
|
|
3341
|
+
token,
|
|
3342
|
+
options.expiresAtMs ?? Number.POSITIVE_INFINITY
|
|
3343
|
+
);
|
|
3344
|
+
log('Published formation invite %s (sApp %s) under owner %s', token, sAppId, signingKey.publicKeyB64);
|
|
3345
|
+
}
|
|
3346
|
+
|
|
3347
|
+
/**
|
|
3348
|
+
* Enroll an approver public key allowed to sign off on `ValidationUrl` redemptions.
|
|
3349
|
+
*
|
|
3350
|
+
* A `FormationInvite` carrying a `ValidationUrl` is only redeemable when the approval
|
|
3351
|
+
* that comes back from that URL is signed by a key present in `CadreControl.ValidationKey`
|
|
3352
|
+
* — this is how a party says which outside approver it trusts. Without an enrollment,
|
|
3353
|
+
* every such invitation is unredeemable.
|
|
3354
|
+
*
|
|
3355
|
+
* Signs with the same self-owner key as {@link publishStrand} (the ed25519 key behind
|
|
3356
|
+
* this node's PeerId, which must be an enrolled `OwnerKey`). Throws loudly if the node
|
|
3357
|
+
* isn't started, exposes no signing key, or the key is blank — a blank key would reach
|
|
3358
|
+
* the database and fail the signature CHECK as an opaque constraint error.
|
|
3359
|
+
*
|
|
3360
|
+
* Enrolling this node's OWN owner key is permitted and harmless: the domain/action tags
|
|
3361
|
+
* baked into each authorization digest mean an owner-key signature can never satisfy the
|
|
3362
|
+
* approval rule and an approval can never satisfy an owner rule. No guard is needed.
|
|
3363
|
+
*
|
|
3364
|
+
* NOTE: enrollment is replicated control state, so a key enrolled here is visible to a
|
|
3365
|
+
* sibling cadre node only once control replication converges. A redemption that arrives
|
|
3366
|
+
* at a node which has not caught up is refused as not-enrolled — the same convergence
|
|
3367
|
+
* gap the schema records on `Strand.StampId`. Enroll before circulating the invitations
|
|
3368
|
+
* that depend on the key.
|
|
3369
|
+
*
|
|
3370
|
+
* @param key - Approver public key to enroll (base64url ed25519 public key).
|
|
3371
|
+
*/
|
|
3372
|
+
async enrollValidationKey(key: string): Promise<void> {
|
|
3373
|
+
const signingKey = this.requireOwnerSigningKey('enroll a validation key');
|
|
3374
|
+
const trimmed = requireEd25519PublicKeyB64(key, 'validation key');
|
|
3375
|
+
await this.controlDatabase!.insertValidationKey(
|
|
3376
|
+
trimmed,
|
|
3377
|
+
signingKey.publicKeyB64,
|
|
3378
|
+
signMessageWith(signingKey.privateKeyB64)
|
|
3379
|
+
);
|
|
3380
|
+
log('Enrolled validation key %s under owner %s', trimmed, signingKey.publicKeyB64);
|
|
3381
|
+
}
|
|
3382
|
+
|
|
3383
|
+
/**
|
|
3384
|
+
* Remove an approver public key.
|
|
3385
|
+
*
|
|
3386
|
+
* Narrows who may approve FUTURE redemptions ONLY. The schema's approval CHECKs run at
|
|
3387
|
+
* write time, so a join that was already approved by this key stays valid — removal is
|
|
3388
|
+
* not retroactive and does not re-examine committed rows.
|
|
3389
|
+
*
|
|
3390
|
+
* Rotation is therefore add-then-remove, in that order: removing the only enrolled key
|
|
3391
|
+
* while `ValidationUrl` invitations are outstanding makes every one of them unredeemable
|
|
3392
|
+
* until a new key is enrolled.
|
|
3393
|
+
*
|
|
3394
|
+
* Removing a key that is not enrolled is a silent no-op (no throw, no `Revocation`
|
|
3395
|
+
* tombstone) — see {@link ControlDatabase.deleteValidationKey}.
|
|
3396
|
+
*
|
|
3397
|
+
* @param key - Approver public key to remove (base64url ed25519 public key).
|
|
3398
|
+
*/
|
|
3399
|
+
async removeValidationKey(key: string): Promise<void> {
|
|
3400
|
+
const signingKey = this.requireOwnerSigningKey('remove a validation key');
|
|
3401
|
+
const trimmed = requireNonBlank(key, 'validation key');
|
|
3402
|
+
await this.controlDatabase!.deleteValidationKey(
|
|
3403
|
+
trimmed,
|
|
3404
|
+
signingKey.publicKeyB64,
|
|
3405
|
+
signMessageWith(signingKey.privateKeyB64)
|
|
3406
|
+
);
|
|
3407
|
+
log('Removed validation key %s under owner %s', trimmed, signingKey.publicKeyB64);
|
|
3408
|
+
}
|
|
3409
|
+
|
|
3410
|
+
/**
|
|
3411
|
+
* The approver keys currently enrolled, sorted. Read-only, so unlike
|
|
3412
|
+
* {@link enrollValidationKey} / {@link removeValidationKey} it needs no owner signing
|
|
3413
|
+
* key — only a started node.
|
|
3414
|
+
*/
|
|
3415
|
+
async listValidationKeys(): Promise<string[]> {
|
|
2156
3416
|
if (!this._running || !this.controlDatabase) {
|
|
2157
|
-
throw new Error('CadreNode must be started before
|
|
3417
|
+
throw new Error('CadreNode must be started before listing validation keys');
|
|
3418
|
+
}
|
|
3419
|
+
return await this.controlDatabase.queryValidationKeys();
|
|
3420
|
+
}
|
|
3421
|
+
|
|
3422
|
+
/**
|
|
3423
|
+
* Resolve the owner keypair every owner-signed control write needs — {@link publishStrand},
|
|
3424
|
+
* {@link unpublishStrand}, {@link publishFormationInvite}, {@link enrollValidationKey},
|
|
3425
|
+
* {@link removeValidationKey}
|
|
3426
|
+
* — failing loudly in one two-part shape (not started / no signing key, naming owner
|
|
3427
|
+
* genesis as the fix). Narrows `controlDatabase` for the caller: a non-null return means
|
|
3428
|
+
* `this.controlDatabase` is non-null too.
|
|
3429
|
+
*
|
|
3430
|
+
* @param action - Infinitive phrase naming the attempted write, e.g. `'enroll a
|
|
3431
|
+
* validation key'`; interpolated into both messages.
|
|
3432
|
+
*/
|
|
3433
|
+
private requireOwnerSigningKey(action: string): { privateKeyB64: string; publicKeyB64: string } {
|
|
3434
|
+
if (!this._running || !this.controlDatabase) {
|
|
3435
|
+
throw new Error(`CadreNode must be started before attempting to ${action}`);
|
|
2158
3436
|
}
|
|
2159
3437
|
const signingKey = this.getSelfSigningKey();
|
|
2160
3438
|
if (!signingKey) {
|
|
2161
3439
|
throw new Error(
|
|
2162
|
-
`Cannot
|
|
3440
|
+
`Cannot ${action}: no owner signing key available ` +
|
|
2163
3441
|
'(node identity is unavailable or does not match the node PeerId). Run owner ' +
|
|
2164
|
-
'genesis (ensureOwnerKey + initializeSeedBootstrap) before
|
|
3442
|
+
'genesis (ensureOwnerKey + initializeSeedBootstrap) before writing owner-signed ' +
|
|
3443
|
+
'control state.'
|
|
2165
3444
|
);
|
|
2166
3445
|
}
|
|
2167
|
-
|
|
2168
|
-
sign(message, signingKey.privateKeyB64, 'ed25519', 'bytes', 'base64url', 'base64url') as string;
|
|
2169
|
-
await this.controlDatabase.insertFormationInvite(token, sAppId, signingKey.publicKeyB64, signMessage, options);
|
|
2170
|
-
log('Published formation invite %s (sApp %s) under owner %s', token, sAppId, signingKey.publicKeyB64);
|
|
3446
|
+
return signingKey;
|
|
2171
3447
|
}
|
|
2172
3448
|
|
|
2173
3449
|
/**
|
|
@@ -2175,6 +3451,15 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2175
3451
|
* control-discovered (`handleStrandAdded`) entry points. Resolves the cohort
|
|
2176
3452
|
* seed, selects the mode, starts the strand, and registers it with the
|
|
2177
3453
|
* hibernation manager before emitting `strand:started`.
|
|
3454
|
+
*
|
|
3455
|
+
* Idempotent no-op when the strand manager already tracks `strand.Id` — the
|
|
3456
|
+
* ordinary founding sequence is `addStrand` (starts it locally) followed by
|
|
3457
|
+
* `publishStrand` (makes it visible), so this node's own `StrandWatcher`
|
|
3458
|
+
* rediscovers a row it already started and calls this again via
|
|
3459
|
+
* `handleStrandAdded`. Without the guard that re-entry would resolve a fresh
|
|
3460
|
+
* (already-connected) cohort seed and re-emit `strand:started` for an
|
|
3461
|
+
* instance that never stopped. The guard runs before the cohort-seed RPC
|
|
3462
|
+
* fan-out, so a rediscovery costs nothing beyond the map lookup.
|
|
2178
3463
|
*/
|
|
2179
3464
|
private async launchStrand(
|
|
2180
3465
|
strand: StrandRow,
|
|
@@ -2182,7 +3467,37 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2182
3467
|
explicitMode?: StrandMode,
|
|
2183
3468
|
founder?: boolean
|
|
2184
3469
|
): Promise<StrandInstance> {
|
|
2185
|
-
const
|
|
3470
|
+
const existing = this.strandManager.getInstance(strand.Id);
|
|
3471
|
+
if (existing) {
|
|
3472
|
+
log('launchStrand: strand %s already tracked locally — skipping re-launch', strand.Id);
|
|
3473
|
+
return existing;
|
|
3474
|
+
}
|
|
3475
|
+
|
|
3476
|
+
// Each strand node gets its own transport identity, derived from the cadre
|
|
3477
|
+
// identity key + strandId (see strand-transport-key.ts). Sharing the
|
|
3478
|
+
// control node's key here gave every node one peerId, which collides at a
|
|
3479
|
+
// shared circuit relay (github.com/gotchoices/sereus/issues/1). The
|
|
3480
|
+
// retained launch config carries this derived key, so hibernate → wake
|
|
3481
|
+
// (resumeStrand) reuses the same peerId.
|
|
3482
|
+
//
|
|
3483
|
+
// NOTE: derivation requires an Ed25519 identity key, so a node configured
|
|
3484
|
+
// with some other key type now fails strand launch outright (surfaced as
|
|
3485
|
+
// `strand:error` / a rejected addStrand) where it previously started the
|
|
3486
|
+
// strand on that key. Ed25519 is already required for every control-DB
|
|
3487
|
+
// signing path, so nothing reachable today hits this; if a non-Ed25519
|
|
3488
|
+
// identity is ever supported, fall back to `undefined` here — libp2p then
|
|
3489
|
+
// generates a random per-strand key, which still avoids the collision but
|
|
3490
|
+
// gives up peerId stability across restarts.
|
|
3491
|
+
const transportKey = this.identityKey
|
|
3492
|
+
? await strandTransportKey(this.identityKey, strand.Id)
|
|
3493
|
+
: undefined;
|
|
3494
|
+
|
|
3495
|
+
// Derived BEFORE seed resolution so the seed pass doubles as the delegate
|
|
3496
|
+
// announcement and every grant is recorded before `startStrand` runs
|
|
3497
|
+
// `libp2p.start()` (the responder records the grant before replying; the
|
|
3498
|
+
// client awaits the replies).
|
|
3499
|
+
const delegatePeerId = transportKey ? peerIdFromPrivateKey(transportKey).toString() : undefined;
|
|
3500
|
+
const seed = await this.resolveCohortSeed(strand.Id, delegatePeerId);
|
|
2186
3501
|
const mode = selectStrandMode(explicitMode, seed.hasOtherPeers);
|
|
2187
3502
|
|
|
2188
3503
|
const instance = await this.strandManager.startStrand({
|
|
@@ -2192,10 +3507,12 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2192
3507
|
network: this.config.network,
|
|
2193
3508
|
profile: this.config.profile,
|
|
2194
3509
|
defaultLatencyHint: this.config.hibernation?.defaultLatencyHint ?? 'interactive',
|
|
2195
|
-
privateKey:
|
|
3510
|
+
privateKey: transportKey,
|
|
2196
3511
|
bootstrapNodes: seed.bootstrapNodes,
|
|
2197
3512
|
mode,
|
|
2198
3513
|
requireSignedSchemas: this.config.requireSignedSchemas,
|
|
3514
|
+
clusterSize: this.config.strandClusterSize,
|
|
3515
|
+
backfill: this.config.strandBackfill,
|
|
2199
3516
|
founder
|
|
2200
3517
|
});
|
|
2201
3518
|
|
|
@@ -2217,8 +3534,19 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2217
3534
|
* empty (`[]`) — mode still follows membership, and the empty seed self-heals
|
|
2218
3535
|
* on the next resume / check-in pass. Returns an empty seed when the control DB
|
|
2219
3536
|
* or node is absent (not yet started / torn down).
|
|
3537
|
+
*
|
|
3538
|
+
* When `delegatePeerId` is given (a strand launch/resume is imminent), the
|
|
3539
|
+
* pass doubles as the delegate ANNOUNCEMENT: our circuit relays are merged
|
|
3540
|
+
* into the RPC targets and every request carries the delegate peerId, so each
|
|
3541
|
+
* receiver records an admission grant BEFORE `libp2p.start()` (re-)dials the
|
|
3542
|
+
* relay reservation — the gate denial there is fatal, not degraded. A
|
|
3543
|
+
* party-member relay answers the RPC (its control node admits us as a
|
|
3544
|
+
* member); a dedicated ops/ relay does not speak the protocol and the
|
|
3545
|
+
* per-peer failure folds to `[]` — harmless, it has no membership gate and
|
|
3546
|
+
* needs no grant. The relay's direct addr rides along as the dial fallback
|
|
3547
|
+
* for a relay we are not yet connected to.
|
|
2220
3548
|
*/
|
|
2221
|
-
private async resolveCohortSeed(strandId: string): Promise<CohortSeed> {
|
|
3549
|
+
private async resolveCohortSeed(strandId: string, delegatePeerId?: string): Promise<CohortSeed> {
|
|
2222
3550
|
if (!this.controlDatabase || !this.controlNode) {
|
|
2223
3551
|
return { bootstrapNodes: [], hasOtherPeers: false };
|
|
2224
3552
|
}
|
|
@@ -2229,13 +3557,118 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2229
3557
|
// RPC only siblings we already have a control connection to — they can answer
|
|
2230
3558
|
// now, and `dialProtocol` by peerId reuses the open connection.
|
|
2231
3559
|
const connected = new Set(this.controlNode.getConnections().map((c) => c.remotePeer.toString()));
|
|
2232
|
-
const targets = otherPeerIds
|
|
3560
|
+
const targets: StrandAddrPeer[] = otherPeerIds
|
|
3561
|
+
.filter((id) => connected.has(id))
|
|
3562
|
+
.map((peerId) => ({ peerId }));
|
|
3563
|
+
const relays = delegatePeerId === undefined ? [] : this.circuitRelayTargets();
|
|
3564
|
+
for (const relay of relays) {
|
|
3565
|
+
if (!targets.some((t) => t.peerId === relay.relayPeerId)) {
|
|
3566
|
+
targets.push(relayStrandAddrPeer(relay));
|
|
3567
|
+
}
|
|
3568
|
+
}
|
|
2233
3569
|
const bootstrapNodes = targets.length
|
|
2234
|
-
? await collectStrandAddrs(this.controlNode, targets
|
|
3570
|
+
? await collectStrandAddrs(this.controlNode, targets, strandId, { delegatePeerId })
|
|
2235
3571
|
: [];
|
|
3572
|
+
// Throttle state for the RELAY targets only — refreshDelegateGrants never
|
|
3573
|
+
// looks up a sibling key, so recording one would only be dead weight.
|
|
3574
|
+
this.recordDelegateAnnounces(relays.map((r) => r.relayPeerId), strandId);
|
|
2236
3575
|
return { bootstrapNodes, hasOtherPeers };
|
|
2237
3576
|
}
|
|
2238
3577
|
|
|
3578
|
+
/**
|
|
3579
|
+
* The circuit relays this node's strand nodes would reserve through: the
|
|
3580
|
+
* union of the RESOLVED listen addrs' circuits — a hand-written
|
|
3581
|
+
* `network.listenAddrs` circuit entry or a `network.relayAddrs` entry, which
|
|
3582
|
+
* the strand node inherits verbatim either way — and the control node's own
|
|
3583
|
+
* live `/p2p-circuit` multiaddrs (a reservation this node discovered rather
|
|
3584
|
+
* than configured).
|
|
3585
|
+
*
|
|
3586
|
+
* NOTE: a relay the STRAND node discovers on its own (autorelay — in neither
|
|
3587
|
+
* source) gets no announcement, and a membership-gated one will deny it; fine
|
|
3588
|
+
* now, every realistic topology feeds one of the two sources.
|
|
3589
|
+
*/
|
|
3590
|
+
private circuitRelayTargets(): CircuitRelayTarget[] {
|
|
3591
|
+
return extractCircuitRelayTargets([
|
|
3592
|
+
...(resolveListenAddrs(this.config.network) ?? []),
|
|
3593
|
+
...(this.controlNode?.getMultiaddrs().map(String) ?? [])
|
|
3594
|
+
]);
|
|
3595
|
+
}
|
|
3596
|
+
|
|
3597
|
+
/**
|
|
3598
|
+
* Record the announce timestamps {@link refreshDelegateGrants} throttles on,
|
|
3599
|
+
* for every relay a delegate-carrying announce pass dialed.
|
|
3600
|
+
*
|
|
3601
|
+
* Recorded OPTIMISTICALLY at announce time: `collectStrandAddrs` folds
|
|
3602
|
+
* per-peer failure to `[]` and reports no per-peer success, and threading
|
|
3603
|
+
* success out would change its API for little gain. A failed INITIAL announce
|
|
3604
|
+
* is fatal-at-start anyway (the relay denies the reservation,
|
|
3605
|
+
* `libp2p.start()` throws, and wake/check-in re-resume retries with a fresh
|
|
3606
|
+
* announce); a failed REFRESH retries within `DELEGATE_GRANT_TTL_MS / 2`
|
|
3607
|
+
* (15 min), still inside the 30 min TTL.
|
|
3608
|
+
*/
|
|
3609
|
+
private recordDelegateAnnounces(relayPeerIds: readonly string[], strandId: string, now = Date.now()): void {
|
|
3610
|
+
for (const relayPeerId of relayPeerIds) {
|
|
3611
|
+
this.delegateAnnounceAt.set(peerStrandKey(relayPeerId, strandId), now);
|
|
3612
|
+
}
|
|
3613
|
+
}
|
|
3614
|
+
|
|
3615
|
+
/**
|
|
3616
|
+
* Re-announce every running strand's delegate peerId to this node's circuit
|
|
3617
|
+
* relays, throttled to once per `DELEGATE_GRANT_TTL_MS / 2` per
|
|
3618
|
+
* (relay, strand). A grant must outlive the reservation: a dropped relay
|
|
3619
|
+
* connection makes the strand's circuit-relay transport re-dial and face the
|
|
3620
|
+
* connection gate again, so the relay must still hold a live grant then.
|
|
3621
|
+
* RELAY targets only — siblings get their announce on every launch/resume
|
|
3622
|
+
* seed pass, and a grant only matters where a reservation can be re-dialed.
|
|
3623
|
+
*
|
|
3624
|
+
* Strands announce CONCURRENTLY: this runs ahead of the reconcile pass's
|
|
3625
|
+
* sibling enumeration, and one unreachable relay costs a dial timeout per
|
|
3626
|
+
* target, which must not stack up per strand.
|
|
3627
|
+
*/
|
|
3628
|
+
private async refreshDelegateGrants(now = Date.now()): Promise<void> {
|
|
3629
|
+
if (!this.controlNode) {
|
|
3630
|
+
return;
|
|
3631
|
+
}
|
|
3632
|
+
// A running strand's delegate peerId is simply its live node's peerId — no
|
|
3633
|
+
// re-derivation here.
|
|
3634
|
+
const running = new Map<string, string>();
|
|
3635
|
+
for (const [strandId, instance] of this.strandManager.getInstances()) {
|
|
3636
|
+
if (instance.libp2pNode) {
|
|
3637
|
+
running.set(strandId, instance.libp2pNode.peerId.toString());
|
|
3638
|
+
}
|
|
3639
|
+
}
|
|
3640
|
+
pruneStoppedStrandAnnounces(this.delegateAnnounceAt, new Set(running.keys()));
|
|
3641
|
+
if (running.size === 0) {
|
|
3642
|
+
return;
|
|
3643
|
+
}
|
|
3644
|
+
const relays = this.circuitRelayTargets();
|
|
3645
|
+
if (relays.length === 0) {
|
|
3646
|
+
return;
|
|
3647
|
+
}
|
|
3648
|
+
await Promise.all([...running].map(
|
|
3649
|
+
([strandId, delegatePeerId]) => this.announceDelegateToDueRelays(strandId, delegatePeerId, relays, now)
|
|
3650
|
+
));
|
|
3651
|
+
}
|
|
3652
|
+
|
|
3653
|
+
/** One strand's share of {@link refreshDelegateGrants}: announce to the relays whose grant is due. */
|
|
3654
|
+
private async announceDelegateToDueRelays(
|
|
3655
|
+
strandId: string,
|
|
3656
|
+
delegatePeerId: string,
|
|
3657
|
+
relays: readonly CircuitRelayTarget[],
|
|
3658
|
+
now: number
|
|
3659
|
+
): Promise<void> {
|
|
3660
|
+
const controlNode = this.controlNode;
|
|
3661
|
+
if (!controlNode) {
|
|
3662
|
+
return;
|
|
3663
|
+
}
|
|
3664
|
+
const due = dueRelayAnnounces(this.delegateAnnounceAt, relays, strandId, now);
|
|
3665
|
+
if (due.length === 0) {
|
|
3666
|
+
return;
|
|
3667
|
+
}
|
|
3668
|
+
await collectStrandAddrs(controlNode, due.map(relayStrandAddrPeer), strandId, { delegatePeerId });
|
|
3669
|
+
this.recordDelegateAnnounces(due.map((relay) => relay.relayPeerId), strandId, now);
|
|
3670
|
+
}
|
|
3671
|
+
|
|
2239
3672
|
/**
|
|
2240
3673
|
* The local strand instance's dialable strand-network multiaddrs for the
|
|
2241
3674
|
* strand-addr RPC, ordered signaling-first (reusing the control node's
|
|
@@ -2254,19 +3687,39 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2254
3687
|
}
|
|
2255
3688
|
|
|
2256
3689
|
/**
|
|
2257
|
-
*
|
|
3690
|
+
* Stop a strand on THIS node only: untrack it from hibernation, drop its sApp config,
|
|
3691
|
+
* stop the local instance, and emit `strand:stopped`. The shared `Strand` row is left
|
|
3692
|
+
* intact, so on the next node restart (or watcher poll) the strand is rediscovered and
|
|
3693
|
+
* surfaces as `strand:discovered` again. Party-wide removal is {@link unpublishStrand}.
|
|
2258
3694
|
*/
|
|
2259
|
-
async
|
|
3695
|
+
async stopStrand(strandId: string): Promise<void> {
|
|
2260
3696
|
if (!this._running) {
|
|
2261
3697
|
throw new Error('CadreNode not running');
|
|
2262
3698
|
}
|
|
2263
3699
|
|
|
2264
|
-
|
|
2265
|
-
|
|
3700
|
+
await this.detachStrand(strandId);
|
|
3701
|
+
}
|
|
2266
3702
|
|
|
2267
|
-
|
|
3703
|
+
/**
|
|
3704
|
+
* Local teardown for one strand, shared by the caller-driven {@link stopStrand} and the
|
|
3705
|
+
* watcher-driven `handleStrandRemoved`: untrack hibernation, drop the sApp config, stop
|
|
3706
|
+
* the instance, emit `strand:stopped`. Touches no control-plane row — which side of the
|
|
3707
|
+
* removal the node is on is the caller's concern, not this method's.
|
|
3708
|
+
*
|
|
3709
|
+
* The stop + emit are skipped when the strand manager holds no instance for `strandId` —
|
|
3710
|
+
* e.g. a party owner that published a strand's row but never ran it locally, or an
|
|
3711
|
+
* explicit {@link stopStrand} for an id this node never started. `hibernationManager`
|
|
3712
|
+
* untrack and the `sAppConfigs` delete stay unconditional (both are no-ops when there is
|
|
3713
|
+
* nothing to remove), so a launch that failed before an instance was ever tracked still
|
|
3714
|
+
* gets its stray sApp config cleared.
|
|
3715
|
+
*/
|
|
3716
|
+
private async detachStrand(strandId: string): Promise<void> {
|
|
3717
|
+
this.hibernationManager.untrackStrand(strandId);
|
|
2268
3718
|
this.sAppConfigs.delete(strandId);
|
|
2269
|
-
|
|
3719
|
+
if (!this.strandManager.hasStrand(strandId)) {
|
|
3720
|
+
log('detachStrand: strand %s not tracked locally — no-op (no stop, no strand:stopped)', strandId);
|
|
3721
|
+
return;
|
|
3722
|
+
}
|
|
2270
3723
|
await this.strandManager.stopStrand(strandId);
|
|
2271
3724
|
this.emit('strand:stopped', { strandId });
|
|
2272
3725
|
}
|
|
@@ -2460,6 +3913,76 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2460
3913
|
return this.controlNode;
|
|
2461
3914
|
}
|
|
2462
3915
|
|
|
3916
|
+
/**
|
|
3917
|
+
* Start keeping a relay reservation: dial the given relay(s) from the control
|
|
3918
|
+
* node, ask the first one that answers for a reservation slot, wait until the
|
|
3919
|
+
* resulting `/p2p-circuit` address makes this node dialable — and keep a
|
|
3920
|
+
* supervisor running that re-drives whenever the reservation is later lost.
|
|
3921
|
+
*
|
|
3922
|
+
* For nodes that listen on the bare `/p2p-circuit` SEARCH addr (a browser tab)
|
|
3923
|
+
* rather than a configured `network.relayAddrs` circuit listener — see
|
|
3924
|
+
* `relay-reservation.ts` for why the two are alternatives and why configuring
|
|
3925
|
+
* both is a regression. Opt-in: nothing calls this for the CLI/host nodes.
|
|
3926
|
+
*
|
|
3927
|
+
* Resolves once the FIRST attempt has settled, exactly as it did when it drove
|
|
3928
|
+
* once; the retries continue in the background until {@link stop} or the next
|
|
3929
|
+
* `reserveRelays` call. Fail-soft — never throws. An unreachable relay, a
|
|
3930
|
+
* missing control node, or a timeout all resolve to a non-`reserved` status, so
|
|
3931
|
+
* a caller can await this during startup without a dead relay aborting the node.
|
|
3932
|
+
*
|
|
3933
|
+
* Passing an empty list stops the supervisor and resets the posture to `none`
|
|
3934
|
+
* without dialing or waiting.
|
|
3935
|
+
*/
|
|
3936
|
+
async reserveRelays(
|
|
3937
|
+
addrs: string[],
|
|
3938
|
+
opts?: RelayReservationSupervisorOptions
|
|
3939
|
+
): Promise<RelayReservationState> {
|
|
3940
|
+
// Stop first, unconditionally: a second call with a different list must not
|
|
3941
|
+
// leave two loops fighting over one pending reservation slot.
|
|
3942
|
+
this.relayReserveSupervisor?.stop();
|
|
3943
|
+
this.relayReserveSupervisor = null;
|
|
3944
|
+
this.relayReserveAddrs = [...addrs];
|
|
3945
|
+
this.relayReserveError = null;
|
|
3946
|
+
if (addrs.length === 0) {
|
|
3947
|
+
return this.getRelayReservationState();
|
|
3948
|
+
}
|
|
3949
|
+
if (!this.controlNode) {
|
|
3950
|
+
// No node to supervise, so this is the one failure the node records itself.
|
|
3951
|
+
this.relayReserveError = 'control node unavailable';
|
|
3952
|
+
return this.getRelayReservationState();
|
|
3953
|
+
}
|
|
3954
|
+
const supervisor = superviseRelayReservation(this.controlNode, addrs, opts);
|
|
3955
|
+
this.relayReserveSupervisor = supervisor;
|
|
3956
|
+
await supervisor.firstAttempt;
|
|
3957
|
+
// Read through `this`, not `supervisor`: an overlapping call may have replaced
|
|
3958
|
+
// it while the first attempt was in flight, and the newest list is the truth.
|
|
3959
|
+
return this.getRelayReservationState();
|
|
3960
|
+
}
|
|
3961
|
+
|
|
3962
|
+
/**
|
|
3963
|
+
* The node's CURRENT relay-reservation posture, recomputed from the control
|
|
3964
|
+
* node's live multiaddrs on every call.
|
|
3965
|
+
*
|
|
3966
|
+
* Deliberately not memoised: a reservation can be lost after
|
|
3967
|
+
* {@link reserveRelays} succeeds (the relay restarts, the connection drops) and
|
|
3968
|
+
* a cached `reserved` would let a caller mint invitations carrying circuit
|
|
3969
|
+
* addresses that no longer route.
|
|
3970
|
+
*
|
|
3971
|
+
* A lost reservation now recovers on its own — the supervisor {@link reserveRelays}
|
|
3972
|
+
* starts re-drives on a backoff, reporting `retrying` meanwhile. `error` here
|
|
3973
|
+
* means nothing is going to try again: no supervisor, or no control node.
|
|
3974
|
+
*/
|
|
3975
|
+
getRelayReservationState(): RelayReservationState {
|
|
3976
|
+
const supervisor = this.relayReserveSupervisor;
|
|
3977
|
+
return resolveRelayReservationState(
|
|
3978
|
+
this.controlNode,
|
|
3979
|
+
this.relayReserveAddrs,
|
|
3980
|
+
supervisor ? supervisor.lastError : this.relayReserveError,
|
|
3981
|
+
supervisor?.driving ?? false,
|
|
3982
|
+
supervisor?.retryAtMs ?? null
|
|
3983
|
+
);
|
|
3984
|
+
}
|
|
3985
|
+
|
|
2463
3986
|
/**
|
|
2464
3987
|
* Get the control database (for advanced queries)
|
|
2465
3988
|
*/
|
|
@@ -2467,6 +3990,22 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2467
3990
|
return this.controlDatabase;
|
|
2468
3991
|
}
|
|
2469
3992
|
|
|
3993
|
+
/**
|
|
3994
|
+
* Open control-network connections right now. A lower-bound proxy for replication
|
|
3995
|
+
* reach: 0 connections ⇒ a control write commits local-only. The private
|
|
3996
|
+
* `committedAlone` write-while-alone seam is defined in terms of this.
|
|
3997
|
+
*
|
|
3998
|
+
* It approximates the precise signal (the block's cluster size), and a caller that
|
|
3999
|
+
* samples it AFTER a write samples a slightly wider window than `committedAlone`
|
|
4000
|
+
* does inside the write itself. It exists so an embedder can warn an owner before a
|
|
4001
|
+
* control write that this machine currently sees none of its siblings, and report
|
|
4002
|
+
* after one that the write may not have travelled — a warning shown unconditionally
|
|
4003
|
+
* is a warning people learn to ignore.
|
|
4004
|
+
*/
|
|
4005
|
+
getControlConnectionCount(): number {
|
|
4006
|
+
return this.controlNode?.getConnections().length ?? 0;
|
|
4007
|
+
}
|
|
4008
|
+
|
|
2470
4009
|
/**
|
|
2471
4010
|
* Force a poll of the strand watcher (for testing)
|
|
2472
4011
|
*/
|
|
@@ -2496,18 +4035,38 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2496
4035
|
throw new Error('CadreNode must be started before initializing seed bootstrap');
|
|
2497
4036
|
}
|
|
2498
4037
|
|
|
4038
|
+
// A node wiring seed-bootstrap with an owner PRIVATE key is declaring that
|
|
4039
|
+
// key an authority of its own party (founder genesis / self-signing owner)
|
|
4040
|
+
// — anchor it in the node-local store. The in-memory set updates
|
|
4041
|
+
// synchronously (see TrustedOwnerStore.trust); only the file persist is
|
|
4042
|
+
// fire-and-forget, logged on failure.
|
|
4043
|
+
// NOTE: non-founder members that also wire seed-bootstrap with their own
|
|
4044
|
+
// derived key (e.g. the phone joiner path in runOwnerGenesis, which needs
|
|
4045
|
+
// it to self-publish its CadrePeer row) self-anchor a key that is not a
|
|
4046
|
+
// party authority. Harmless while such a node never mints an invite: the
|
|
4047
|
+
// store does not replicate, so a node trusting itself grants nothing to
|
|
4048
|
+
// others. But `createInvite` now hands out the anchor's contents as the
|
|
4049
|
+
// invitee's pins, so the moment a non-founder member mints an invite it
|
|
4050
|
+
// exports its own non-authority key as a cadre owner key. If that becomes
|
|
4051
|
+
// reachable (today only cadre-cli/cadre-host owners mint invites), gate this
|
|
4052
|
+
// self-anchor on the actual OwnerKey genesis insert instead.
|
|
4053
|
+
if (this.trustedOwnerStore) {
|
|
4054
|
+
void this.trustedOwnerStore
|
|
4055
|
+
.trust(ed25519PublicKeyFromPrivate(ownerPrivateKey), 'genesis')
|
|
4056
|
+
.catch((error) => log('Trusted-owner genesis anchor persist failed: %o', error));
|
|
4057
|
+
}
|
|
4058
|
+
|
|
2499
4059
|
this.seedBootstrapService = new SeedBootstrapService({
|
|
2500
4060
|
partyId: this.config.controlNetwork.partyId,
|
|
2501
4061
|
ownerPrivateKey,
|
|
2502
4062
|
inviteAddressResolver: () => this.resolveInviteAddresses(),
|
|
2503
4063
|
trustPolicy: this.config.seedTrustPolicy,
|
|
4064
|
+
// Seed trust anchors on the node-local store (seeded just above with this
|
|
4065
|
+
// node's own genesis key), never on the replicated OwnerKey table.
|
|
4066
|
+
...(this.trustedOwnerStore ? { trustedOwners: this.trustedOwnerStore } : {}),
|
|
2504
4067
|
});
|
|
2505
4068
|
|
|
2506
|
-
this.seedBootstrapService.setEventCallbacks(
|
|
2507
|
-
onSeedReceived: (partyId, peerId) => this.emit('seed:received', { partyId, peerId }),
|
|
2508
|
-
onSeedApplied: (partyId, peersAdded) => this.emit('seed:applied', { partyId, peersAdded }),
|
|
2509
|
-
onSeedError: (partyId, error) => this.emit('seed:error', { partyId, error }),
|
|
2510
|
-
});
|
|
4069
|
+
this.seedBootstrapService.setEventCallbacks(this.seedEventCallbacks());
|
|
2511
4070
|
|
|
2512
4071
|
this.seedBootstrapService.initialize(this.controlNode, this.controlDatabase);
|
|
2513
4072
|
log('Seed bootstrap service initialized');
|
|
@@ -2538,7 +4097,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2538
4097
|
}
|
|
2539
4098
|
|
|
2540
4099
|
/**
|
|
2541
|
-
* Enumerate the cadre's `CadrePeer` membership
|
|
4100
|
+
* Enumerate the cadre's `CadrePeer` membership — the ADDRESSABLE surface (see
|
|
4101
|
+
* {@link isMember}). Rows whose stamp is retired in `CadreControl.Revocation`
|
|
4102
|
+
* never reach here: {@link ControlDatabase.queryCadrePeers} excludes them, so a
|
|
4103
|
+
* revoked peer is not addressable either.
|
|
2542
4104
|
*/
|
|
2543
4105
|
async listMembers(): Promise<Array<{ peerId: string; multiaddr: string | null }>> {
|
|
2544
4106
|
if (!this.controlDatabase) {
|
|
@@ -2552,7 +4114,12 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2552
4114
|
*
|
|
2553
4115
|
* This is the ADDRESSABLE surface ("do I have a dialable address record for this
|
|
2554
4116
|
* peer") — it includes this node's own self-published row. Address resolution and
|
|
2555
|
-
* push fan-out use this.
|
|
4117
|
+
* push fan-out use this. A peer whose stamp is retired in `CadreControl.Revocation`
|
|
4118
|
+
* is excluded here too (the filter lives in
|
|
4119
|
+
* {@link ControlDatabase.queryCadrePeers}): revocation removes a peer from the
|
|
4120
|
+
* addressable surface, not only the trust-facing one, so a revoked peer is no
|
|
4121
|
+
* longer dialed, RPC'd, or handed out as an address. The trust-facing gate is
|
|
4122
|
+
* {@link isAuthorizedMember}.
|
|
2556
4123
|
*/
|
|
2557
4124
|
async isMember(peerId: string): Promise<boolean> {
|
|
2558
4125
|
const members = await this.listMembers();
|
|
@@ -2561,27 +4128,86 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2561
4128
|
|
|
2562
4129
|
/**
|
|
2563
4130
|
* Enumerate the party's AUTHORIZED members — the trust-facing set, distinct from
|
|
2564
|
-
* the addressable set ({@link listMembers}).
|
|
2565
|
-
*
|
|
2566
|
-
*
|
|
4131
|
+
* the addressable set ({@link listMembers}). A peer is authorized iff ALL hold:
|
|
4132
|
+
*
|
|
4133
|
+
* 1. it is not this node itself — a node publishes its own `CadrePeer` address
|
|
4134
|
+
* row so its dialable address rides in seeds, but "self" is not a peer this
|
|
4135
|
+
* node authorized;
|
|
4136
|
+
* 2. its `CadrePeer` row carries a complete voucher (`StampId`, `VouchOwner`,
|
|
4137
|
+
* `VouchSig` all non-null);
|
|
4138
|
+
* 3. `VouchOwner` is in the NODE-LOCAL trusted-owner anchor
|
|
4139
|
+
* ({@link getTrustedOwnerStore}) — never the replicated `OwnerKey` table,
|
|
4140
|
+
* which any stranger can genesis-pollute; and
|
|
4141
|
+
* 4. `VouchSig` verifies as that owner's signature over the row's voucher
|
|
4142
|
+
* digest ({@link verifyCadrePeerVoucher}), so the anchored owner really
|
|
4143
|
+
* vouched THIS peer id under THIS row's nonce; and
|
|
4144
|
+
* 5. the row's `StampId` is NOT retired in `CadreControl.Revocation` — enforced
|
|
4145
|
+
* upstream in {@link ControlDatabase.queryCadrePeers}, which drops retired rows
|
|
4146
|
+
* before ANY reader (this predicate included) sees them, so a row resurrected
|
|
4147
|
+
* by replaying the captured admission approval on a node that had not yet
|
|
4148
|
+
* converged on the tombstone (the write-time `NotRevoked` CHECK only sees
|
|
4149
|
+
* local rows) is still inert to every reader that has the tombstone.
|
|
4150
|
+
*
|
|
4151
|
+
* Fail-closed at every step: a missing anchor (pre-start), an empty anchor (a
|
|
4152
|
+
* not-yet-enrolled node authorizes no one), a null/partial voucher, an
|
|
4153
|
+
* unanchored `VouchOwner`, or a bad signature all yield "not authorized" —
|
|
4154
|
+
* having an address row is NOT membership. The control-network wake and
|
|
2567
4155
|
* strand-address gates consult this set, NOT the addressable one.
|
|
2568
4156
|
*
|
|
2569
|
-
* NOTE:
|
|
2570
|
-
*
|
|
2571
|
-
*
|
|
2572
|
-
*
|
|
2573
|
-
*
|
|
4157
|
+
* NOTE (rotation): if a party owner rotates keys and only the NEW key is pinned
|
|
4158
|
+
* in the anchor, rows the OLD key vouched fail check 3 until re-vouched — a
|
|
4159
|
+
* legit member goes un-authorized on readers that only pin the new key. Full
|
|
4160
|
+
* rotation handling (re-vouch on rotate) is the
|
|
4161
|
+
* `flip-strand-membership-rotation-known-gap` work, not this predicate's.
|
|
2574
4162
|
*/
|
|
2575
4163
|
async listAuthorizedMembers(): Promise<Array<{ peerId: string; multiaddr: string | null }>> {
|
|
4164
|
+
if (!this.controlDatabase) {
|
|
4165
|
+
throw new Error('CadreNode must be started before listing members');
|
|
4166
|
+
}
|
|
2576
4167
|
const selfPeerId = this.peerId?.toString();
|
|
2577
|
-
const
|
|
2578
|
-
|
|
4168
|
+
const rows = await this.controlDatabase.queryCadrePeers();
|
|
4169
|
+
const authorized = rows
|
|
4170
|
+
.filter(row => row.peerId !== selfPeerId && this.hasAnchoredVoucher(row))
|
|
4171
|
+
.map(({ peerId, multiaddr }) => ({ peerId, multiaddr }));
|
|
4172
|
+
// A node whose anchor was never seeded (no invite pin, no operator pin, not a
|
|
4173
|
+
// founder) refuses every wake and strand-addr request, which from the outside
|
|
4174
|
+
// looks like an unexplained "non-member" rejection. Say so once per call rather
|
|
4175
|
+
// than leaving the operator to infer it from silence.
|
|
4176
|
+
if (authorized.length === 0 && rows.length > 0 && (this.trustedOwnerStore?.all().size ?? 0) === 0) {
|
|
4177
|
+
log('listAuthorizedMembers: %d CadrePeer row(s) but the node-local trusted-owner anchor is EMPTY — authorizing no one (this node has no invite/operator owner-key pin)', rows.length);
|
|
4178
|
+
}
|
|
4179
|
+
return authorized;
|
|
4180
|
+
}
|
|
4181
|
+
|
|
4182
|
+
/**
|
|
4183
|
+
* Checks 2–4 of the authorized-membership predicate (see
|
|
4184
|
+
* {@link listAuthorizedMembers}) for one `CadrePeer` row: complete voucher,
|
|
4185
|
+
* `VouchOwner` in the node-local anchor, signature valid over the row's
|
|
4186
|
+
* (PeerId, StampId) voucher digest.
|
|
4187
|
+
*
|
|
4188
|
+
* NOTE: verifies the ed25519 signature on every call (no memo of already-
|
|
4189
|
+
* verified (peerId, stampId, vouchSig) triples). Cadres are a handful of
|
|
4190
|
+
* devices and the gates run per inbound request, so this is cheap today; if
|
|
4191
|
+
* membership or gate traffic ever grows, cache verified triples keyed on the
|
|
4192
|
+
* row's `StampId` (which rotates with every re-vouch).
|
|
4193
|
+
*/
|
|
4194
|
+
private hasAnchoredVoucher(row: CadrePeerVoucherFields): boolean {
|
|
4195
|
+
if (row.stampId === null || row.vouchOwner === null || row.vouchSig === null) {
|
|
4196
|
+
return false;
|
|
4197
|
+
}
|
|
4198
|
+
if (!this.trustedOwnerStore?.has(row.vouchOwner)) {
|
|
4199
|
+
return false;
|
|
4200
|
+
}
|
|
4201
|
+
return verifyCadrePeerVoucher(row.peerId, row.stampId, row.vouchOwner, row.vouchSig);
|
|
2579
4202
|
}
|
|
2580
4203
|
|
|
2581
4204
|
/**
|
|
2582
4205
|
* Probe whether a given peer is an AUTHORIZED party member (see
|
|
2583
4206
|
* {@link listAuthorizedMembers}) — the gate the control-network wake and
|
|
2584
|
-
* strand-address responders consult, NOT {@link isMember} (the addressable
|
|
4207
|
+
* strand-address responders consult, NOT {@link isMember} (the addressable
|
|
4208
|
+
* surface). Deliberately scans the full membership (one `CadrePeer` query)
|
|
4209
|
+
* rather than adding a single-row read path: cadres are small, and one code
|
|
4210
|
+
* path keeps the predicate impossible to drift from the list.
|
|
2585
4211
|
*/
|
|
2586
4212
|
async isAuthorizedMember(peerId: string): Promise<boolean> {
|
|
2587
4213
|
const members = await this.listAuthorizedMembers();
|
|
@@ -2636,13 +4262,13 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2636
4262
|
// No owner key - this node only receives seeds
|
|
2637
4263
|
inviteAddressResolver: () => this.resolveInviteAddresses(),
|
|
2638
4264
|
trustPolicy: this.config.seedTrustPolicy,
|
|
4265
|
+
// A listener-only node accepts a wire-delivered seed solely against this
|
|
4266
|
+
// anchor (there is no per-call override on the inbound handler): with no
|
|
4267
|
+
// genesis/invite/operator pin it authorizes nobody, which is the point.
|
|
4268
|
+
...(this.trustedOwnerStore ? { trustedOwners: this.trustedOwnerStore } : {}),
|
|
2639
4269
|
});
|
|
2640
4270
|
|
|
2641
|
-
this.seedBootstrapService.setEventCallbacks(
|
|
2642
|
-
onSeedReceived: (partyId, peerId) => this.emit('seed:received', { partyId, peerId }),
|
|
2643
|
-
onSeedApplied: (partyId, peersAdded) => this.emit('seed:applied', { partyId, peersAdded }),
|
|
2644
|
-
onSeedError: (partyId, error) => this.emit('seed:error', { partyId, error }),
|
|
2645
|
-
});
|
|
4271
|
+
this.seedBootstrapService.setEventCallbacks(this.seedEventCallbacks());
|
|
2646
4272
|
|
|
2647
4273
|
this.seedBootstrapService.initialize(this.controlNode, this.controlDatabase);
|
|
2648
4274
|
log('Seed listener enabled');
|
|
@@ -2655,6 +4281,123 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2655
4281
|
return this.seedBootstrapService;
|
|
2656
4282
|
}
|
|
2657
4283
|
|
|
4284
|
+
/**
|
|
4285
|
+
* The event callbacks every {@link SeedBootstrapService} this node owns is
|
|
4286
|
+
* wired with — shared by the owner-capable service ({@link initializeSeedBootstrap})
|
|
4287
|
+
* and the listener-only one ({@link enableSeedListener}) so the two cannot drift.
|
|
4288
|
+
*
|
|
4289
|
+
* A seed applied by the INBOUND protocol handler writes no `CadrePeer` row of
|
|
4290
|
+
* its own (it merges the libp2p peer store and dials owners), so the automatic
|
|
4291
|
+
* write-driven refresh never fires for it — yet applying it can ANCHOR a new
|
|
4292
|
+
* owner key, which flips rows already present from unauthorized to authorized.
|
|
4293
|
+
* Hence the explicit refresh here; without it a peer the freshly anchored owner
|
|
4294
|
+
* vouched for stays denied until the next timed cohort reconcile.
|
|
4295
|
+
*/
|
|
4296
|
+
private seedEventCallbacks(): SeedEventCallbacks {
|
|
4297
|
+
return {
|
|
4298
|
+
onSeedReceived: (partyId, peerId) => this.emit('seed:received', { partyId, peerId }),
|
|
4299
|
+
onSeedApplied: (partyId, peersAdded, seed) => {
|
|
4300
|
+
this.emit('seed:applied', { partyId, peersAdded });
|
|
4301
|
+
// A wire-delivered seed never passes through the `applySeed` wrapper, so
|
|
4302
|
+
// this is where its owner peers become cold-start bootstrap targets.
|
|
4303
|
+
this.recordSeedBootstrapPeers(seed);
|
|
4304
|
+
void this.refreshMembershipGate('seed-applied');
|
|
4305
|
+
},
|
|
4306
|
+
onSeedError: (partyId, error) => this.emit('seed:error', { partyId, error }),
|
|
4307
|
+
};
|
|
4308
|
+
}
|
|
4309
|
+
|
|
4310
|
+
/**
|
|
4311
|
+
* Re-materialize the authorized-peer snapshot that the fail-closed per-stream
|
|
4312
|
+
* control-DB gate ({@link authorizeInboundControlStream}) judges against.
|
|
4313
|
+
*
|
|
4314
|
+
* NO `CadrePeer` writer needs to call this: the control database notifies it
|
|
4315
|
+
* after every committed member-row write (`ControlDatabase.mutateCadrePeer`,
|
|
4316
|
+
* wired in {@link start}), which is exactly what makes the refresh automatic
|
|
4317
|
+
* rather than a caller obligation. It stays public for the changes that write
|
|
4318
|
+
* NO row locally and so raise no notification:
|
|
4319
|
+
*
|
|
4320
|
+
* - a membership row that arrived by REPLICATION (otherwise picked up on the
|
|
4321
|
+
* next timed cohort reconcile — bounded staleness by design), and
|
|
4322
|
+
* - a newly anchored trusted owner key ({@link applySeed}), which flips rows
|
|
4323
|
+
* ALREADY present from unauthorized to authorized without touching them.
|
|
4324
|
+
*
|
|
4325
|
+
* Coalescing: marks the snapshot stale and resolves once a refresh that began
|
|
4326
|
+
* after this call has completed, so a caller that awaits it always observes
|
|
4327
|
+
* its own change. Concurrent callers share one read. Idempotent and
|
|
4328
|
+
* best-effort (a failed read keeps the previous snapshot); never rejects.
|
|
4329
|
+
*/
|
|
4330
|
+
async refreshMembershipGate(reason = 'external-write'): Promise<void> {
|
|
4331
|
+
this.membershipGateDirty = true;
|
|
4332
|
+
if (this.membershipGateDeferDepth > 0) {
|
|
4333
|
+
// Flushed once when the enclosing scope exits.
|
|
4334
|
+
return;
|
|
4335
|
+
}
|
|
4336
|
+
while (this.membershipGateDirty) {
|
|
4337
|
+
// Re-checked after the await: the drain may have taken its last look at
|
|
4338
|
+
// the flag before we set it, in which case we need a fresh one.
|
|
4339
|
+
this.membershipGateDrain ??= this.drainMembershipGate(reason);
|
|
4340
|
+
await this.membershipGateDrain;
|
|
4341
|
+
}
|
|
4342
|
+
}
|
|
4343
|
+
|
|
4344
|
+
/**
|
|
4345
|
+
* The single in-flight refresh loop behind {@link refreshMembershipGate}: read
|
|
4346
|
+
* until the stale flag stays clear, then release the slot.
|
|
4347
|
+
*
|
|
4348
|
+
* NOTE: correct only because {@link membershipGateDirty} is guaranteed TRUE at
|
|
4349
|
+
* entry — its sole caller sets it synchronously, with no await in between. Were
|
|
4350
|
+
* this ever invoked with the flag already clear, the body would never suspend,
|
|
4351
|
+
* so the `finally` would null the slot BEFORE the caller's `??=` filled it, and
|
|
4352
|
+
* the slot would be left holding an already-settled drain that every later
|
|
4353
|
+
* refresh reuses — an unbreakable await/re-check spin. If a second caller is
|
|
4354
|
+
* ever added, have it mark the flag first (or assert it here).
|
|
4355
|
+
*/
|
|
4356
|
+
private async drainMembershipGate(reason: string): Promise<void> {
|
|
4357
|
+
try {
|
|
4358
|
+
while (this.membershipGateDirty) {
|
|
4359
|
+
// Cleared BEFORE the read (not after) so a write landing mid-read marks
|
|
4360
|
+
// the snapshot stale again and earns another pass, and so a throw still
|
|
4361
|
+
// terminates the loop.
|
|
4362
|
+
this.membershipGateDirty = false;
|
|
4363
|
+
await this.refreshAuthorizedControlPeers(reason);
|
|
4364
|
+
}
|
|
4365
|
+
} catch (error) {
|
|
4366
|
+
// Upholds `refreshMembershipGate`'s never-rejects contract. The read helper
|
|
4367
|
+
// already swallows its own failures, so this is defensive — but a rejection
|
|
4368
|
+
// here would otherwise surface out of every awaiting writer.
|
|
4369
|
+
log('membership gate drain (%s) failed — keeping previous snapshot: %o', reason, error);
|
|
4370
|
+
} finally {
|
|
4371
|
+
this.membershipGateDrain = null;
|
|
4372
|
+
}
|
|
4373
|
+
}
|
|
4374
|
+
|
|
4375
|
+
/**
|
|
4376
|
+
* Collapse a burst of `CadrePeer` writes into ONE gate refresh at scope exit.
|
|
4377
|
+
*
|
|
4378
|
+
* For loops that re-touch many rows (the write-while-alone drains), where a
|
|
4379
|
+
* per-row refresh would mean one full membership read per row for a snapshot
|
|
4380
|
+
* that only has to be correct once the loop settles.
|
|
4381
|
+
*
|
|
4382
|
+
* The depth counter is instance-level, so it also suppresses a genuinely
|
|
4383
|
+
* CONCURRENT external write's refresh for the life of the scope: that writer's
|
|
4384
|
+
* promise resolves before its peer is admitted, and admission lands at scope
|
|
4385
|
+
* exit instead. Acceptable — scope exit is milliseconds away (bounded by the
|
|
4386
|
+
* drain), versus the ~15 s reconcile interval that was the alternative before
|
|
4387
|
+
* any of this existed — but it is the reason these scopes stay short and rare.
|
|
4388
|
+
*/
|
|
4389
|
+
private async deferMembershipGateRefresh<T>(reason: string, body: () => Promise<T>): Promise<T> {
|
|
4390
|
+
this.membershipGateDeferDepth++;
|
|
4391
|
+
try {
|
|
4392
|
+
return await body();
|
|
4393
|
+
} finally {
|
|
4394
|
+
this.membershipGateDeferDepth--;
|
|
4395
|
+
if (this.membershipGateDeferDepth === 0 && this.membershipGateDirty) {
|
|
4396
|
+
await this.refreshMembershipGate(reason);
|
|
4397
|
+
}
|
|
4398
|
+
}
|
|
4399
|
+
}
|
|
4400
|
+
|
|
2658
4401
|
/**
|
|
2659
4402
|
* Authorize a new peer to join the cadre.
|
|
2660
4403
|
* Signs the peer ID with the owner key and inserts into CadrePeer table.
|
|
@@ -2666,6 +4409,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2666
4409
|
if (!this.seedBootstrapService) {
|
|
2667
4410
|
throw new Error('Seed bootstrap service not initialized. Call initializeSeedBootstrap() first.');
|
|
2668
4411
|
}
|
|
4412
|
+
// The insert notifies the control DB's membership hub, so the per-stream gate
|
|
4413
|
+
// has already admitted the just-vouched peer by the time this resolves.
|
|
2669
4414
|
await this.seedBootstrapService.authorizePeer({ peerId, multiaddrs });
|
|
2670
4415
|
// Queue for re-replication if this committed local-only (no connected cohort).
|
|
2671
4416
|
this.noteControlWrite(peerId, 'authorize');
|
|
@@ -2681,6 +4426,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2681
4426
|
if (!this.seedBootstrapService) {
|
|
2682
4427
|
throw new Error('Seed bootstrap service not initialized. Call initializeSeedBootstrap() first.');
|
|
2683
4428
|
}
|
|
4429
|
+
// The delete notifies the membership hub, so the removed peer is out of the
|
|
4430
|
+
// per-stream gate by the time this resolves.
|
|
2684
4431
|
await this.seedBootstrapService.removePeer(peerId);
|
|
2685
4432
|
// Track + loudly flag a delete that committed local-only (security-relevant).
|
|
2686
4433
|
this.noteControlWrite(peerId, 'remove');
|
|
@@ -2725,13 +4472,43 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2725
4472
|
const tempService = new SeedBootstrapService({
|
|
2726
4473
|
partyId: seed.partyId,
|
|
2727
4474
|
trustPolicy: this.config.seedTrustPolicy,
|
|
4475
|
+
// The anchor is node-scoped, not service-scoped: a throwaway service
|
|
4476
|
+
// must consult (and persist an accepted signer into) the SAME store the
|
|
4477
|
+
// persistent one would, or a cold-start enrollment via this path would
|
|
4478
|
+
// anchor nothing and re-prompt for the pin on every later seed.
|
|
4479
|
+
...(this.trustedOwnerStore ? { trustedOwners: this.trustedOwnerStore } : {}),
|
|
2728
4480
|
});
|
|
2729
4481
|
if (this.controlNode && this.controlDatabase) {
|
|
2730
4482
|
tempService.initialize(this.controlNode, this.controlDatabase, { registerHandler: false });
|
|
2731
4483
|
}
|
|
2732
|
-
|
|
4484
|
+
const tempResult = await tempService.applySeed(seed, options);
|
|
4485
|
+
this.noteAppliedSeed(tempResult, seed);
|
|
4486
|
+
// Not a row write (so nothing notifies): applying a seed can anchor a new
|
|
4487
|
+
// owner key, which flips rows ALREADY present into the authorized set.
|
|
4488
|
+
await this.refreshMembershipGate('applySeed');
|
|
4489
|
+
return tempResult;
|
|
4490
|
+
}
|
|
4491
|
+
const result = await this.seedBootstrapService.applySeed(seed, options);
|
|
4492
|
+
this.noteAppliedSeed(result, seed);
|
|
4493
|
+
await this.refreshMembershipGate('applySeed');
|
|
4494
|
+
return result;
|
|
4495
|
+
}
|
|
4496
|
+
|
|
4497
|
+
/**
|
|
4498
|
+
* Post-process an {@link applySeed} outcome: retain the seed's owner peers as
|
|
4499
|
+
* cold-start bootstrap targets, and surface a seed that was accepted but whose
|
|
4500
|
+
* every owner dial failed — the node is now seeded yet unconnected, and the
|
|
4501
|
+
* cold-start reconcile branch is what gets it out of that state.
|
|
4502
|
+
*/
|
|
4503
|
+
private noteAppliedSeed(result: ApplySeedResult, seed: ControlNetworkSeed): void {
|
|
4504
|
+
if (!result.success) {
|
|
4505
|
+
return;
|
|
4506
|
+
}
|
|
4507
|
+
this.recordSeedBootstrapPeers(seed);
|
|
4508
|
+
if (result.ownerDialsAttempted > 0 && result.ownerDialsFailed === result.ownerDialsAttempted) {
|
|
4509
|
+
log('applySeed: seeded but no owner reachable (%d/%d dial(s) failed) — cold-start reconcile will retry',
|
|
4510
|
+
result.ownerDialsFailed, result.ownerDialsAttempted);
|
|
2733
4511
|
}
|
|
2734
|
-
return await this.seedBootstrapService.applySeed(seed, options);
|
|
2735
4512
|
}
|
|
2736
4513
|
|
|
2737
4514
|
/**
|
|
@@ -2789,6 +4566,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2789
4566
|
if (!this.seedBootstrapService) {
|
|
2790
4567
|
throw new Error('Seed bootstrap service not initialized. Call initializeSeedBootstrap() first.');
|
|
2791
4568
|
}
|
|
4569
|
+
// The drone's `CadrePeer` insert notifies the membership hub on the way to the
|
|
4570
|
+
// seed, so the per-stream gate already admits it here.
|
|
2792
4571
|
return await this.seedBootstrapService.addDrone(options);
|
|
2793
4572
|
}
|
|
2794
4573
|
|
|
@@ -2800,7 +4579,12 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2800
4579
|
if (!this.seedBootstrapService) {
|
|
2801
4580
|
throw new Error('Seed bootstrap service not initialized. Call initializeSeedBootstrap() first.');
|
|
2802
4581
|
}
|
|
2803
|
-
|
|
4582
|
+
const result = await this.seedBootstrapService.createInvite(token, expiresIn);
|
|
4583
|
+
// The invitee dials this node BEFORE it is authorized (dialInvite →
|
|
4584
|
+
// acceptPhone), so hold the inbound connection gate open for strangers for
|
|
4585
|
+
// the invite's validity (or a bounded default when it never expires).
|
|
4586
|
+
this.openEnrollmentWindow(result.invite.expiresAt ?? Date.now() + DEFAULT_ENROLLMENT_WINDOW_MS);
|
|
4587
|
+
return result;
|
|
2804
4588
|
}
|
|
2805
4589
|
|
|
2806
4590
|
/**
|
|
@@ -2811,6 +4595,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2811
4595
|
if (!this.seedBootstrapService) {
|
|
2812
4596
|
throw new Error('Seed bootstrap service not initialized. Call initializeSeedBootstrap() first.');
|
|
2813
4597
|
}
|
|
4598
|
+
// The just-accepted phone opens control-DB streams next; its `CadrePeer` insert
|
|
4599
|
+
// notifies the membership hub, so it is already admitted when this resolves.
|
|
2814
4600
|
await this.seedBootstrapService.acceptPhone(options, issuedInvite);
|
|
2815
4601
|
}
|
|
2816
4602
|
|
|
@@ -2822,6 +4608,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2822
4608
|
if (!this.seedBootstrapService) {
|
|
2823
4609
|
throw new Error('Seed bootstrap service not initialized. Call initializeSeedBootstrap() first.');
|
|
2824
4610
|
}
|
|
4611
|
+
// The service authorizes the phone (a `CadrePeer` insert, which notifies the
|
|
4612
|
+
// membership hub) on the way to the seed, so the per-stream gate admits it
|
|
4613
|
+
// before this resolves — not on the next reconcile, by which time the phone's
|
|
4614
|
+
// own control-DB schema load may have died denied.
|
|
2825
4615
|
return await this.seedBootstrapService.addPhoneWithRelay(phonePeerId);
|
|
2826
4616
|
}
|
|
2827
4617
|
|