@serfab/cadre-core 0.8.1 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -4
- package/dist/bootstrap-peer-store-file.d.ts +13 -0
- package/dist/bootstrap-peer-store-file.js +42 -0
- package/dist/bootstrap-peer-store-file.js.map +1 -0
- package/dist/bootstrap-peer-store.d.ts +73 -0
- package/dist/bootstrap-peer-store.js +128 -0
- package/dist/bootstrap-peer-store.js.map +1 -0
- package/dist/cadre-node.d.ts +1006 -101
- package/dist/cadre-node.js +1977 -283
- package/dist/cadre-node.js.map +1 -1
- package/dist/control-authorization.d.ts +93 -0
- package/dist/control-authorization.js +57 -0
- package/dist/control-authorization.js.map +1 -0
- package/dist/control-cohort.d.ts +15 -15
- package/dist/control-cohort.js +14 -14
- package/dist/control-cohort.js.map +1 -1
- package/dist/control-database.d.ts +1033 -76
- package/dist/control-database.js +1520 -153
- package/dist/control-database.js.map +1 -1
- package/dist/control-formation-recorder.d.ts +91 -12
- package/dist/control-formation-recorder.js +131 -16
- package/dist/control-formation-recorder.js.map +1 -1
- package/dist/control-schema.d.ts +1 -1
- package/dist/control-schema.js +774 -186
- package/dist/control-schema.js.map +1 -1
- package/dist/control-stream.d.ts +38 -4
- package/dist/control-stream.js +70 -4
- package/dist/control-stream.js.map +1 -1
- package/dist/control-write-retry.d.ts +151 -0
- package/dist/control-write-retry.js +411 -0
- package/dist/control-write-retry.js.map +1 -0
- package/dist/delegate-admission.d.ts +134 -0
- package/dist/delegate-admission.js +244 -0
- package/dist/delegate-admission.js.map +1 -0
- package/dist/device-token.d.ts +12 -12
- package/dist/device-token.js +15 -14
- package/dist/device-token.js.map +1 -1
- package/dist/ed25519-key.d.ts +68 -0
- package/dist/ed25519-key.js +106 -0
- package/dist/ed25519-key.js.map +1 -0
- package/dist/file-durable-slot.d.ts +20 -0
- package/dist/file-durable-slot.js +53 -0
- package/dist/file-durable-slot.js.map +1 -0
- package/dist/formation-approval.d.ts +157 -0
- package/dist/formation-approval.js +375 -0
- package/dist/formation-approval.js.map +1 -0
- package/dist/fs-atomic.d.ts +38 -0
- package/dist/fs-atomic.js +97 -0
- package/dist/fs-atomic.js.map +1 -0
- package/dist/identity-key.d.ts +49 -0
- package/dist/identity-key.js +81 -0
- package/dist/identity-key.js.map +1 -0
- package/dist/index.d.ts +16 -6
- package/dist/index.js +44 -10
- package/dist/index.js.map +1 -1
- package/dist/key-store-file.d.ts +2 -24
- package/dist/key-store-file.js +10 -77
- package/dist/key-store-file.js.map +1 -1
- package/dist/key-store.d.ts +1 -1
- package/dist/key-store.js +1 -1
- package/dist/membership-connection-gater.d.ts +146 -0
- package/dist/membership-connection-gater.js +180 -0
- package/dist/membership-connection-gater.js.map +1 -0
- package/dist/node-local-snapshot.d.ts +109 -0
- package/dist/node-local-snapshot.js +182 -0
- package/dist/node-local-snapshot.js.map +1 -0
- package/dist/peer-authorization.d.ts +150 -11
- package/dist/peer-authorization.js +170 -13
- package/dist/peer-authorization.js.map +1 -1
- package/dist/peer-record.d.ts +24 -11
- package/dist/peer-record.js +29 -13
- package/dist/peer-record.js.map +1 -1
- package/dist/push-node.d.ts +38 -0
- package/dist/push-node.js +48 -0
- package/dist/push-node.js.map +1 -0
- package/dist/push-notifier-apns.d.ts +10 -2
- package/dist/push-notifier-apns.js +14 -10
- package/dist/push-notifier-apns.js.map +1 -1
- package/dist/push-notifier-fcm.d.ts +7 -1
- package/dist/push-notifier-fcm.js +9 -10
- package/dist/push-notifier-fcm.js.map +1 -1
- package/dist/push-notifier.d.ts +16 -35
- package/dist/push-notifier.js +16 -44
- package/dist/push-notifier.js.map +1 -1
- package/dist/relay-addrs.d.ts +45 -0
- package/dist/relay-addrs.js +103 -0
- package/dist/relay-addrs.js.map +1 -0
- package/dist/relay-reservation.d.ts +266 -0
- package/dist/relay-reservation.js +613 -0
- package/dist/relay-reservation.js.map +1 -0
- package/dist/seed-bootstrap.d.ts +263 -79
- package/dist/seed-bootstrap.js +413 -211
- package/dist/seed-bootstrap.js.map +1 -1
- package/dist/seed-trust-policy.d.ts +50 -24
- package/dist/seed-trust-policy.js +28 -19
- package/dist/seed-trust-policy.js.map +1 -1
- package/dist/strand-addr-protocol.d.ts +42 -13
- package/dist/strand-addr-protocol.js +58 -52
- package/dist/strand-addr-protocol.js.map +1 -1
- package/dist/strand-backfill.d.ts +109 -0
- package/dist/strand-backfill.js +279 -0
- package/dist/strand-backfill.js.map +1 -0
- package/dist/strand-database.d.ts +5 -5
- package/dist/strand-database.js +4 -4
- package/dist/strand-database.js.map +1 -1
- package/dist/strand-formation-manager.d.ts +38 -9
- package/dist/strand-formation-manager.js +120 -17
- package/dist/strand-formation-manager.js.map +1 -1
- package/dist/strand-formation-protocol.d.ts +104 -5
- package/dist/strand-formation-protocol.js +233 -5
- package/dist/strand-formation-protocol.js.map +1 -1
- package/dist/strand-instance-manager.d.ts +40 -2
- package/dist/strand-instance-manager.js +93 -15
- package/dist/strand-instance-manager.js.map +1 -1
- package/dist/strand-member-key.d.ts +6 -6
- package/dist/strand-member-key.js +6 -6
- package/dist/strand-member-key.js.map +1 -1
- package/dist/strand-member-registry.d.ts +12 -12
- package/dist/strand-member-registry.js +22 -14
- package/dist/strand-member-registry.js.map +1 -1
- package/dist/strand-membership-writer.d.ts +532 -138
- package/dist/strand-membership-writer.js +800 -179
- package/dist/strand-membership-writer.js.map +1 -1
- package/dist/strand-solicitation.d.ts +99 -33
- package/dist/strand-solicitation.js +100 -52
- package/dist/strand-solicitation.js.map +1 -1
- package/dist/strand-transport-key.d.ts +40 -0
- package/dist/strand-transport-key.js +56 -0
- package/dist/strand-transport-key.js.map +1 -0
- package/dist/strand-wake-protocol.d.ts +9 -7
- package/dist/strand-wake-protocol.js +22 -46
- package/dist/strand-wake-protocol.js.map +1 -1
- package/dist/strand-watcher.d.ts +17 -2
- package/dist/strand-watcher.js +47 -3
- package/dist/strand-watcher.js.map +1 -1
- package/dist/trusted-owner-store-file.d.ts +12 -0
- package/dist/trusted-owner-store-file.js +39 -0
- package/dist/trusted-owner-store-file.js.map +1 -0
- package/dist/trusted-owner-store.d.ts +64 -0
- package/dist/trusted-owner-store.js +126 -0
- package/dist/trusted-owner-store.js.map +1 -0
- package/dist/types.d.ts +325 -50
- package/dist/types.js +13 -0
- package/dist/types.js.map +1 -1
- package/package.json +23 -8
- package/src/bootstrap-peer-store-file.ts +44 -0
- package/src/bootstrap-peer-store.ts +172 -0
- package/src/cadre-node.ts +2129 -295
- package/src/control-authorization.ts +114 -0
- package/src/control-cohort.ts +17 -17
- package/src/control-database.ts +1766 -191
- package/src/control-formation-recorder.ts +174 -21
- package/src/control-schema.ts +786 -198
- package/src/control-stream.ts +81 -4
- package/src/control-write-retry.ts +473 -0
- package/src/delegate-admission.ts +302 -0
- package/src/device-token.ts +19 -14
- package/src/ed25519-key.ts +134 -0
- package/src/file-durable-slot.ts +61 -0
- package/src/formation-approval.ts +604 -0
- package/src/fs-atomic.ts +107 -0
- package/src/identity-key.ts +109 -0
- package/src/index.ts +157 -24
- package/src/key-store-file.ts +10 -81
- package/src/key-store.ts +1 -1
- package/src/membership-connection-gater.ts +204 -0
- package/src/node-local-snapshot.ts +260 -0
- package/src/peer-authorization.ts +219 -13
- package/src/peer-record.ts +31 -13
- package/src/push-node.ts +64 -0
- package/src/push-notifier-apns.ts +16 -12
- package/src/push-notifier-fcm.ts +9 -10
- package/src/push-notifier.ts +16 -57
- package/src/relay-addrs.ts +111 -0
- package/src/relay-reservation.ts +823 -0
- package/src/seed-bootstrap.ts +490 -226
- package/src/seed-trust-policy.ts +65 -32
- package/src/strand-addr-protocol.ts +84 -53
- package/src/strand-backfill.ts +366 -0
- package/src/strand-database.ts +6 -6
- package/src/strand-formation-manager.ts +145 -19
- package/src/strand-formation-protocol.ts +310 -9
- package/src/strand-instance-manager.ts +113 -16
- package/src/strand-member-key.ts +7 -7
- package/src/strand-member-registry.ts +28 -23
- package/src/strand-membership-writer.ts +1430 -640
- package/src/strand-solicitation.ts +155 -90
- package/src/strand-transport-key.ts +58 -0
- package/src/strand-wake-protocol.ts +29 -46
- package/src/strand-watcher.ts +66 -3
- package/src/trusted-owner-store-file.ts +41 -0
- package/src/trusted-owner-store.ts +171 -0
- package/src/types.ts +342 -51
- package/dist/authority-key.d.ts +0 -41
- package/dist/authority-key.js +0 -52
- package/dist/authority-key.js.map +0 -1
- package/src/authority-key.ts +0 -68
package/src/cadre-node.ts
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import debug from 'debug';
|
|
2
2
|
import { toString as uint8ArrayToString, fromString as uint8ArrayFromString } from 'uint8arrays';
|
|
3
3
|
import type { Libp2p, PeerId, PrivateKey, Connection } from '@libp2p/interface';
|
|
4
|
-
import {
|
|
5
|
-
import { peerIdFromString } from '@libp2p/peer-id';
|
|
4
|
+
import { peerIdFromString, peerIdFromPrivateKey } from '@libp2p/peer-id';
|
|
6
5
|
import { createLibp2pNode } from '@optimystic/db-p2p';
|
|
7
6
|
import { multiaddr } from '@multiformats/multiaddr';
|
|
8
7
|
import type { Multiaddr } from '@multiformats/multiaddr';
|
|
@@ -30,12 +29,20 @@ import type {
|
|
|
30
29
|
Libp2pNodeWithRepo,
|
|
31
30
|
PushPlatform,
|
|
32
31
|
DeviceTokenRecord,
|
|
32
|
+
CadrePeerVoucherFields,
|
|
33
|
+
CadrePeerRow,
|
|
34
|
+
PeerAddressRecord,
|
|
33
35
|
ResolveDeviceTokenOpts
|
|
34
36
|
} from './types.js';
|
|
35
|
-
import { DEFAULT_CHECKIN_WINDOW_MS } from './types.js';
|
|
37
|
+
import { CONTROL_CLUSTER_POLICY, CONTROL_REPLICATION_BREADTH, DEFAULT_CHECKIN_WINDOW_MS } from './types.js';
|
|
36
38
|
import { sign } from '@optimystic/quereus-plugin-crypto';
|
|
37
|
-
import {
|
|
39
|
+
import { ed25519KeyPairFromLibp2p, ed25519PublicKeyFromPrivate, requireEd25519PublicKeyB64, type Ed25519KeyPair } from './ed25519-key.js';
|
|
40
|
+
import { strandTransportKey } from './strand-transport-key.js';
|
|
38
41
|
import { DEFAULT_IDENTITY_KEY_ID } from './key-store.js';
|
|
42
|
+
import { loadOrCreateIdentityKey } from './identity-key.js';
|
|
43
|
+
import { MemoryTrustedOwnerStore, type TrustedOwnerStore, type TrustSource } from './trusted-owner-store.js';
|
|
44
|
+
import { MemoryBootstrapPeerStore, type BootstrapPeerStore } from './bootstrap-peer-store.js';
|
|
45
|
+
import { verifyCadrePeerVoucher } from './peer-authorization.js';
|
|
39
46
|
import { ed25519PublicKeyB64FromPeerId } from './seed-bootstrap.js';
|
|
40
47
|
import {
|
|
41
48
|
signPeerRecord,
|
|
@@ -43,6 +50,7 @@ import {
|
|
|
43
50
|
isPeerRecordFresh,
|
|
44
51
|
orderSignalingFirst,
|
|
45
52
|
isSignalingAddr,
|
|
53
|
+
trailingPeerId,
|
|
46
54
|
currentMemberTrustPolicy,
|
|
47
55
|
DEFAULT_PEER_RECORD_MAX_AGE_MS,
|
|
48
56
|
DEFAULT_PEER_RECORD_HEARTBEAT_MS
|
|
@@ -63,15 +71,32 @@ import {
|
|
|
63
71
|
} from './control-cohort.js';
|
|
64
72
|
import { EnrollmentService } from './enrollment.js';
|
|
65
73
|
import { HibernationManager, type HibernationCallbacks } from './hibernation-manager.js';
|
|
66
|
-
import { ControlDatabase } from './control-database.js';
|
|
67
|
-
import { SeedBootstrapService } from './seed-bootstrap.js';
|
|
74
|
+
import { ControlDatabase, type RevokedRowRef } from './control-database.js';
|
|
75
|
+
import { SeedBootstrapService, type SeedEventCallbacks } from './seed-bootstrap.js';
|
|
68
76
|
import type { SeedTrustPolicy } from './seed-trust-policy.js';
|
|
69
77
|
import {
|
|
70
78
|
StrandSolicitationService,
|
|
71
79
|
type StrandSolicitationServiceOptions
|
|
72
80
|
} from './strand-solicitation.js';
|
|
81
|
+
import { createMembershipConnectionGater, DEFAULT_ENROLLMENT_WINDOW_MS } from './membership-connection-gater.js';
|
|
73
82
|
import { StrandWakeService, dialWake } from './strand-wake-protocol.js';
|
|
74
|
-
import { StrandAddrService, collectStrandAddrs } from './strand-addr-protocol.js';
|
|
83
|
+
import { StrandAddrService, collectStrandAddrs, type StrandAddrPeer } from './strand-addr-protocol.js';
|
|
84
|
+
import {
|
|
85
|
+
DelegateAdmissionStore,
|
|
86
|
+
extractCircuitRelayTargets,
|
|
87
|
+
dueRelayAnnounces,
|
|
88
|
+
pruneStoppedStrandAnnounces,
|
|
89
|
+
peerStrandKey,
|
|
90
|
+
type CircuitRelayTarget
|
|
91
|
+
} from './delegate-admission.js';
|
|
92
|
+
import { resolveListenAddrs } from './relay-addrs.js';
|
|
93
|
+
import {
|
|
94
|
+
superviseRelayReservation,
|
|
95
|
+
resolveRelayReservationState,
|
|
96
|
+
type RelayReservationState,
|
|
97
|
+
type RelayReservationSupervisor,
|
|
98
|
+
type RelayReservationSupervisorOptions
|
|
99
|
+
} from './relay-reservation.js';
|
|
75
100
|
import { PushFanoutService } from './push-fanout.js';
|
|
76
101
|
import type { WakeAck, WakeRequest } from './types.js';
|
|
77
102
|
import {
|
|
@@ -94,6 +119,37 @@ const TURN_CONSUME_WINDOW_MS = 1000;
|
|
|
94
119
|
|
|
95
120
|
type EventHandler<T> = (data: T) => void;
|
|
96
121
|
|
|
122
|
+
/**
|
|
123
|
+
* A circuit relay as a strand-addr RPC target: dialed by peerId, with its
|
|
124
|
+
* direct addr riding along as the fallback for a relay we hold no connection to.
|
|
125
|
+
*/
|
|
126
|
+
function relayStrandAddrPeer(relay: CircuitRelayTarget): StrandAddrPeer {
|
|
127
|
+
return { peerId: relay.relayPeerId, addrs: [multiaddr(relay.relayAddr)] };
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* Build the signing callback the `ControlDatabase` writers expect: they hand it the
|
|
132
|
+
* canonical row-bound message BYTES (see `buildAuthorizationMessage`), and it ed25519-signs
|
|
133
|
+
* them directly (no pre-hash) with the owner private key, returning a base64url signature.
|
|
134
|
+
*/
|
|
135
|
+
function signMessageWith(privateKeyB64: string): (message: Uint8Array) => string {
|
|
136
|
+
return (message: Uint8Array): string =>
|
|
137
|
+
sign(message, privateKeyB64, 'ed25519', 'bytes', 'base64url', 'base64url') as string;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Reject a blank identifier before it reaches the control database, where it would fail an
|
|
142
|
+
* authorization CHECK as an opaque constraint error rather than an actionable message.
|
|
143
|
+
* Returns the trimmed value so callers write the same bytes they validated.
|
|
144
|
+
*/
|
|
145
|
+
function requireNonBlank(value: string, label: string): string {
|
|
146
|
+
const trimmed = value.trim();
|
|
147
|
+
if (trimmed.length === 0) {
|
|
148
|
+
throw new Error(`A ${label} is required (received an empty or whitespace-only value)`);
|
|
149
|
+
}
|
|
150
|
+
return trimmed;
|
|
151
|
+
}
|
|
152
|
+
|
|
97
153
|
/**
|
|
98
154
|
* CadreNode is the main entry point for a cadre member.
|
|
99
155
|
* It manages:
|
|
@@ -109,11 +165,20 @@ export class CadreNode implements SAppIdLookup {
|
|
|
109
165
|
* The resolved node identity key, set once by {@link resolveIdentityKey}
|
|
110
166
|
* during {@link start} (from `config.keyStore`, else `config.privateKey`).
|
|
111
167
|
* Left undefined when neither is configured — libp2p then generates an
|
|
112
|
-
* ephemeral key internally and there is no exposed
|
|
168
|
+
* ephemeral key internally and there is no exposed owner key. Every
|
|
113
169
|
* identity-dependent path (control node creation, self-record signing, strand
|
|
114
170
|
* launch) reads this resolved field, never `config.privateKey` directly.
|
|
115
171
|
*/
|
|
116
172
|
private identityKey: PrivateKey | undefined;
|
|
173
|
+
/**
|
|
174
|
+
* Node-local, NON-replicated trusted-owner anchor (see
|
|
175
|
+
* `trusted-owner-store.ts`). Constructed (or adopted from
|
|
176
|
+
* `config.trustedOwners.store`) by {@link initializeTrustedOwnerStore} during
|
|
177
|
+
* {@link start}; deliberately NOT cleared by {@link cleanup}, so an in-memory
|
|
178
|
+
* anchor survives a stop()→start() cycle of the same node instance. Never
|
|
179
|
+
* sourced from replicated control state.
|
|
180
|
+
*/
|
|
181
|
+
private trustedOwnerStore: TrustedOwnerStore | null = null;
|
|
117
182
|
private controlNode: Libp2p | null = null;
|
|
118
183
|
private controlDatabase: ControlDatabase | null = null;
|
|
119
184
|
private strandWatcher: StrandWatcher | null = null;
|
|
@@ -132,9 +197,9 @@ export class CadreNode implements SAppIdLookup {
|
|
|
132
197
|
private strandAddrService: StrandAddrService | null = null;
|
|
133
198
|
/**
|
|
134
199
|
* Server-side push-wake fan-out. Constructed by {@link start} only when
|
|
135
|
-
* `config.push` (
|
|
136
|
-
* exactly as before (no notifier, no fan-out). Owns who/when
|
|
137
|
-
* hibernating mobile peers on strand activity.
|
|
200
|
+
* `config.push` (an injected `PushNotifier` + policy) is present — without it
|
|
201
|
+
* the node behaves exactly as before (no notifier, no fan-out). Owns who/when
|
|
202
|
+
* to wake hibernating mobile peers on strand activity.
|
|
138
203
|
*/
|
|
139
204
|
private pushFanoutService: PushFanoutService | null = null;
|
|
140
205
|
/** Backing field for the {@link running} / {@link isRunning} getters. */
|
|
@@ -154,6 +219,26 @@ export class CadreNode implements SAppIdLookup {
|
|
|
154
219
|
private windowWaiters: Set<{ timer: ReturnType<typeof setTimeout>; resolve: () => void }> = new Set();
|
|
155
220
|
private eventHandlers: Map<keyof CadreNodeEvents, Set<EventHandler<never>>> = new Map();
|
|
156
221
|
|
|
222
|
+
/**
|
|
223
|
+
* Relay multiaddrs the caller asked {@link reserveRelays} to reserve through.
|
|
224
|
+
* Empty (the default) means nobody asked, so {@link getRelayReservationState}
|
|
225
|
+
* reports `none` — this whole surface is opt-in and dormant for the CLI/host
|
|
226
|
+
* nodes, which use the configured `network.relayAddrs` route instead.
|
|
227
|
+
*/
|
|
228
|
+
private relayReserveAddrs: string[] = [];
|
|
229
|
+
/**
|
|
230
|
+
* The running retry loop for {@link relayReserveAddrs}, or `null` when nobody
|
|
231
|
+
* asked for a reservation / there was no control node to supervise. It owns the
|
|
232
|
+
* in-flight flag, the last failure and the next-attempt time; this class only
|
|
233
|
+
* starts it, stops it and reads it.
|
|
234
|
+
*/
|
|
235
|
+
private relayReserveSupervisor: RelayReservationSupervisor | null = null;
|
|
236
|
+
/**
|
|
237
|
+
* Reason {@link reserveRelays} produced no reservation when there is no
|
|
238
|
+
* supervisor to hold one — i.e. the pre-start `control node unavailable` case.
|
|
239
|
+
*/
|
|
240
|
+
private relayReserveError: string | null = null;
|
|
241
|
+
|
|
157
242
|
/** Map of strandId -> sAppConfig for sAppId filtering and management */
|
|
158
243
|
private sAppConfigs: Map<string, SAppConfig> = new Map();
|
|
159
244
|
|
|
@@ -165,6 +250,113 @@ export class CadreNode implements SAppIdLookup {
|
|
|
165
250
|
*/
|
|
166
251
|
private latestInviteAddresses: string[] | null = null;
|
|
167
252
|
|
|
253
|
+
/**
|
|
254
|
+
* Epoch ms until which the control-network inbound gate admits
|
|
255
|
+
* not-yet-authorized peers (see {@link admitInboundControlConnection} /
|
|
256
|
+
* {@link openEnrollmentWindow}). 0 = no window open. Opened automatically by
|
|
257
|
+
* {@link createInvite}; deliberately NOT reset on stop() — a stop/start cycle
|
|
258
|
+
* inside an outstanding invite's validity must not strand the invitee.
|
|
259
|
+
*
|
|
260
|
+
* NOTE: in-memory only, so a PROCESS restart mid-invite closes the door until
|
|
261
|
+
* the owner re-mints or the invitee is authorized. Fine while invites are
|
|
262
|
+
* short-lived and redeemed promptly; if long-lived invites or restart-prone
|
|
263
|
+
* hosts become normal, persist the window (or derive it from the issued-invite
|
|
264
|
+
* records `SeedBootstrapService` already keeps).
|
|
265
|
+
*/
|
|
266
|
+
private enrollmentWindowUntil = 0;
|
|
267
|
+
|
|
268
|
+
/**
|
|
269
|
+
* Lazily-parsed PeerIds of `controlNetwork.bootstrapNodes` (see
|
|
270
|
+
* {@link getBootstrapPeerIds}) — infrastructure the inbound gate always admits.
|
|
271
|
+
*/
|
|
272
|
+
private bootstrapPeerIds: Set<string> | null = null;
|
|
273
|
+
|
|
274
|
+
/**
|
|
275
|
+
* Materialized in-memory snapshot of the AUTHORIZED member peer ids (the
|
|
276
|
+
* {@link listAuthorizedMembers} result), consulted by the per-stream
|
|
277
|
+
* control-DB gate ({@link authorizeInboundControlStream}). That predicate
|
|
278
|
+
* runs on EVERY inbound Optimystic control-DB stream and must never await a
|
|
279
|
+
* control-DB read: those reads pull blocks over the very protocols the
|
|
280
|
+
* predicate gates, which closes a circular wait that ends in mutual denial
|
|
281
|
+
* (the upstream gate is fail-closed on timeout, unlike the fail-open
|
|
282
|
+
* connection gater). So the set is refreshed OUT OF BAND instead — after
|
|
283
|
+
* start, on every {@link reconcileControlCohort} pass (15s cadence), and
|
|
284
|
+
* immediately after every LOCAL membership mutation so a just-vouched peer
|
|
285
|
+
* is admitted without waiting for the timer. That last refresh is AUTOMATIC:
|
|
286
|
+
* the control database notifies {@link refreshMembershipGate} after every
|
|
287
|
+
* committed `CadrePeer` write (see `ControlDatabase.mutateCadrePeer`), so no
|
|
288
|
+
* writer has to remember. A change that arrives by REPLICATION is picked up
|
|
289
|
+
* on the next timed refresh — bounded staleness, acceptable because this gate
|
|
290
|
+
* is defense in depth: rows an unadmitted peer manages to write are still
|
|
291
|
+
* disbelieved at read time by the voucher-anchored predicate.
|
|
292
|
+
*/
|
|
293
|
+
private authorizedControlPeers: Set<string> = new Set();
|
|
294
|
+
|
|
295
|
+
/**
|
|
296
|
+
* Coalescing state for {@link refreshMembershipGate}: a pending "the snapshot
|
|
297
|
+
* is stale" flag, the single in-flight drain that consumes it, and the depth
|
|
298
|
+
* of open {@link deferMembershipGateRefresh} scopes (a burst of writes inside
|
|
299
|
+
* one scope collapses to a single refresh at scope exit).
|
|
300
|
+
*/
|
|
301
|
+
private membershipGateDirty = false;
|
|
302
|
+
private membershipGateDrain: Promise<void> | null = null;
|
|
303
|
+
private membershipGateDeferDepth = 0;
|
|
304
|
+
|
|
305
|
+
/**
|
|
306
|
+
* In-memory delegate admission grants (see `delegate-admission.ts`): the
|
|
307
|
+
* strand-node transport peerIds this party's members have announced over the
|
|
308
|
+
* strand-addr RPC, admitted at the CONNECTION level so a member's NAT'd
|
|
309
|
+
* strand node can hold a circuit-relay reservation on this node. Consulted
|
|
310
|
+
* ONLY by {@link admitInboundControlConnection}; the fail-closed per-stream
|
|
311
|
+
* gate ({@link authorizeInboundControlStream}) never honors it.
|
|
312
|
+
*/
|
|
313
|
+
private readonly delegateAdmission = new DelegateAdmissionStore();
|
|
314
|
+
|
|
315
|
+
/**
|
|
316
|
+
* When this node last ANNOUNCED a delegate, keyed `<targetPeerId>\n<strandId>`
|
|
317
|
+
* (the peer announced TO, not the delegate). Throttles
|
|
318
|
+
* {@link refreshDelegateGrants} to once per `DELEGATE_GRANT_TTL_MS / 2` per
|
|
319
|
+
* (relay, strand) so the 15 s reconcile tick never becomes per-tick RPC
|
|
320
|
+
* chatter; the launch/resume announce passes record here too. Keys whose
|
|
321
|
+
* strand is no longer running are pruned on each reconcile pass.
|
|
322
|
+
*/
|
|
323
|
+
private readonly delegateAnnounceAt = new Map<string, number>();
|
|
324
|
+
|
|
325
|
+
/**
|
|
326
|
+
* Cold-start bootstrap dial targets: the owner-flagged peers of every seed
|
|
327
|
+
* this node has applied, keyed by peer id, valued by the seed's multiaddr
|
|
328
|
+
* strings (parsed lazily, at dial time). Written by
|
|
329
|
+
* {@link recordSeedBootstrapPeers}; read only by
|
|
330
|
+
* {@link dialColdStartBootstrap} while the control database still has no
|
|
331
|
+
* siblings to dial.
|
|
332
|
+
*
|
|
333
|
+
* A STORE rather than a plain map (see `bootstrap-peer-store.ts`), because the
|
|
334
|
+
* targets must outlive the process: a node seeded into a party it could not
|
|
335
|
+
* reach has nothing else on disk naming that party's addresses (`applySeed`
|
|
336
|
+
* writes no control row, and `CadrePeer` fills in only after a connection
|
|
337
|
+
* succeeds), so an in-memory-only set left it stranded permanently across a
|
|
338
|
+
* restart. Constructed (or adopted from `config.bootstrapPeers.store`) by
|
|
339
|
+
* {@link initializeBootstrapPeerStore} during {@link start}; deliberately NOT
|
|
340
|
+
* cleared by {@link cleanup}, so it survives a stop()→start() cycle of the same
|
|
341
|
+
* node instance — same lifecycle as {@link trustedOwnerStore}. Durability
|
|
342
|
+
* depends on the injected backend: Node gets a file-backed store from the CLI,
|
|
343
|
+
* RN/browser stay ephemeral for now.
|
|
344
|
+
*
|
|
345
|
+
* Deliberately NOT the libp2p peer store, which {@link peerStoreAddrs} already
|
|
346
|
+
* reads for the steady-state path. The peer store is shared with everything
|
|
347
|
+
* libp2p discovers, so "dial every entry" would grow into dialing arbitrary
|
|
348
|
+
* discovered peers as the node lives longer; this store holds exactly the peers
|
|
349
|
+
* an owner-signed, trust-anchored seed nominated as owners. `applySeed` merges
|
|
350
|
+
* the same addresses into the peer store as well, so the two never disagree —
|
|
351
|
+
* this one is just the precisely-scoped subset.
|
|
352
|
+
*
|
|
353
|
+
* A later seed OVERWRITES an entry rather than merging, so a re-seed after an
|
|
354
|
+
* owner's address changes replaces the stale address instead of accumulating
|
|
355
|
+
* dead ones. Entries are never evicted: they are the node's only way back into
|
|
356
|
+
* the party if it is ever stranded again.
|
|
357
|
+
*/
|
|
358
|
+
private bootstrapPeerStore: BootstrapPeerStore | null = null;
|
|
359
|
+
|
|
168
360
|
/** Initial self-registration timer (see {@link scheduleSelfRegistration}). */
|
|
169
361
|
private selfRegistrationTimer: ReturnType<typeof setTimeout> | null = null;
|
|
170
362
|
/** TTL heartbeat that re-publishes the self record before it goes stale. */
|
|
@@ -187,7 +379,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
187
379
|
private reconcileControlCohortInFlight: Promise<void> | null = null;
|
|
188
380
|
/**
|
|
189
381
|
* Single-flight guard for {@link registerSelf}. Concurrent callers (the explicit
|
|
190
|
-
* CLI `--
|
|
382
|
+
* CLI `--owner` publish, the 1s startup timer, the TTL heartbeat, and the
|
|
191
383
|
* address-change listener) share one in-flight publish so two of them can never
|
|
192
384
|
* both read "no row yet" and race a duplicate INSERT (a `CadrePeer` PK conflict).
|
|
193
385
|
*/
|
|
@@ -195,21 +387,44 @@ export class CadreNode implements SAppIdLookup {
|
|
|
195
387
|
|
|
196
388
|
// ── Write-while-alone re-replication (control-write-ensure-replicated) ──────
|
|
197
389
|
/**
|
|
198
|
-
* Re-replication queue for
|
|
199
|
-
* node was alone — `controlNode.getConnections().length === 0` at write
|
|
200
|
-
* means the Optimystic commit was local-only (the block's cluster was ≤1)
|
|
201
|
-
* never broadcast. Maps the affected subject peerId →
|
|
202
|
-
* the cohort grows.
|
|
203
|
-
*
|
|
204
|
-
*
|
|
390
|
+
* Re-replication queue for owner `authorizePeer` writes that committed while
|
|
391
|
+
* this node was alone — `controlNode.getConnections().length === 0` at write
|
|
392
|
+
* time means the Optimystic commit was local-only (the block's cluster was ≤1)
|
|
393
|
+
* and never broadcast. Maps the affected subject peerId → 'authorize', to
|
|
394
|
+
* re-issue once the cohort grows. Removals are NOT tracked here — a delete
|
|
395
|
+
* leaves a `Revocation` tombstone, tracked by {@link pendingRevocations}; a
|
|
396
|
+
* removePeer also clears any queued authorize for the same subject (the row is
|
|
397
|
+
* gone, so re-issuing the insert would be wrong). Drained on the 0→≥1
|
|
398
|
+
* control-connection transition by {@link drainPendingControlReplication}.
|
|
205
399
|
*/
|
|
206
|
-
private pendingPeerWrites: Map<string, 'authorize'
|
|
400
|
+
private pendingPeerWrites: Map<string, 'authorize'> = new Map();
|
|
401
|
+
/**
|
|
402
|
+
* Re-replication queue for `Revocation` tombstones that committed while this
|
|
403
|
+
* node was alone, keyed on the retired `StampId`. Fed by the control DB's
|
|
404
|
+
* committed-delete seam ({@link noteGuardedDelete}), so one queue covers all
|
|
405
|
+
* four guarded tables (`CadrePeer` / `DeviceToken` / `Strand` /
|
|
406
|
+
* `ValidationKey`). Drained (owner-signed `ReissuedAt` bump, which re-broadcasts
|
|
407
|
+
* the tombstone) by {@link drainPendingRevocations}. Cleared on stop alongside
|
|
408
|
+
* {@link pendingPeerWrites} — the tombstone rows are durable, so the next
|
|
409
|
+
* lifetime's first-growth sweep re-covers anything dropped here.
|
|
410
|
+
*/
|
|
411
|
+
private pendingRevocations: Map<string, RevokedRowRef> = new Map();
|
|
412
|
+
/**
|
|
413
|
+
* Guards the once-per-lifetime first-growth revocation sweep (re-issue EVERY
|
|
414
|
+
* locally-held tombstone, covering removals from before this node started).
|
|
415
|
+
* Deliberately separate from {@link reconstructedLocalOnlyWrites} and set only
|
|
416
|
+
* after a sweep pass SUCCEEDS (including the nothing-held case) — a throwing
|
|
417
|
+
* sweep retries on the next growth edge instead of being lost for the lifetime.
|
|
418
|
+
* Reset on stop, so a stop()→start() cycle sweeps again (that second start is
|
|
419
|
+
* exactly a "removals from before this lifetime" case).
|
|
420
|
+
*/
|
|
421
|
+
private reissuedHeldRevocations = false;
|
|
207
422
|
/** This node's own `CadrePeer` self-write committed local-only (re-touched on growth). */
|
|
208
423
|
private pendingSelfPeerWrite = false;
|
|
209
424
|
/** This node's own `DeviceToken` self-write committed local-only (re-touched on growth). */
|
|
210
425
|
private pendingSelfDeviceWrite = false;
|
|
211
426
|
/**
|
|
212
|
-
* Guards the one-shot, first-cohort-growth reconstruction: an
|
|
427
|
+
* Guards the one-shot, first-cohort-growth reconstruction: an owner re-touches
|
|
213
428
|
* every membership row it may have authored that could be unreplicated (covering
|
|
214
429
|
* writes made before this process started, which the in-memory queue cannot know).
|
|
215
430
|
* Set true after the first drain so later passes only drain the in-memory queue.
|
|
@@ -380,6 +595,17 @@ export class CadreNode implements SAppIdLookup {
|
|
|
380
595
|
|
|
381
596
|
log('Starting CadreNode for party: %s', this.config.controlNetwork.partyId);
|
|
382
597
|
|
|
598
|
+
// NOTE: the only direct `console.*` in this library — a boot-time operator warning, not a
|
|
599
|
+
// diagnostic trace (those use `debug`, which an operator never sees without `DEBUG=`). If a
|
|
600
|
+
// second such warning ever appears here, route both through a `CadreNodeEvents` entry the
|
|
601
|
+
// embedder surfaces instead of growing a console surface inside a library.
|
|
602
|
+
if (this.config.network?.announceAddrs && this.config.network.announceAddrs.length > 0) {
|
|
603
|
+
console.warn(
|
|
604
|
+
'network.announceAddrs is set but not yet supported (no upstream db-p2p option to apply it) ' +
|
|
605
|
+
'— this node will keep advertising its listen/relay addresses instead.'
|
|
606
|
+
);
|
|
607
|
+
}
|
|
608
|
+
|
|
383
609
|
try {
|
|
384
610
|
const tTotal = performance.now();
|
|
385
611
|
|
|
@@ -393,6 +619,18 @@ export class CadreNode implements SAppIdLookup {
|
|
|
393
619
|
// store fails closed before a node is created.
|
|
394
620
|
await this.resolveIdentityKey();
|
|
395
621
|
|
|
622
|
+
// Bring up the node-local trusted-owner anchor (and seed config pins)
|
|
623
|
+
// before any network bring-up: a mis-scoped injected store fails closed
|
|
624
|
+
// here, and out-of-band pins are anchored before the first seed/peer
|
|
625
|
+
// interaction could consult them.
|
|
626
|
+
await this.initializeTrustedOwnerStore();
|
|
627
|
+
|
|
628
|
+
// Bring up the node-local cold-start bootstrap-peer store alongside the
|
|
629
|
+
// anchor, and for the same two reasons: a mis-scoped injected store fails
|
|
630
|
+
// closed before any network bring-up, and the retained dial targets are
|
|
631
|
+
// loaded before the first reconcile pass could consult them.
|
|
632
|
+
this.initializeBootstrapPeerStore();
|
|
633
|
+
|
|
396
634
|
// Create the control network libp2p node
|
|
397
635
|
let t0 = performance.now();
|
|
398
636
|
this.controlNode = await this.createControlNode();
|
|
@@ -417,6 +655,16 @@ export class CadreNode implements SAppIdLookup {
|
|
|
417
655
|
timing('[start] controlDatabase.initialize: %dms', Math.round(performance.now() - t0));
|
|
418
656
|
log('Control database initialized');
|
|
419
657
|
|
|
658
|
+
// Attach the per-stream gate to the control DB's membership hub, so every
|
|
659
|
+
// committed `CadrePeer` write re-materializes the authorized snapshot on its
|
|
660
|
+
// own. Wired before anything can write. `_running` is still false for the
|
|
661
|
+
// rest of start() and the refresh early-returns while it is — a write in that
|
|
662
|
+
// window is covered by start's own pass below.
|
|
663
|
+
this.controlDatabase.setMembershipChangeListener((reason) => this.refreshMembershipGate(reason));
|
|
664
|
+
// Committed-delete seam → write-while-alone revocation queue. Same wiring
|
|
665
|
+
// window argument as the membership listener above.
|
|
666
|
+
this.controlDatabase.setGuardedDeleteListener((revocation) => this.noteGuardedDelete(revocation));
|
|
667
|
+
|
|
420
668
|
// Create strand queryable using the control database
|
|
421
669
|
const queryable = this.createStrandQueryable();
|
|
422
670
|
|
|
@@ -440,10 +688,11 @@ export class CadreNode implements SAppIdLookup {
|
|
|
440
688
|
this.hibernationManager.start();
|
|
441
689
|
|
|
442
690
|
// Register the control-network push-wake receiver: a same-cadre peer can
|
|
443
|
-
// signal us to bring a hibernating strand online. Gated on
|
|
444
|
-
// membership; the wake routes through the same
|
|
691
|
+
// signal us to bring a hibernating strand online. Gated on AUTHORIZED
|
|
692
|
+
// membership (not the addressable surface); the wake routes through the same
|
|
693
|
+
// path as a local wake.
|
|
445
694
|
this.strandWakeService = new StrandWakeService({
|
|
446
|
-
isMember: (peerId) => this.
|
|
695
|
+
isMember: (peerId) => this.isAuthorizedMember(peerId),
|
|
447
696
|
getStrand: (strandId) => this.strandManager.getInstance(strandId),
|
|
448
697
|
wake: (strandId) => this.wakeStrand(strandId),
|
|
449
698
|
});
|
|
@@ -452,33 +701,23 @@ export class CadreNode implements SAppIdLookup {
|
|
|
452
701
|
// Register the control-network strand-address responder: a same-cadre peer
|
|
453
702
|
// resolving a strand's bootstrap seed asks us for our live strand-network
|
|
454
703
|
// multiaddrs (its CadrePeer row only knows our *control* address). Gated on
|
|
455
|
-
//
|
|
704
|
+
// AUTHORIZED membership; answers only for strands we are actively meshing.
|
|
705
|
+
// The same RPC doubles as the delegate-announce channel: a member's request
|
|
706
|
+
// may carry the derived peerId its strand node runs as, and we record an
|
|
707
|
+
// admission grant so our connection gate (and thus our relay server, when
|
|
708
|
+
// enabled) admits it — see delegate-admission.ts.
|
|
456
709
|
this.strandAddrService = new StrandAddrService({
|
|
457
|
-
isMember: (peerId) => this.
|
|
710
|
+
isMember: (peerId) => this.isAuthorizedMember(peerId),
|
|
458
711
|
getStrandMultiaddrs: (strandId) => this.getStrandMultiaddrs(strandId),
|
|
712
|
+
onDelegateAnnounce: (announcer, strandId, delegate) =>
|
|
713
|
+
this.grantDelegateAdmission(announcer, strandId, delegate),
|
|
459
714
|
});
|
|
460
715
|
this.strandAddrService.initialize(this.controlNode);
|
|
461
716
|
|
|
462
|
-
// Server-side push-wake fan-out: only when push
|
|
463
|
-
// The PushNotifier reaches for node:http2/node:crypto, so it is loaded via a
|
|
464
|
-
// guarded DYNAMIC import — a cross-platform (RN/browser) node that never sets
|
|
465
|
-
// config.push never pulls those modules into its static graph. The fan-out
|
|
466
|
-
// service itself is import-clean (PushNotifier type only) and statically imported.
|
|
717
|
+
// Server-side push-wake fan-out: only when push is configured.
|
|
467
718
|
if (this.config.push) {
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
this.pushFanoutService = new PushFanoutService({
|
|
471
|
-
listMembers: () => this.listMembers(),
|
|
472
|
-
getStrand: (strandId) => this.strandManager.getInstance(strandId),
|
|
473
|
-
selfPeerId: () => this.controlNode?.peerId.toString(),
|
|
474
|
-
pushWake: (peerId, strandId, reason) => this.pushWake(peerId, strandId, reason),
|
|
475
|
-
resolveDeviceToken: (peerId) => this.resolveDeviceToken(peerId),
|
|
476
|
-
expireDeviceToken: (peerId) => this.expireDeviceToken(peerId),
|
|
477
|
-
notifier,
|
|
478
|
-
cooldownMs: this.config.push.cooldownMs,
|
|
479
|
-
debounceMs: this.config.push.debounceMs,
|
|
480
|
-
});
|
|
481
|
-
log('Push-wake fan-out enabled (FCM=%s, APNs=%s)', !!this.config.push.fcm, !!this.config.push.apns);
|
|
719
|
+
this.pushFanoutService = this.buildPushFanout(this.config.push);
|
|
720
|
+
log('Push-wake fan-out enabled (injected notifier)');
|
|
482
721
|
}
|
|
483
722
|
|
|
484
723
|
this._running = true;
|
|
@@ -491,6 +730,12 @@ export class CadreNode implements SAppIdLookup {
|
|
|
491
730
|
// write-while-alone re-replication drain must fire on the first 0→≥1 edge.
|
|
492
731
|
this.wireControlConnectionListeners();
|
|
493
732
|
|
|
733
|
+
// Seed the per-stream gate's materialized authorized set now that the
|
|
734
|
+
// control DB is up (non-blocking; the helper never rejects). Until it
|
|
735
|
+
// lands the snapshot is empty, which the stream gate treats as cold
|
|
736
|
+
// start and admits.
|
|
737
|
+
void this.refreshMembershipGate('start');
|
|
738
|
+
|
|
494
739
|
// Schedule self-registration in background
|
|
495
740
|
this.scheduleSelfRegistration();
|
|
496
741
|
|
|
@@ -501,6 +746,32 @@ export class CadreNode implements SAppIdLookup {
|
|
|
501
746
|
}
|
|
502
747
|
}
|
|
503
748
|
|
|
749
|
+
/**
|
|
750
|
+
* Build the server-side push-wake fan-out over an injected {@link PushNotifier}.
|
|
751
|
+
*
|
|
752
|
+
* The notifier reaches for `node:http2`/`node:crypto`, so the cross-platform
|
|
753
|
+
* core never constructs it — the Node host builds it from
|
|
754
|
+
* `@serfab/cadre-core/push-node` and passes the instance in
|
|
755
|
+
* `CadreNodeConfig.push`. This node owns that instance's lifecycle from here:
|
|
756
|
+
* {@link cleanup} closes the fan-out, which closes the notifier (freeing the
|
|
757
|
+
* APNs HTTP/2 session). Every other primitive the fan-out needs (member
|
|
758
|
+
* enumeration, participation, direct dial, token resolve/expire) is an
|
|
759
|
+
* import-clean closure over this node.
|
|
760
|
+
*/
|
|
761
|
+
private buildPushFanout(push: NonNullable<CadreNodeConfig['push']>): PushFanoutService {
|
|
762
|
+
return new PushFanoutService({
|
|
763
|
+
listMembers: () => this.listMembers(),
|
|
764
|
+
getStrand: (strandId) => this.strandManager.getInstance(strandId),
|
|
765
|
+
selfPeerId: () => this.controlNode?.peerId.toString(),
|
|
766
|
+
pushWake: (peerId, strandId, reason) => this.pushWake(peerId, strandId, reason),
|
|
767
|
+
resolveDeviceToken: (peerId) => this.resolveDeviceToken(peerId),
|
|
768
|
+
expireDeviceToken: (peerId) => this.expireDeviceToken(peerId),
|
|
769
|
+
notifier: push.notifier,
|
|
770
|
+
cooldownMs: push.cooldownMs,
|
|
771
|
+
debounceMs: push.debounceMs,
|
|
772
|
+
});
|
|
773
|
+
}
|
|
774
|
+
|
|
504
775
|
/**
|
|
505
776
|
* Stop the cadre node
|
|
506
777
|
*/
|
|
@@ -511,7 +782,18 @@ export class CadreNode implements SAppIdLookup {
|
|
|
511
782
|
}
|
|
512
783
|
|
|
513
784
|
log('Stopping CadreNode');
|
|
785
|
+
// Stop the retry loop BEFORE tearing the node down, not after: `cleanup()`
|
|
786
|
+
// stops the control node partway through, and a tick that fires during it
|
|
787
|
+
// would dial and then poll a half-torn-down node for the whole reserve
|
|
788
|
+
// timeout — logging spurious dial failures and holding the process open.
|
|
789
|
+
this.relayReserveSupervisor?.stop();
|
|
790
|
+
this.relayReserveSupervisor = null;
|
|
514
791
|
await this.cleanup();
|
|
792
|
+
// Drop the rest of the relay-reservation posture: a stopped node holds no
|
|
793
|
+
// reservation, so a restarted instance must not report the previous run's
|
|
794
|
+
// `reserved`.
|
|
795
|
+
this.relayReserveAddrs = [];
|
|
796
|
+
this.relayReserveError = null;
|
|
515
797
|
this._running = false;
|
|
516
798
|
this.emit('control:disconnected', undefined);
|
|
517
799
|
log('CadreNode stopped');
|
|
@@ -556,11 +838,11 @@ export class CadreNode implements SAppIdLookup {
|
|
|
556
838
|
* before any network bring-up. Resolution order:
|
|
557
839
|
*
|
|
558
840
|
* 1. Both `keyStore` and `privateKey` set ⇒ configuration error (throws).
|
|
559
|
-
* 2. `keyStore` set ⇒
|
|
560
|
-
* {@link DEFAULT_IDENTITY_KEY_ID})
|
|
561
|
-
*
|
|
562
|
-
*
|
|
563
|
-
*
|
|
841
|
+
* 2. `keyStore` set ⇒ {@link loadOrCreateIdentityKey} against `identityKeyId`
|
|
842
|
+
* (default {@link DEFAULT_IDENTITY_KEY_ID}): load when the slot is present,
|
|
843
|
+
* generate + persist when it is empty. A rejected `get` (access denied /
|
|
844
|
+
* backend failure) PROPAGATES — we never generate a new key on a read error,
|
|
845
|
+
* which would silently orphan the real identity.
|
|
564
846
|
* 3. `privateKey` set ⇒ use it directly.
|
|
565
847
|
* 4. Neither ⇒ leave undefined; libp2p generates an ephemeral key.
|
|
566
848
|
*
|
|
@@ -581,21 +863,11 @@ export class CadreNode implements SAppIdLookup {
|
|
|
581
863
|
}
|
|
582
864
|
|
|
583
865
|
if (keyStore) {
|
|
584
|
-
|
|
585
|
-
//
|
|
586
|
-
//
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
// Corrupt/garbage bytes throw here; surface loudly rather than
|
|
590
|
-
// regenerating (which would orphan the real identity).
|
|
591
|
-
this.identityKey = privateKeyFromProtobuf(bytes);
|
|
592
|
-
log('Identity key loaded from key store (slot present)');
|
|
593
|
-
return;
|
|
594
|
-
}
|
|
595
|
-
const generated = await generateKeyPair('Ed25519');
|
|
596
|
-
await keyStore.set(keyId, privateKeyToProtobuf(generated));
|
|
597
|
-
this.identityKey = generated;
|
|
598
|
-
log('Identity key generated and persisted to key store (first run)');
|
|
866
|
+
// Shared with the embedding app: `reference-app-rn` resolves the very same
|
|
867
|
+
// key before constructing this node so it can sign its ICE-manifest request
|
|
868
|
+
// with it (identity-key.ts). One copy of the rule, so neither side can
|
|
869
|
+
// drift into generating a second identity.
|
|
870
|
+
this.identityKey = await loadOrCreateIdentityKey(keyStore, identityKeyId ?? DEFAULT_IDENTITY_KEY_ID);
|
|
599
871
|
return;
|
|
600
872
|
}
|
|
601
873
|
|
|
@@ -606,9 +878,129 @@ export class CadreNode implements SAppIdLookup {
|
|
|
606
878
|
// Neither configured: libp2p generates an ephemeral key internally.
|
|
607
879
|
}
|
|
608
880
|
|
|
881
|
+
/**
|
|
882
|
+
* Construct (or adopt) the node-local trusted-owner anchor and seed the
|
|
883
|
+
* out-of-band pinned keys from `config.trustedOwners`. The store is NEVER
|
|
884
|
+
* sourced from the replicated control DB — its entries come only from
|
|
885
|
+
* genesis self-trust ({@link initializeSeedBootstrap}), config pins (here),
|
|
886
|
+
* or runtime enrollment pins ({@link trustOwnerKeys}).
|
|
887
|
+
*
|
|
888
|
+
* Idempotent across stop()→start(): the store instance is kept, and
|
|
889
|
+
* re-seeding config pins is a no-op ({@link TrustedOwnerStore.trust} is
|
|
890
|
+
* idempotent). An injected store scoped to a different party is a
|
|
891
|
+
* configuration error (fail closed before any network bring-up).
|
|
892
|
+
*/
|
|
893
|
+
private async initializeTrustedOwnerStore(): Promise<void> {
|
|
894
|
+
const { trustedOwners } = this.config;
|
|
895
|
+
const partyId = this.config.controlNetwork.partyId;
|
|
896
|
+
if (!this.trustedOwnerStore) {
|
|
897
|
+
const store = trustedOwners?.store ?? new MemoryTrustedOwnerStore(partyId);
|
|
898
|
+
if (store.partyId !== partyId) {
|
|
899
|
+
throw new Error(
|
|
900
|
+
`CadreNodeConfig: trustedOwners.store is scoped to party ${store.partyId}, ` +
|
|
901
|
+
`but this node serves party ${partyId} — refusing to mix trust anchors`
|
|
902
|
+
);
|
|
903
|
+
}
|
|
904
|
+
this.trustedOwnerStore = store;
|
|
905
|
+
}
|
|
906
|
+
// Validate every pin before trusting any: a malformed entry must not leave
|
|
907
|
+
// earlier, valid pins anchored while start() then fails on a later one —
|
|
908
|
+
// config pins land all-or-nothing, same as the runtime seam below.
|
|
909
|
+
const pinnedKeys = (trustedOwners?.pinnedKeys ?? []).map(key => requireEd25519PublicKeyB64(key, 'pinned owner key'));
|
|
910
|
+
for (const key of pinnedKeys) {
|
|
911
|
+
await this.trustedOwnerStore.trust(key, trustedOwners?.pinnedSource ?? 'operator');
|
|
912
|
+
}
|
|
913
|
+
}
|
|
914
|
+
|
|
915
|
+
/**
|
|
916
|
+
* Construct (or adopt) the node-local cold-start bootstrap-peer store (see
|
|
917
|
+
* {@link bootstrapPeerStore}). Synchronous: an injected store has already
|
|
918
|
+
* loaded its persisted targets by the time it is handed in (its `open` is the
|
|
919
|
+
* async part), and the in-memory fallback has nothing to load.
|
|
920
|
+
*
|
|
921
|
+
* Idempotent across stop()→start(): the store instance is kept, so retained
|
|
922
|
+
* targets survive a restart of the same node instance even with the ephemeral
|
|
923
|
+
* default. An injected store scoped to a different party is a configuration
|
|
924
|
+
* error (fail closed before any network bring-up) — a foreign party's addresses
|
|
925
|
+
* must never enter this node's dial loop.
|
|
926
|
+
*/
|
|
927
|
+
private initializeBootstrapPeerStore(): void {
|
|
928
|
+
const partyId = this.config.controlNetwork.partyId;
|
|
929
|
+
if (this.bootstrapPeerStore) {
|
|
930
|
+
return;
|
|
931
|
+
}
|
|
932
|
+
const store = this.config.bootstrapPeers?.store ?? new MemoryBootstrapPeerStore(partyId);
|
|
933
|
+
if (store.partyId !== partyId) {
|
|
934
|
+
throw new Error(
|
|
935
|
+
`CadreNodeConfig: bootstrapPeers.store is scoped to party ${store.partyId}, ` +
|
|
936
|
+
`but this node serves party ${partyId} — refusing to mix cold-start dial targets`
|
|
937
|
+
);
|
|
938
|
+
}
|
|
939
|
+
this.bootstrapPeerStore = store;
|
|
940
|
+
}
|
|
941
|
+
|
|
942
|
+
/**
|
|
943
|
+
* The node-local cold-start bootstrap-peer store (null before {@link start}) —
|
|
944
|
+
* the retained owner addresses {@link reconcileControlCohort}'s cold-start
|
|
945
|
+
* branch re-dials. Exposed for diagnostics and for a host that wants to show
|
|
946
|
+
* "what would this node dial if it is stranded?".
|
|
947
|
+
*/
|
|
948
|
+
getBootstrapPeerStore(): BootstrapPeerStore | null {
|
|
949
|
+
return this.bootstrapPeerStore;
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
/**
|
|
953
|
+
* The node-local trusted-owner anchor (null before {@link start}). This is
|
|
954
|
+
* the set the authorized-membership predicate and seed-trust anchor consult —
|
|
955
|
+
* never the replicated `OwnerKey` table, which any stranger can pollute.
|
|
956
|
+
*/
|
|
957
|
+
getTrustedOwnerStore(): TrustedOwnerStore | null {
|
|
958
|
+
return this.trustedOwnerStore;
|
|
959
|
+
}
|
|
960
|
+
|
|
961
|
+
/**
|
|
962
|
+
* Persist out-of-band-established owner keys into the node-local anchor —
|
|
963
|
+
* the runtime enrollment seam: call with `CadreInvite.ownerKeys` when
|
|
964
|
+
* redeeming an invite (BEFORE the first `applySeed`, so the anchor already
|
|
965
|
+
* holds the pins when seed trust consults it), or with an operator-supplied
|
|
966
|
+
* pin. Idempotent. ('genesis' provenance is reserved for the node's own
|
|
967
|
+
* founding key, seeded internally by {@link initializeSeedBootstrap}.)
|
|
968
|
+
*
|
|
969
|
+
* Validates every key's shape before trusting any of them (all-or-nothing):
|
|
970
|
+
* a malformed entry anywhere in `keys` rejects the whole call before a
|
|
971
|
+
* single key is anchored. For the invite route this means a `CadreInvite`
|
|
972
|
+
* carrying one malformed `ownerKeys` entry fails the redemption outright —
|
|
973
|
+
* consistent with the anchor's existing whole-record-or-nothing policy for
|
|
974
|
+
* a corrupt persisted entry (see `trusted-owner-store.ts`'s
|
|
975
|
+
* `unusableEntry: 'discard-all'`) — rather than silently anchoring a subset
|
|
976
|
+
* and leaving the caller to notice a key went missing.
|
|
977
|
+
*/
|
|
978
|
+
async trustOwnerKeys(keys: Iterable<string>, source: Exclude<TrustSource, 'genesis'>): Promise<void> {
|
|
979
|
+
if (!this.trustedOwnerStore) {
|
|
980
|
+
throw new Error('CadreNode must be started before trusting owner keys');
|
|
981
|
+
}
|
|
982
|
+
const validated = Array.from(keys, key => requireEd25519PublicKeyB64(key, 'pinned owner key'));
|
|
983
|
+
for (const key of validated) {
|
|
984
|
+
await this.trustedOwnerStore.trust(key, source);
|
|
985
|
+
}
|
|
986
|
+
}
|
|
987
|
+
|
|
609
988
|
private async createControlNode(): Promise<Libp2p> {
|
|
989
|
+
return await createLibp2pNode(this.buildControlNodeOptions());
|
|
990
|
+
}
|
|
991
|
+
|
|
992
|
+
/**
|
|
993
|
+
* Map this node's config onto the control network's libp2p node options.
|
|
994
|
+
*
|
|
995
|
+
* Split out of {@link createControlNode} purely so the mapping is assertable without
|
|
996
|
+
* standing up a real libp2p node — `packages/cadre-core/test/cadre-node-control-node-options.spec.ts`
|
|
997
|
+
* calls it on a bare `new CadreNode(config)`. Read nothing else into the split; the
|
|
998
|
+
* only caller in production is `createControlNode`.
|
|
999
|
+
*/
|
|
1000
|
+
private buildControlNodeOptions(): Parameters<typeof createLibp2pNode>[0] {
|
|
610
1001
|
const { controlNetwork, network, storage, profile } = this.config;
|
|
611
1002
|
const identityKey = this.identityKey;
|
|
1003
|
+
const listenAddrs = resolveListenAddrs(network);
|
|
612
1004
|
|
|
613
1005
|
// Determine relay mode: if explicitly set in config, use that;
|
|
614
1006
|
// otherwise default to true for storage profile nodes (better connectivity/uptime)
|
|
@@ -629,16 +1021,290 @@ export class CadreNode implements SAppIdLookup {
|
|
|
629
1021
|
storage: controlStorageProvider,
|
|
630
1022
|
fretProfile: profile === 'storage' ? 'core' : 'edge',
|
|
631
1023
|
relay: enableRelay,
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
1024
|
+
// Fixed, and deliberately above any party's node count: every member reads the
|
|
1025
|
+
// whole control database, so a cohort that excludes a member leaves it dependent
|
|
1026
|
+
// on read repair — which cannot converge at a two-member cohort. Not a knob;
|
|
1027
|
+
// see CONTROL_REPLICATION_BREADTH. `assumedClusterSize: 2` is declared explicitly in
|
|
1028
|
+
// CONTROL_CLUSTER_POLICY: only the admission gate defaults to 2, while the read-repair
|
|
1029
|
+
// corroboration floor would otherwise fall back to this clusterSize of 16.
|
|
1030
|
+
clusterSize: CONTROL_REPLICATION_BREADTH,
|
|
1031
|
+
clusterPolicy: CONTROL_CLUSTER_POLICY,
|
|
1032
|
+
arachnode: { enableRingZulu: profile === 'storage' },
|
|
635
1033
|
...(identityKey && { privateKey: identityKey }),
|
|
636
1034
|
...(network?.transports && { transports: network.transports }),
|
|
637
|
-
|
|
638
|
-
|
|
1035
|
+
// Configured `listenAddrs`, plus a `/p2p-circuit` listener for every
|
|
1036
|
+
// `network.relayAddrs` entry — that listener is what reserves the relay slot.
|
|
1037
|
+
...(listenAddrs && { listenAddrs }),
|
|
1038
|
+
// The CONTROL node composes the membership admission gate onto any
|
|
1039
|
+
// caller-supplied gater (deny from either wins on inbound; all other
|
|
1040
|
+
// hooks pass through). Strand cohort nodes keep the raw configured gater
|
|
1041
|
+
// (see strand-instance-manager.ts) — their peers are legitimately
|
|
1042
|
+
// cross-party, so cadre membership must not gate them.
|
|
1043
|
+
connectionGater: createMembershipConnectionGater(
|
|
1044
|
+
{ admitInbound: (remotePeerId) => this.admitInboundControlConnection(remotePeerId) },
|
|
1045
|
+
network?.connectionGater
|
|
1046
|
+
),
|
|
1047
|
+
// Fail-closed per-stream authorization for the four Optimystic control-DB
|
|
1048
|
+
// protocols — the members-only layer the connection gater's stranger
|
|
1049
|
+
// carve-outs cannot express (see authorizeInboundControlStream). Control
|
|
1050
|
+
// node only: strand cohort nodes serve cross-party peers.
|
|
1051
|
+
authorizeInboundStream: (remotePeerId, protocol) =>
|
|
1052
|
+
this.authorizeInboundControlStream(remotePeerId, protocol)
|
|
639
1053
|
};
|
|
640
1054
|
|
|
641
|
-
return
|
|
1055
|
+
return nodeOptions;
|
|
1056
|
+
}
|
|
1057
|
+
|
|
1058
|
+
/**
|
|
1059
|
+
* Decide whether an inbound CONTROL-network connection from `remotePeerId`
|
|
1060
|
+
* should be admitted — the policy behind the connection gater wired in
|
|
1061
|
+
* {@link createControlNode}. Deny only on a positive "unauthorized outsider
|
|
1062
|
+
* while no stranger path is open" determination; everything ambiguous admits
|
|
1063
|
+
* and defers to the fail-closed per-stream gates (see
|
|
1064
|
+
* `membership-connection-gater.ts` for the layer's rationale and the
|
|
1065
|
+
* stranger-open protocol allowlist).
|
|
1066
|
+
*
|
|
1067
|
+
* Admits when ANY of:
|
|
1068
|
+
* 1. a shared-baseline check admits ({@link admitControlPeerUnconditionally}
|
|
1069
|
+
* — not running / DB torn down, absent-or-empty trusted-owner anchor, or
|
|
1070
|
+
* configured bootstrap/relay infrastructure);
|
|
1071
|
+
* 2. an enrollment window is open ({@link openEnrollmentWindow}, opened by
|
|
1072
|
+
* {@link createInvite}) — the invitee dials in before it is authorized;
|
|
1073
|
+
* 3. the peer holds a live DELEGATE ADMISSION GRANT — an authorized member
|
|
1074
|
+
* announced it (over the strand-addr RPC) as the transport peerId of its
|
|
1075
|
+
* own strand node, so a NAT'd member's strand node can hold a
|
|
1076
|
+
* circuit-relay reservation here (see `delegate-admission.ts`).
|
|
1077
|
+
* Connection only: the per-stream gate below never honors a grant;
|
|
1078
|
+
* 4. the authorized-member set is empty — cold start: the rows that would
|
|
1079
|
+
* authorize anyone arrive by replication over these very connections;
|
|
1080
|
+
* 5. the peer IS an authorized member; or
|
|
1081
|
+
* 6. an open invitation is OUTSTANDING — at least one unexpired,
|
|
1082
|
+
* not-fully-consumed invitation this node minted or persisted (see
|
|
1083
|
+
* `StrandSolicitationService.hasOutstandingInvitation`). A formation
|
|
1084
|
+
* initiator is another party's peer by design and its token is only
|
|
1085
|
+
* checkable inside the protocol, so the gate asks the coarser question
|
|
1086
|
+
* "does this node expect a stranger at all?". Merely REGISTERING the
|
|
1087
|
+
* responder ({@link initializeStrandSolicitation}) no longer suspends
|
|
1088
|
+
* stranger denial — eager registration (as `reference-app-rn` does at
|
|
1089
|
+
* bring-up) and `formStrand`'s lazy initialization both leave the gate
|
|
1090
|
+
* armed, because neither mints an invitation.
|
|
1091
|
+
*
|
|
1092
|
+
* Ordering is semantically free (the checks are OR'd) but decides who pays:
|
|
1093
|
+
* checks 1-3 are in-memory, 4/5 share one control-DB read, and only a peer
|
|
1094
|
+
* already on the deny path reaches check 6's invitation lookup.
|
|
1095
|
+
*
|
|
1096
|
+
* Caveats of check 6, both self-healing:
|
|
1097
|
+
* - the in-memory mint registry dies with the process, so after a restart
|
|
1098
|
+
* only invitations persisted as `FormationInvite` rows still hold the
|
|
1099
|
+
* exemption open (re-mint otherwise — same story as the enrollment window);
|
|
1100
|
+
* - a peer holding a token whose `FormationInvite` row has not replicated to
|
|
1101
|
+
* this node yet is denied even though the formation handler would have
|
|
1102
|
+
* accepted it, exactly like the unreplicated-membership-row case below.
|
|
1103
|
+
*
|
|
1104
|
+
* NOTE: check 4/5 runs a control-DB read per inbound connection
|
|
1105
|
+
* (`listAuthorizedMembers`); connections are rare and cadres small, and this
|
|
1106
|
+
* layer is fail-open behind `ADMISSION_DECISION_TIMEOUT_MS`, so the live
|
|
1107
|
+
* read is safe here — unlike the per-stream gate, which must consult the
|
|
1108
|
+
* materialized {@link authorizedControlPeers} snapshot instead.
|
|
1109
|
+
* NOTE: a sibling whose membership row has not yet replicated to
|
|
1110
|
+
* this node is denied until the row converges (typically via the owner);
|
|
1111
|
+
* either side's next outbound reconcile dial (outbound is never gated)
|
|
1112
|
+
* re-establishes the link — self-healing, but visible as a transient deny.
|
|
1113
|
+
*/
|
|
1114
|
+
private async admitInboundControlConnection(remotePeerId: string): Promise<boolean> {
|
|
1115
|
+
if (this.admitControlPeerUnconditionally(remotePeerId)) {
|
|
1116
|
+
return true;
|
|
1117
|
+
}
|
|
1118
|
+
if (Date.now() < this.enrollmentWindowUntil) {
|
|
1119
|
+
return true;
|
|
1120
|
+
}
|
|
1121
|
+
if (this.delegateAdmission.has(remotePeerId)) {
|
|
1122
|
+
return true;
|
|
1123
|
+
}
|
|
1124
|
+
const authorized = await this.listAuthorizedMembers();
|
|
1125
|
+
if (authorized.length === 0) {
|
|
1126
|
+
return true;
|
|
1127
|
+
}
|
|
1128
|
+
if (authorized.some((m) => m.peerId === remotePeerId)) {
|
|
1129
|
+
return true;
|
|
1130
|
+
}
|
|
1131
|
+
try {
|
|
1132
|
+
if (await this.strandSolicitationService?.hasOutstandingInvitation()) {
|
|
1133
|
+
return true;
|
|
1134
|
+
}
|
|
1135
|
+
} catch (error) {
|
|
1136
|
+
log('admitInboundControlConnection: outstanding-invitation check threw for %s — admitting (fail-open): %o', remotePeerId, error);
|
|
1137
|
+
return true;
|
|
1138
|
+
}
|
|
1139
|
+
log('admitInboundControlConnection: DENYING inbound from %s — not an authorized member and no enrollment path open', remotePeerId);
|
|
1140
|
+
return false;
|
|
1141
|
+
}
|
|
1142
|
+
|
|
1143
|
+
/**
|
|
1144
|
+
* Hold the control-network inbound gate open for not-yet-authorized peers
|
|
1145
|
+
* until `untilEpochMs` (extends, never shrinks, an already-open window).
|
|
1146
|
+
* {@link createInvite} calls this automatically; a host running an
|
|
1147
|
+
* out-of-band enrollment flow (e.g. accepting a phone whose invite this node
|
|
1148
|
+
* never minted) can open it explicitly before the stranger dials in.
|
|
1149
|
+
*/
|
|
1150
|
+
openEnrollmentWindow(untilEpochMs: number): void {
|
|
1151
|
+
this.enrollmentWindowUntil = Math.max(this.enrollmentWindowUntil, untilEpochMs);
|
|
1152
|
+
log('Enrollment window open until %d', this.enrollmentWindowUntil);
|
|
1153
|
+
}
|
|
1154
|
+
|
|
1155
|
+
/**
|
|
1156
|
+
* Record (or refresh) a delegate admission grant: `delegatePeerId` is the
|
|
1157
|
+
* transport peerId of `announcerPeerId`'s strand-`strandId` node, admitted
|
|
1158
|
+
* at the CONNECTION level (all a circuit-relay reservation needs) for
|
|
1159
|
+
* `DELEGATE_GRANT_TTL_MS`. Called by the strand-addr responder after its
|
|
1160
|
+
* authorized-membership gate has passed; public so tests can drive the
|
|
1161
|
+
* admission policy without a full strand launch. A re-announce for the same
|
|
1162
|
+
* (announcer, strand) REPLACES the previous delegate rather than
|
|
1163
|
+
* accumulating. Never honored by {@link authorizeInboundControlStream}.
|
|
1164
|
+
*/
|
|
1165
|
+
grantDelegateAdmission(announcerPeerId: string, strandId: string, delegatePeerId: string): void {
|
|
1166
|
+
this.delegateAdmission.grant(announcerPeerId, strandId, delegatePeerId);
|
|
1167
|
+
}
|
|
1168
|
+
|
|
1169
|
+
/** Is `remotePeerId` covered by a live delegate admission grant? */
|
|
1170
|
+
hasDelegateAdmission(remotePeerId: string): boolean {
|
|
1171
|
+
return this.delegateAdmission.has(remotePeerId);
|
|
1172
|
+
}
|
|
1173
|
+
|
|
1174
|
+
/**
|
|
1175
|
+
* The admission checks SHARED by the connection gate
|
|
1176
|
+
* ({@link admitInboundControlConnection}) and the per-stream control-DB gate
|
|
1177
|
+
* ({@link authorizeInboundControlStream}) — factored so the two layers cannot
|
|
1178
|
+
* drift. All in-memory and cheap (the stream gate runs this per inbound
|
|
1179
|
+
* stream). Returns true when the peer must be admitted BEFORE any membership
|
|
1180
|
+
* source is consulted:
|
|
1181
|
+
* - the node is not fully up (`start()` in progress / DB torn down) — both
|
|
1182
|
+
* gates exist from libp2p bring-up, before the control DB does;
|
|
1183
|
+
* - the trusted-owner anchor is absent or empty — an un-enrolled node has
|
|
1184
|
+
* no basis to judge anyone and MUST accept its enrollment seed;
|
|
1185
|
+
* - the peer is one of the configured control bootstrap/relay nodes —
|
|
1186
|
+
* operator-configured infrastructure, not cadre members.
|
|
1187
|
+
* False only means "no unconditional admit": the caller then judges the peer
|
|
1188
|
+
* against its own membership source (a live DB read for the fail-open
|
|
1189
|
+
* connection gate; the materialized snapshot for the fail-closed stream gate).
|
|
1190
|
+
*/
|
|
1191
|
+
private admitControlPeerUnconditionally(remotePeerId: string): boolean {
|
|
1192
|
+
if (!this._running || !this.controlDatabase) {
|
|
1193
|
+
return true;
|
|
1194
|
+
}
|
|
1195
|
+
if (!this.trustedOwnerStore || this.trustedOwnerStore.all().size === 0) {
|
|
1196
|
+
return true;
|
|
1197
|
+
}
|
|
1198
|
+
return this.getBootstrapPeerIds().has(remotePeerId);
|
|
1199
|
+
}
|
|
1200
|
+
|
|
1201
|
+
/**
|
|
1202
|
+
* Per-stream authorization for the four Optimystic control-DB protocols
|
|
1203
|
+
* (`/optimystic/control-<party>/{repo,cluster,sync,block-transfer}/…`),
|
|
1204
|
+
* wired as `authorizeInboundStream` in {@link createControlNode} — the
|
|
1205
|
+
* fail-closed layer behind the fail-open connection gater. Control node
|
|
1206
|
+
* ONLY: strand cohort nodes legitimately serve cross-party peers.
|
|
1207
|
+
*
|
|
1208
|
+
* The STRICT SUBSET of {@link admitInboundControlConnection}: the same
|
|
1209
|
+
* "no basis to judge" admissions (shared via
|
|
1210
|
+
* {@link admitControlPeerUnconditionally}), minus the stranger carve-outs
|
|
1211
|
+
* (enrollment window, outstanding open invitation, delegate admission
|
|
1212
|
+
* grant). The first two exist so a stranger can reach `/sereus/seed/1.0.0`
|
|
1213
|
+
* and `/sereus/formation/1.0.0` — neither is gated here, and admitting a
|
|
1214
|
+
* stranger to `repo` during an enrollment window is exactly the hole this
|
|
1215
|
+
* gate closes. A DELEGATE-admitted connection (a member's strand node
|
|
1216
|
+
* holding a circuit-relay reservation, see `delegate-admission.ts`) is
|
|
1217
|
+
* likewise exactly a case this gate must still refuse: the delegate gets
|
|
1218
|
+
* the connection and never the control DB.
|
|
1219
|
+
*
|
|
1220
|
+
* Admits when ANY of:
|
|
1221
|
+
* 1. a shared-baseline check admits (not running / no DB, empty anchor,
|
|
1222
|
+
* configured bootstrap infra);
|
|
1223
|
+
* 2. the materialized authorized set is empty — cold start, before the
|
|
1224
|
+
* rows that would authorize anyone have replicated in;
|
|
1225
|
+
* 3. the peer IS in the materialized authorized set.
|
|
1226
|
+
*
|
|
1227
|
+
* Deliberately SYNCHRONOUS and pure in-memory — see
|
|
1228
|
+
* {@link authorizedControlPeers} for why a control-DB read here would
|
|
1229
|
+
* deadlock into mutual denial. With a sync predicate the upstream deadline
|
|
1230
|
+
* (`authorizeInboundStreamTimeoutMs`) never trips, so it stays at its
|
|
1231
|
+
* default.
|
|
1232
|
+
*
|
|
1233
|
+
* NOTE: the snapshot keys on PEER ID, so a node with no persistent identity
|
|
1234
|
+
* key never lands a `CadrePeer` row ({@link registerSelf} skips) and its
|
|
1235
|
+
* siblings deny its control-DB streams once their own snapshot is non-empty —
|
|
1236
|
+
* owner status is a KEY, not a peer id, so being the owner does not exempt it.
|
|
1237
|
+
* Harmless today (a real deployment persists its identity; the ephemeral-owner
|
|
1238
|
+
* case appears only in tests, which configure the owner as bootstrap infra —
|
|
1239
|
+
* `push-wake-e2e.integration.ts` design note 3). If an ephemeral-identity node
|
|
1240
|
+
* ever becomes a supported deployment, this gate needs a key-based admission.
|
|
1241
|
+
*/
|
|
1242
|
+
private authorizeInboundControlStream(remotePeerId: string, protocol: string): boolean {
|
|
1243
|
+
if (this.admitControlPeerUnconditionally(remotePeerId)) {
|
|
1244
|
+
return true;
|
|
1245
|
+
}
|
|
1246
|
+
if (this.authorizedControlPeers.size === 0) {
|
|
1247
|
+
return true;
|
|
1248
|
+
}
|
|
1249
|
+
if (this.authorizedControlPeers.has(remotePeerId)) {
|
|
1250
|
+
return true;
|
|
1251
|
+
}
|
|
1252
|
+
// Upstream logs its own denial line; this one carries the WHY.
|
|
1253
|
+
log('authorizeInboundControlStream: DENYING %s on %s — not in the materialized authorized set (%d member(s))',
|
|
1254
|
+
remotePeerId, protocol, this.authorizedControlPeers.size);
|
|
1255
|
+
return false;
|
|
1256
|
+
}
|
|
1257
|
+
|
|
1258
|
+
/**
|
|
1259
|
+
* Refresh {@link authorizedControlPeers} from the control DB, best-effort: a
|
|
1260
|
+
* failed read keeps the previous snapshot (never clears it), so a transient
|
|
1261
|
+
* DB error can neither flip the stream gate's cold-start carve-out back open
|
|
1262
|
+
* nor drop a legitimate member mid-flight. Never rejects.
|
|
1263
|
+
*
|
|
1264
|
+
* The SOLE caller is {@link drainMembershipGate}, which serializes refreshes —
|
|
1265
|
+
* so two reads can no longer settle out of order.
|
|
1266
|
+
*/
|
|
1267
|
+
private async refreshAuthorizedControlPeers(reason: string): Promise<void> {
|
|
1268
|
+
if (!this._running || !this.controlDatabase) {
|
|
1269
|
+
return;
|
|
1270
|
+
}
|
|
1271
|
+
try {
|
|
1272
|
+
const members = await this.listAuthorizedMembers();
|
|
1273
|
+
this.authorizedControlPeers = new Set(members.map((m) => m.peerId));
|
|
1274
|
+
log('refreshAuthorizedControlPeers(%s): %d authorized peer(s)', reason, this.authorizedControlPeers.size);
|
|
1275
|
+
} catch (error) {
|
|
1276
|
+
log('refreshAuthorizedControlPeers(%s) failed — keeping previous snapshot: %o', reason, error);
|
|
1277
|
+
}
|
|
1278
|
+
}
|
|
1279
|
+
|
|
1280
|
+
/**
|
|
1281
|
+
* PeerIds of the configured control bootstrap nodes (relay/bootstrap
|
|
1282
|
+
* infrastructure — always admitted inbound). Parsed once, lazily; an
|
|
1283
|
+
* address without a `/p2p/<id>` component contributes nothing.
|
|
1284
|
+
*/
|
|
1285
|
+
private getBootstrapPeerIds(): Set<string> {
|
|
1286
|
+
if (!this.bootstrapPeerIds) {
|
|
1287
|
+
const ids = new Set<string>();
|
|
1288
|
+
for (const addr of this.config.controlNetwork.bootstrapNodes) {
|
|
1289
|
+
try {
|
|
1290
|
+
// Last `/p2p/<id>` component: on a plain bootstrap addr the node's own
|
|
1291
|
+
// id; on a circuit addr the dial target (which is what gets admitted).
|
|
1292
|
+
let id: string | undefined;
|
|
1293
|
+
for (const component of multiaddr(addr).getComponents()) {
|
|
1294
|
+
if (component.name === 'p2p' && component.value) {
|
|
1295
|
+
id = component.value;
|
|
1296
|
+
}
|
|
1297
|
+
}
|
|
1298
|
+
if (id) {
|
|
1299
|
+
ids.add(id);
|
|
1300
|
+
}
|
|
1301
|
+
} catch (error) {
|
|
1302
|
+
log('getBootstrapPeerIds: skipping unparsable bootstrap addr %s: %o', addr, error);
|
|
1303
|
+
}
|
|
1304
|
+
}
|
|
1305
|
+
this.bootstrapPeerIds = ids;
|
|
1306
|
+
}
|
|
1307
|
+
return this.bootstrapPeerIds;
|
|
642
1308
|
}
|
|
643
1309
|
|
|
644
1310
|
private createStrandQueryable(): StrandQueryable {
|
|
@@ -657,7 +1323,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
657
1323
|
/**
|
|
658
1324
|
* Schedule the node's initial self-record publish + ongoing refresh shortly
|
|
659
1325
|
* after start (non-blocking). {@link registerSelf} is idempotent and safely
|
|
660
|
-
* no-ops when it cannot yet sign/insert (e.g.
|
|
1326
|
+
* no-ops when it cannot yet sign/insert (e.g. owner key not installed),
|
|
661
1327
|
* so the timer is harmless even when registration only becomes possible later.
|
|
662
1328
|
*/
|
|
663
1329
|
private scheduleSelfRegistration(): void {
|
|
@@ -690,10 +1356,14 @@ export class CadreNode implements SAppIdLookup {
|
|
|
690
1356
|
* (signaling/`p2p-circuit` first), signed with the ed25519 key behind its
|
|
691
1357
|
* PeerId (the resolved node identity from `keyStore`/`config.privateKey`).
|
|
692
1358
|
* - If the row already exists: a self-signed UPDATE bumping `UpdatedAt`.
|
|
693
|
-
* - If not, and the node is its own
|
|
694
|
-
* also carries the self-signature.
|
|
695
|
-
*
|
|
696
|
-
*
|
|
1359
|
+
* - If not, and the node is its own owner: an owner-signed INSERT that
|
|
1360
|
+
* also carries the self-signature. That INSERT is idempotent, so if an owner
|
|
1361
|
+
* {@link authorizePeer} of this node's own id seated the row inside the
|
|
1362
|
+
* read-then-insert window it no-ops — the publish then falls through to the
|
|
1363
|
+
* UPDATE path (re-reading and re-signing against the row that landed) rather
|
|
1364
|
+
* than leaving the authorize's null `Sig` in place, and reports `refreshed`.
|
|
1365
|
+
* - Otherwise: logs and returns (a non-owner node with no row yet must
|
|
1366
|
+
* wait for an owner to insert it; it can then self-refresh).
|
|
697
1367
|
*
|
|
698
1368
|
* Safe to call repeatedly (heartbeat / address-change driven); each successful
|
|
699
1369
|
* publish strictly increases `UpdatedAt`. Concurrent calls are collapsed into a
|
|
@@ -735,29 +1405,67 @@ export class CadreNode implements SAppIdLookup {
|
|
|
735
1405
|
const addrs = await this.collectSelfAddrs();
|
|
736
1406
|
const existing = await this.controlDatabase.queryPeerRecord(peerId);
|
|
737
1407
|
|
|
738
|
-
|
|
1408
|
+
if (!existing) {
|
|
1409
|
+
if (!this.seedBootstrapService) {
|
|
1410
|
+
log('registerSelf: not yet a CadrePeer member and no owner service to self-insert; skipping (an owner must add this peer first)');
|
|
1411
|
+
return 'skipped';
|
|
1412
|
+
}
|
|
1413
|
+
// First-time row: requires an owner signature (the node is its own
|
|
1414
|
+
// owner). insertSelfPeerRecord throws if no owner key is present.
|
|
1415
|
+
const record = this.signSelfRecord(peerId, signingKey, addrs, null);
|
|
1416
|
+
if (await this.seedBootstrapService.insertSelfPeerRecord(record)) {
|
|
1417
|
+
if (this.committedAlone()) this.pendingSelfPeerWrite = true;
|
|
1418
|
+
log('registerSelf: inserted own CadrePeer record (owner-signed, updatedAt=%d, %d addrs, sig=%s…)', record.updatedAt, addrs.length, record.sig.slice(0, 16));
|
|
1419
|
+
return 'inserted';
|
|
1420
|
+
}
|
|
1421
|
+
// An owner authorize of this node's OWN peer id seated the row (null `Sig`)
|
|
1422
|
+
// inside the read-then-insert window, so the idempotent insert no-op'd. Fall
|
|
1423
|
+
// through to the self-update path, which re-signs against the row that landed.
|
|
1424
|
+
//
|
|
1425
|
+
// NOTE: this path reports 'refreshed' — honest about the write, which really was
|
|
1426
|
+
// an UPDATE, though the caller's intent was a first publish. No caller branches on
|
|
1427
|
+
// 'inserted' today (the CLI only logs it); if one ever needs "this was the row's
|
|
1428
|
+
// first publish", add a fourth SelfRegistrationOutcome rather than re-labelling.
|
|
1429
|
+
log('registerSelf: own CadrePeer row appeared mid-publish (concurrent authorize); self-updating to carry the signature');
|
|
1430
|
+
}
|
|
1431
|
+
|
|
1432
|
+
// Null only on the fall-through above, where the pre-race read missed the row a
|
|
1433
|
+
// concurrent authorize then seated — so re-read to get that row's `UpdatedAt`, which
|
|
1434
|
+
// the strictly-greater self-update rule is measured against.
|
|
1435
|
+
//
|
|
1436
|
+
// NOTE: when `existing` is set it predates this publish, so a removePeer(self) racing
|
|
1437
|
+
// in would leave the UPDATE below matching no rows while still reporting 'refreshed'.
|
|
1438
|
+
// Closing that needs updateSelfPeerRecord to report rows-affected — only worth doing
|
|
1439
|
+
// if removing self ever becomes something that happens concurrently in practice.
|
|
1440
|
+
const current = existing ?? await this.controlDatabase.queryPeerRecord(peerId);
|
|
1441
|
+
if (!current) {
|
|
1442
|
+
log('registerSelf: own CadrePeer row vanished mid-publish; skipping (next refresh re-inserts)');
|
|
1443
|
+
return 'skipped';
|
|
1444
|
+
}
|
|
1445
|
+
const record = this.signSelfRecord(peerId, signingKey, addrs, current);
|
|
1446
|
+
await this.controlDatabase.updateSelfPeerRecord(record);
|
|
1447
|
+
if (this.committedAlone()) this.pendingSelfPeerWrite = true;
|
|
1448
|
+
log('registerSelf: refreshed own CadrePeer record (updatedAt=%d, %d addrs, sig=%s…)', record.updatedAt, addrs.length, record.sig.slice(0, 16));
|
|
1449
|
+
return 'refreshed';
|
|
1450
|
+
}
|
|
1451
|
+
|
|
1452
|
+
/**
|
|
1453
|
+
* Sign this node's address record for publication, stamped strictly later than
|
|
1454
|
+
* `existing` (the row it is about to replace, or null for a first insert) so a
|
|
1455
|
+
* same-millisecond re-publish still satisfies the monotonic `UpdatedAt` rule the
|
|
1456
|
+
* `CadrePeer.AuthorizedUpdate` self-branch enforces.
|
|
1457
|
+
*/
|
|
1458
|
+
private signSelfRecord(
|
|
1459
|
+
peerId: string,
|
|
1460
|
+
signingKey: { privateKeyB64: string; publicKeyB64: string },
|
|
1461
|
+
addrs: string[],
|
|
1462
|
+
existing: PeerAddressRecord | null
|
|
1463
|
+
): PeerAddressRecord {
|
|
739
1464
|
const updatedAt = Math.max(Date.now(), (existing?.updatedAt ?? 0) + 1);
|
|
740
|
-
|
|
1465
|
+
return signPeerRecord(
|
|
741
1466
|
{ peerId, publicKey: signingKey.publicKeyB64, addrs, updatedAt },
|
|
742
1467
|
signingKey.privateKeyB64
|
|
743
1468
|
);
|
|
744
|
-
|
|
745
|
-
if (existing) {
|
|
746
|
-
await this.controlDatabase.updateSelfPeerRecord(record);
|
|
747
|
-
if (this.committedAlone()) this.pendingSelfPeerWrite = true;
|
|
748
|
-
log('registerSelf: refreshed own CadrePeer record (updatedAt=%d, %d addrs)', updatedAt, addrs.length);
|
|
749
|
-
return 'refreshed';
|
|
750
|
-
}
|
|
751
|
-
if (this.seedBootstrapService) {
|
|
752
|
-
// First-time row: requires an authority signature (the node is its own
|
|
753
|
-
// authority). insertSelfPeerRecord throws if no authority key is present.
|
|
754
|
-
await this.seedBootstrapService.insertSelfPeerRecord(record);
|
|
755
|
-
if (this.committedAlone()) this.pendingSelfPeerWrite = true;
|
|
756
|
-
log('registerSelf: inserted own CadrePeer record (authority-signed, updatedAt=%d, %d addrs)', updatedAt, addrs.length);
|
|
757
|
-
return 'inserted';
|
|
758
|
-
}
|
|
759
|
-
log('registerSelf: not yet a CadrePeer member and no authority service to self-insert; skipping (an authority must add this peer first)');
|
|
760
|
-
return 'skipped';
|
|
761
1469
|
}
|
|
762
1470
|
|
|
763
1471
|
/**
|
|
@@ -774,43 +1482,43 @@ export class CadreNode implements SAppIdLookup {
|
|
|
774
1482
|
return null;
|
|
775
1483
|
}
|
|
776
1484
|
try {
|
|
777
|
-
const { privateKeyB64, publicKeyB64 } =
|
|
1485
|
+
const { privateKeyB64, publicKeyB64 } = ed25519KeyPairFromLibp2p(this.identityKey);
|
|
778
1486
|
if (publicKeyB64 === ed25519PublicKeyB64FromPeerId(peerId)) {
|
|
779
1487
|
return { privateKeyB64, publicKeyB64 };
|
|
780
1488
|
}
|
|
781
1489
|
log('getSelfSigningKey: resolved identity key does not match control node peerId; cannot self-sign');
|
|
782
1490
|
} catch (error) {
|
|
783
|
-
// Logs the error shape only;
|
|
1491
|
+
// Logs the error shape only; ed25519KeyPairFromLibp2p never embeds key material.
|
|
784
1492
|
log('getSelfSigningKey: failed to derive ed25519 key from identity key: %o', error);
|
|
785
1493
|
}
|
|
786
1494
|
return null;
|
|
787
1495
|
}
|
|
788
1496
|
|
|
789
1497
|
/**
|
|
790
|
-
* The
|
|
791
|
-
* identity key. In the single-key reference model the
|
|
792
|
-
* *derived from* the node identity (see {@link
|
|
1498
|
+
* The owner keypair (base64url Ed25519) derived from this node's resolved
|
|
1499
|
+
* identity key. In the single-key reference model the owner signing key is
|
|
1500
|
+
* *derived from* the node identity (see {@link ed25519KeyPairFromLibp2p}), so the
|
|
793
1501
|
* same key material protected in a secure enclave backs both.
|
|
794
1502
|
*
|
|
795
|
-
* Exposed so the hosting app retains control of
|
|
1503
|
+
* Exposed so the hosting app retains control of owner genesis: cadre-core
|
|
796
1504
|
* resolves + protects the identity, then the app sources this pair to drive
|
|
797
|
-
* `
|
|
798
|
-
* never silently runs genesis. A future separate-
|
|
1505
|
+
* `ensureOwnerKey(pub)` + `initializeSeedBootstrap(priv)` itself — cadre-core
|
|
1506
|
+
* never silently runs genesis. A future separate-owner slot would return a
|
|
799
1507
|
* distinct key here instead of the identity-derived one.
|
|
800
1508
|
*
|
|
801
|
-
* @returns The base64url seed/public-key
|
|
1509
|
+
* @returns The base64url seed/public-key owner pair.
|
|
802
1510
|
* @throws If called before {@link start} has resolved the identity, or when the
|
|
803
1511
|
* node runs on an ephemeral libp2p key (no `keyStore`/`privateKey` configured),
|
|
804
1512
|
* since that key is internal to libp2p and not exposed.
|
|
805
1513
|
*/
|
|
806
|
-
|
|
1514
|
+
getIdentityOwnerKey(): Ed25519KeyPair {
|
|
807
1515
|
if (!this.identityKey) {
|
|
808
1516
|
throw new Error(
|
|
809
|
-
'
|
|
810
|
-
'configure `keyStore` or `privateKey` (an ephemeral libp2p identity exposes no
|
|
1517
|
+
'getIdentityOwnerKey: node identity not resolved — call start() first, and ' +
|
|
1518
|
+
'configure `keyStore` or `privateKey` (an ephemeral libp2p identity exposes no owner key)'
|
|
811
1519
|
);
|
|
812
1520
|
}
|
|
813
|
-
return
|
|
1521
|
+
return ed25519KeyPairFromLibp2p(this.identityKey);
|
|
814
1522
|
}
|
|
815
1523
|
|
|
816
1524
|
/**
|
|
@@ -903,13 +1611,17 @@ export class CadreNode implements SAppIdLookup {
|
|
|
903
1611
|
this.turnConnectionCloseHandler = null;
|
|
904
1612
|
|
|
905
1613
|
// Reset the write-while-alone re-replication state so a stop()→start() cycle
|
|
906
|
-
// re-arms the growth edge and re-runs
|
|
907
|
-
//
|
|
908
|
-
//
|
|
909
|
-
//
|
|
1614
|
+
// re-arms the growth edge and re-runs BOTH one-shot passes. Any queued
|
|
1615
|
+
// local-only writes are moot once the node is torn down: owner inserts are
|
|
1616
|
+
// re-covered by the next start's reconstruction, and delete tombstones by its
|
|
1617
|
+
// first-growth revocation sweep — which is why the sweep flag must reset too,
|
|
1618
|
+
// or the second lifetime would skip the sweep and strand exactly the
|
|
1619
|
+
// committed-alone tombstones it exists to carry.
|
|
910
1620
|
this.hasControlConnection = false;
|
|
911
1621
|
this.reconstructedLocalOnlyWrites = false;
|
|
1622
|
+
this.reissuedHeldRevocations = false;
|
|
912
1623
|
this.pendingPeerWrites.clear();
|
|
1624
|
+
this.pendingRevocations.clear();
|
|
913
1625
|
this.pendingSelfPeerWrite = false;
|
|
914
1626
|
this.pendingSelfDeviceWrite = false;
|
|
915
1627
|
}
|
|
@@ -951,7 +1663,12 @@ export class CadreNode implements SAppIdLookup {
|
|
|
951
1663
|
|
|
952
1664
|
// Self-signature over (peerId, addrs, updatedAt).
|
|
953
1665
|
if (!verifyPeerRecordSignature(record)) {
|
|
954
|
-
|
|
1666
|
+
// Field detail distinguishes "holding the owner-vouch revision (Sig null,
|
|
1667
|
+
// no addrs — a not-yet-self-published row)" from a genuinely corrupt or
|
|
1668
|
+
// mixed-revision row. Sig/addrs are public replicated data.
|
|
1669
|
+
log('resolvePeerAddrs: signature verification failed for %s (updatedAt=%d, addrs=%o, sig=%s)',
|
|
1670
|
+
peerId, record.updatedAt, record.addrs,
|
|
1671
|
+
record.sig ? `${record.sig.slice(0, 16)}…` : '(empty)');
|
|
955
1672
|
return [];
|
|
956
1673
|
}
|
|
957
1674
|
|
|
@@ -1045,16 +1762,81 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1045
1762
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
1046
1763
|
return;
|
|
1047
1764
|
}
|
|
1765
|
+
// Ride this pass's cadence to refresh the per-stream gate's materialized
|
|
1766
|
+
// authorized set — membership changes that ARRIVED BY REPLICATION (rather
|
|
1767
|
+
// than a local write) are picked up here, bounding the snapshot's staleness
|
|
1768
|
+
// to the reconcile interval.
|
|
1769
|
+
// NOTE: this refresh and the sibling enumeration below each run their own
|
|
1770
|
+
// CadrePeer query (two reads per pass); if those reads ever get costly,
|
|
1771
|
+
// share one row-set across both.
|
|
1772
|
+
await this.refreshMembershipGate('reconcile');
|
|
1773
|
+
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
1774
|
+
return;
|
|
1775
|
+
}
|
|
1776
|
+
// Refresh relay delegate grants BEFORE the sibling enumeration below: a
|
|
1777
|
+
// solo cadre with a party relay has no siblings to dial but must still keep
|
|
1778
|
+
// its running strands' grants alive (a dropped relay connection re-dials
|
|
1779
|
+
// the reservation and faces the connection gate again).
|
|
1780
|
+
await this.refreshDelegateGrants();
|
|
1781
|
+
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
1782
|
+
return;
|
|
1783
|
+
}
|
|
1048
1784
|
const selfPeerId = this.controlNode.peerId.toString();
|
|
1049
1785
|
|
|
1786
|
+
// Reap guarded rows this node still holds for incarnations an already-committed
|
|
1787
|
+
// Revocation tombstone retires — the half of a removal replication cannot carry
|
|
1788
|
+
// (a delete of an already-absent row replays as nothing). Connected-only: a reap
|
|
1789
|
+
// is a write, and a write committed alone is local-only and forks this node's own
|
|
1790
|
+
// history, which is the condition this work exists to stop creating
|
|
1791
|
+
// (tickets/blocked/forked-control-collection-sync-livelocks.md). Nothing is urgent
|
|
1792
|
+
// here — the stale row already reads as absent everywhere — so waiting for
|
|
1793
|
+
// connectivity costs nothing and keeps the reap fork-CLOSING rather than
|
|
1794
|
+
// fork-creating.
|
|
1795
|
+
//
|
|
1796
|
+
// BEFORE step 1, not after: the pass early-returns when it finds no siblings, and a
|
|
1797
|
+
// cadre whose only sibling row is the tombstoned one lands in exactly that branch
|
|
1798
|
+
// (listMembers() filters retired stamps, so the row needing the reap is invisible to
|
|
1799
|
+
// the sibling count). Placed after step 1 it would never run for the smallest and
|
|
1800
|
+
// most likely case.
|
|
1801
|
+
//
|
|
1802
|
+
// Shares no mutable state with drainPendingRevocations, which re-issues tombstones
|
|
1803
|
+
// while this consumes them: different tables (Revocation update vs guarded-row
|
|
1804
|
+
// delete) and both take the write lock per statement. The two can overlap freely —
|
|
1805
|
+
// do not add a mutex.
|
|
1806
|
+
//
|
|
1807
|
+
// NOTE: the gate is sampled ONCE for the whole sweep, so a disconnect landing
|
|
1808
|
+
// mid-sweep lets the remaining rows commit alone — the fork this gate exists to
|
|
1809
|
+
// avoid. Bounded by how long a sweep runs, which is bounded by the backlog of
|
|
1810
|
+
// unreaped rows (0 in steady state). If that backlog can ever be large, move the
|
|
1811
|
+
// check inside reapRevokedRows' loop via an injected predicate.
|
|
1812
|
+
if (this.getControlConnectionCount() > 0) {
|
|
1813
|
+
try {
|
|
1814
|
+
const reaped = await this.controlDatabase.reapRevokedRows(selfPeerId);
|
|
1815
|
+
if (reaped > 0) {
|
|
1816
|
+
log('reconcileControlCohort: reaped %d revoked control row(s)', reaped);
|
|
1817
|
+
}
|
|
1818
|
+
} catch (error) {
|
|
1819
|
+
// Best-effort like every other step in this pass: reconnecting siblings
|
|
1820
|
+
// outranks garbage collection, so a reap failure never aborts the reconcile.
|
|
1821
|
+
log('reconcileControlCohort: reap pass failed (continuing): %o', error);
|
|
1822
|
+
}
|
|
1823
|
+
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
1824
|
+
return;
|
|
1825
|
+
}
|
|
1826
|
+
}
|
|
1827
|
+
|
|
1050
1828
|
// 1. Enumerate known siblings. This membership read is itself a pull-on-read
|
|
1051
1829
|
// that helps the CadrePeer table converge — a reader-only node converges
|
|
1052
1830
|
// purely by these reads.
|
|
1053
1831
|
const members = await this.listMembers();
|
|
1054
1832
|
const siblings = members.filter((m) => m.peerId !== selfPeerId);
|
|
1055
1833
|
if (siblings.length === 0) {
|
|
1056
|
-
//
|
|
1057
|
-
// dial
|
|
1834
|
+
// No rows to dial from. Either a solo cadre (nothing to do) or a COLD
|
|
1835
|
+
// START whose seed dial never landed — and those look identical from here,
|
|
1836
|
+
// because filling this table needs a connection and getting a connection
|
|
1837
|
+
// needs this table. Fall back to the seed's bootstrap addresses; a solo
|
|
1838
|
+
// cadre has none and this is a no-op. Must not throw or busy-loop.
|
|
1839
|
+
await this.dialColdStartBootstrap();
|
|
1058
1840
|
return;
|
|
1059
1841
|
}
|
|
1060
1842
|
// Re-guard after the await: a stop() may have raced the membership read.
|
|
@@ -1062,18 +1844,23 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1062
1844
|
return;
|
|
1063
1845
|
}
|
|
1064
1846
|
|
|
1065
|
-
// 2. Classify backbone (
|
|
1066
|
-
|
|
1847
|
+
// 2. Classify backbone (owner) members and select a bounded dial set.
|
|
1848
|
+
// NOTE: deliberately the REPLICATED table, not the node-local anchor — this
|
|
1849
|
+
// only *prefers* owner peers as dial targets, so pollution costs a wasted
|
|
1850
|
+
// dial while anchoring would drop legitimate co-owners this node never
|
|
1851
|
+
// pinned. Same call as `SeedPeer.isOwner` in seed-bootstrap.ts. Move it to
|
|
1852
|
+
// the anchor if owner status here ever gates something trusted.
|
|
1853
|
+
const ownerKeys = await this.controlDatabase.getOwnerKeys();
|
|
1067
1854
|
if (!this._running || !this.controlNode) {
|
|
1068
1855
|
return;
|
|
1069
1856
|
}
|
|
1070
1857
|
const targetDegree = this.config.network?.controlCohort?.targetDegree
|
|
1071
1858
|
?? DEFAULT_CONTROL_COHORT_TARGET_DEGREE;
|
|
1072
|
-
const { dials,
|
|
1073
|
-
if (
|
|
1859
|
+
const { dials, cappedNonOwner } = selectControlCohortDials(siblings, ownerKeys, targetDegree);
|
|
1860
|
+
if (cappedNonOwner > 0) {
|
|
1074
1861
|
// Don't silently bound coverage — surface what the out-degree cap dropped.
|
|
1075
|
-
log('reconcileControlCohort: capped %d non-
|
|
1076
|
-
|
|
1862
|
+
log('reconcileControlCohort: capped %d non-owner sibling(s) at targetDegree=%d',
|
|
1863
|
+
cappedNonOwner, targetDegree);
|
|
1077
1864
|
}
|
|
1078
1865
|
|
|
1079
1866
|
// 3. Skip already-connected peers (no re-dial / churn for live connections).
|
|
@@ -1102,7 +1889,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1102
1889
|
*
|
|
1103
1890
|
* A per-peer failure (NAT, offline, relay down, connection-gater denial) is
|
|
1104
1891
|
* logged and swallowed so one unreachable sibling never aborts the pass —
|
|
1105
|
-
* exactly like {@link SeedBootstrapService.applySeed}'s
|
|
1892
|
+
* exactly like {@link SeedBootstrapService.applySeed}'s owner-dial loop. A
|
|
1106
1893
|
* failed dial is simply retried on the next pass.
|
|
1107
1894
|
*/
|
|
1108
1895
|
private async dialControlSibling(sibling: CohortPeerRow): Promise<boolean> {
|
|
@@ -1162,11 +1949,159 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1162
1949
|
}
|
|
1163
1950
|
}
|
|
1164
1951
|
|
|
1952
|
+
/**
|
|
1953
|
+
* Retain a just-applied seed's owner-flagged peers as cold-start bootstrap
|
|
1954
|
+
* dial targets (see {@link bootstrapPeerStore}).
|
|
1955
|
+
*
|
|
1956
|
+
* Called for every seed this node accepts, on BOTH intake paths — the
|
|
1957
|
+
* {@link applySeed} wrapper (which may run on a throwaway service, so the
|
|
1958
|
+
* service itself is the wrong place to keep this) and the inbound
|
|
1959
|
+
* `/sereus/seed/1.0.0` handler via `onSeedApplied`. Peers with no address are
|
|
1960
|
+
* skipped: there is nothing to dial.
|
|
1961
|
+
*
|
|
1962
|
+
* `isOwner` is the seed's own claim about its peers, exactly as
|
|
1963
|
+
* `SeedBootstrapService.applySeed` uses it to choose its dial targets. That is
|
|
1964
|
+
* sound here for the same reason it is sound there — the whole seed is
|
|
1965
|
+
* signature-checked against a trust-anchored signer before this runs, and the
|
|
1966
|
+
* flag only *selects a dial target*; a dial grants no authority.
|
|
1967
|
+
*
|
|
1968
|
+
* Self is excluded. `createSeed` projects EVERY `CadrePeer` row, so an owner
|
|
1969
|
+
* that applies a seed minted after it joined finds itself in the owner list;
|
|
1970
|
+
* retaining that would make the cold-start pass dial this node forever (the
|
|
1971
|
+
* steady-state pass filters self out of its sibling list for the same reason).
|
|
1972
|
+
*/
|
|
1973
|
+
private recordSeedBootstrapPeers(seed: ControlNetworkSeed): void {
|
|
1974
|
+
const store = this.bootstrapPeerStore;
|
|
1975
|
+
if (!store) {
|
|
1976
|
+
// Unreachable in production: start() builds the store before any network
|
|
1977
|
+
// bring-up, and both intake paths need a started node (an uninitialized
|
|
1978
|
+
// SeedBootstrapService rejects the seed, so noteAppliedSeed returns early).
|
|
1979
|
+
log('recordSeedBootstrapPeers: no bootstrap-peer store yet; retaining nothing');
|
|
1980
|
+
return;
|
|
1981
|
+
}
|
|
1982
|
+
const selfPeerId = this.controlNode?.peerId.toString();
|
|
1983
|
+
for (const peer of seed.peers) {
|
|
1984
|
+
if (!peer.isOwner || peer.multiaddrs.length === 0 || peer.peerId === selfPeerId) {
|
|
1985
|
+
continue;
|
|
1986
|
+
}
|
|
1987
|
+
// Sync-visible by contract; the promise tracks durability only. A persist
|
|
1988
|
+
// failure costs restart survival, never this session's retry set — the same
|
|
1989
|
+
// trade `SeedBootstrapService.anchorAcceptedSigner` makes — so log and carry
|
|
1990
|
+
// on rather than failing the seed that was already accepted.
|
|
1991
|
+
void store.record(peer.peerId, peer.multiaddrs).catch((error: unknown) => {
|
|
1992
|
+
log('recordSeedBootstrapPeers: persisting bootstrap peer %s failed (retained in memory): %o',
|
|
1993
|
+
peer.peerId, error);
|
|
1994
|
+
});
|
|
1995
|
+
}
|
|
1996
|
+
}
|
|
1997
|
+
|
|
1998
|
+
/**
|
|
1999
|
+
* Cold-start branch of the reconcile pass: re-dial the seed's owner peers
|
|
2000
|
+
* while the control database still has no siblings to dial.
|
|
2001
|
+
*
|
|
2002
|
+
* `SeedBootstrapService.applySeed` dials those owners exactly ONCE, best-effort.
|
|
2003
|
+
* When that single dial fails — owner momentarily down, relay reservation not
|
|
2004
|
+
* up yet, NAT traversal lost the race — the joining node has an empty
|
|
2005
|
+
* `CadrePeer` table and no connection, and the steady-state pass above cannot
|
|
2006
|
+
* help: it dials only siblings enumerated from that very table. Retrying here
|
|
2007
|
+
* is the only way back in.
|
|
2008
|
+
*
|
|
2009
|
+
* Unbounded, at the reconcile cadence, with no backoff: the branch is already
|
|
2010
|
+
* gated by "the control database has no siblings", so it stops the moment the
|
|
2011
|
+
* node is actually in the party, and a node that is stranded MUST keep trying —
|
|
2012
|
+
* a give-up rule would turn a transient outage into a permanent one. The steady
|
|
2013
|
+
* cost is one dial per bootstrap peer per pass (seeds nominate one or a few
|
|
2014
|
+
* owners), each failing fast against an unreachable address.
|
|
2015
|
+
* NOTE: if seeds ever carry many owner peers, or the reconcile cadence tightens
|
|
2016
|
+
* well below its 15 s default, add per-peer backoff here.
|
|
2017
|
+
*
|
|
2018
|
+
* Best-effort per peer, exactly like {@link dialControlSibling}: one dead
|
|
2019
|
+
* address never aborts the pass.
|
|
2020
|
+
*/
|
|
2021
|
+
private async dialColdStartBootstrap(): Promise<void> {
|
|
2022
|
+
const controlNode = this.controlNode;
|
|
2023
|
+
// Snapshot up front: the store hands back a copy, so a seed applied mid-pass
|
|
2024
|
+
// cannot mutate what we are iterating.
|
|
2025
|
+
const targets = this.bootstrapPeerStore?.all();
|
|
2026
|
+
if (!controlNode || !targets || targets.size === 0) {
|
|
2027
|
+
return;
|
|
2028
|
+
}
|
|
2029
|
+
// Same skip rule as step 3 of the steady-state pass: no churn on live links.
|
|
2030
|
+
const connected = new Set(controlNode.getConnections().map((c) => c.remotePeer.toString()));
|
|
2031
|
+
let dialed = 0;
|
|
2032
|
+
for (const [peerId, entry] of targets) {
|
|
2033
|
+
if (!this._running || !this.controlNode) {
|
|
2034
|
+
return;
|
|
2035
|
+
}
|
|
2036
|
+
if (connected.has(peerId)) {
|
|
2037
|
+
continue;
|
|
2038
|
+
}
|
|
2039
|
+
if (await this.dialBootstrapPeer(peerId, entry.addrs)) {
|
|
2040
|
+
dialed++;
|
|
2041
|
+
}
|
|
2042
|
+
}
|
|
2043
|
+
log('reconcileControlCohort: cold-start pass complete (bootstrap=%d, connected=%d, dialed=%d)',
|
|
2044
|
+
targets.size, connected.size, dialed);
|
|
2045
|
+
}
|
|
2046
|
+
|
|
2047
|
+
/**
|
|
2048
|
+
* Bind a bootstrap peer's seed addresses to its peer id, so the dial
|
|
2049
|
+
* authenticates the peer it is aiming at rather than trusting whoever answers.
|
|
2050
|
+
*
|
|
2051
|
+
* An address that already carries `/p2p/<id>` must carry THIS peer's id or it
|
|
2052
|
+
* is dropped (a seed that disagrees with itself is not a dial target); one
|
|
2053
|
+
* that carries none — an `addressOverrides`-published record, say — gets the
|
|
2054
|
+
* id encapsulated. Unencapsulatable addresses are dropped rather than dialed
|
|
2055
|
+
* bare.
|
|
2056
|
+
*/
|
|
2057
|
+
private bootstrapDialAddrs(peerId: string, addrs: string[]): Multiaddr[] {
|
|
2058
|
+
const out: Multiaddr[] = [];
|
|
2059
|
+
for (const addr of this.parseMultiaddrs(addrs)) {
|
|
2060
|
+
const embedded = trailingPeerId(addr);
|
|
2061
|
+
if (embedded === peerId) {
|
|
2062
|
+
out.push(addr);
|
|
2063
|
+
continue;
|
|
2064
|
+
}
|
|
2065
|
+
if (embedded !== null) {
|
|
2066
|
+
log('reconcileControlCohort(cold-start): addr %s names peer %s, not %s; dropping',
|
|
2067
|
+
addr.toString(), embedded, peerId);
|
|
2068
|
+
continue;
|
|
2069
|
+
}
|
|
2070
|
+
try {
|
|
2071
|
+
out.push(addr.encapsulate(`/p2p/${peerId}`));
|
|
2072
|
+
} catch (error) {
|
|
2073
|
+
log('reconcileControlCohort(cold-start): cannot bind %s to %s: %o', addr.toString(), peerId, error);
|
|
2074
|
+
}
|
|
2075
|
+
}
|
|
2076
|
+
return out;
|
|
2077
|
+
}
|
|
2078
|
+
|
|
2079
|
+
/** Dial one cold-start bootstrap peer, best-effort. Returns whether it connected. */
|
|
2080
|
+
private async dialBootstrapPeer(peerId: string, addrs: string[]): Promise<boolean> {
|
|
2081
|
+
const controlNode = this.controlNode;
|
|
2082
|
+
if (!controlNode) {
|
|
2083
|
+
return false;
|
|
2084
|
+
}
|
|
2085
|
+
const parsed = this.bootstrapDialAddrs(peerId, addrs);
|
|
2086
|
+
if (parsed.length === 0) {
|
|
2087
|
+
log('reconcileControlCohort(cold-start): no dialable address for bootstrap peer %s; skipping', peerId);
|
|
2088
|
+
return false;
|
|
2089
|
+
}
|
|
2090
|
+
try {
|
|
2091
|
+
log('reconcileControlCohort(cold-start): dialing bootstrap peer %s (%d addr(s))', peerId, parsed.length);
|
|
2092
|
+
await controlNode.dial(parsed);
|
|
2093
|
+
return true;
|
|
2094
|
+
} catch (error) {
|
|
2095
|
+
log('reconcileControlCohort(cold-start): dial of bootstrap peer %s failed (continuing): %o', peerId, error);
|
|
2096
|
+
return false;
|
|
2097
|
+
}
|
|
2098
|
+
}
|
|
2099
|
+
|
|
1165
2100
|
// ============================================================================
|
|
1166
2101
|
// Write-while-alone re-replication (control-write-ensure-replicated)
|
|
1167
2102
|
//
|
|
1168
2103
|
// Optimystic's coordinator commits a control write WITHOUT broadcasting when the
|
|
1169
|
-
// block's cluster has ≤1 member. An
|
|
2104
|
+
// block's cluster has ≤1 member. An owner that authorizes/removes a peer or
|
|
1170
2105
|
// (re)publishes its own record while no sibling is connected therefore writes a
|
|
1171
2106
|
// row that exists ONLY in its local control DB — a sibling that connects later
|
|
1172
2107
|
// never observes it (pull-on-read routes to the block's cluster, which never
|
|
@@ -1187,38 +2122,63 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1187
2122
|
* is caught by the cohort's periodic pull-on-read instead. Re-issuing an already-
|
|
1188
2123
|
* replicated row is harmless (an idempotent monotonic bump), so the coarse proxy
|
|
1189
2124
|
* is acceptable as the agreed first cut.
|
|
2125
|
+
*
|
|
2126
|
+
* Reads {@link getControlConnectionCount} — the public, embedder-facing form of
|
|
2127
|
+
* the same sample — so the two cannot drift.
|
|
1190
2128
|
*/
|
|
1191
2129
|
private committedAlone(): boolean {
|
|
1192
|
-
return
|
|
2130
|
+
return this.getControlConnectionCount() === 0;
|
|
1193
2131
|
}
|
|
1194
2132
|
|
|
1195
2133
|
/**
|
|
1196
|
-
* Record (or clear) a just-committed
|
|
2134
|
+
* Record (or clear) a just-committed owner membership write in the
|
|
1197
2135
|
* write-while-alone re-replication queue. If the control node had no connections
|
|
1198
2136
|
* at commit, the Optimystic write was local-only and must be re-issued once the
|
|
1199
2137
|
* cohort grows; otherwise the write replicated and any stale queue entry for this
|
|
1200
2138
|
* subject is dropped.
|
|
1201
2139
|
*
|
|
1202
|
-
* A `remove`
|
|
1203
|
-
*
|
|
1204
|
-
*
|
|
1205
|
-
*
|
|
2140
|
+
* A `remove` never queues here — a delete's re-replicable half is its
|
|
2141
|
+
* `Revocation` tombstone, which the committed-delete seam routes into
|
|
2142
|
+
* {@link pendingRevocations} ({@link noteGuardedDelete}). The `remove` arm only
|
|
2143
|
+
* drops any queued authorize for the subject (the row is gone; re-issuing the
|
|
2144
|
+
* insert would resurrect it) and, when the commit was alone, logs the
|
|
2145
|
+
* security-relevant window loudly.
|
|
1206
2146
|
*/
|
|
1207
2147
|
private noteControlWrite(peerId: string, kind: 'authorize' | 'remove'): void {
|
|
2148
|
+
if (kind === 'remove') {
|
|
2149
|
+
this.pendingPeerWrites.delete(peerId);
|
|
2150
|
+
if (this.committedAlone()) {
|
|
2151
|
+
log('removePeer(%s) committed while ALONE (0 control connections): the deletion is ' +
|
|
2152
|
+
'local-only, so a revoked peer may persist in the cohort until the queued ' +
|
|
2153
|
+
'Revocation tombstone is re-issued on cohort growth (drainPendingRevocations).', peerId);
|
|
2154
|
+
}
|
|
2155
|
+
return;
|
|
2156
|
+
}
|
|
1208
2157
|
if (!this.committedAlone()) {
|
|
1209
2158
|
this.pendingPeerWrites.delete(peerId);
|
|
1210
2159
|
return;
|
|
1211
2160
|
}
|
|
1212
|
-
this.pendingPeerWrites.set(peerId,
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
2161
|
+
this.pendingPeerWrites.set(peerId, 'authorize');
|
|
2162
|
+
log('authorizePeer(%s) committed while alone (0 control connections); queued for ' +
|
|
2163
|
+
're-replication on cohort growth', peerId);
|
|
2164
|
+
}
|
|
2165
|
+
|
|
2166
|
+
/**
|
|
2167
|
+
* Committed-delete seam handler ({@link GuardedDeleteListener}, wired in
|
|
2168
|
+
* {@link start}): a guarded-table delete and its `Revocation` tombstone
|
|
2169
|
+
* committed. If the node was alone the tombstone is local-only — queue it for
|
|
2170
|
+
* an owner-signed re-issue on cohort growth; otherwise it broadcast, so drop
|
|
2171
|
+
* any stale queue entry for the stamp. Synchronous by contract — bookkeeping
|
|
2172
|
+
* only, never throws into the delete path.
|
|
2173
|
+
*/
|
|
2174
|
+
private noteGuardedDelete(revocation: RevokedRowRef): void {
|
|
2175
|
+
if (!this.committedAlone()) {
|
|
2176
|
+
this.pendingRevocations.delete(revocation.stampId);
|
|
2177
|
+
return;
|
|
1221
2178
|
}
|
|
2179
|
+
this.pendingRevocations.set(revocation.stampId, revocation);
|
|
2180
|
+
log('%s delete of %s committed while alone (0 control connections); Revocation tombstone ' +
|
|
2181
|
+
'(stamp %s) queued for re-issue on cohort growth', revocation.tableName, revocation.rowKey, revocation.stampId);
|
|
1222
2182
|
}
|
|
1223
2183
|
|
|
1224
2184
|
/**
|
|
@@ -1250,10 +2210,23 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1250
2210
|
return;
|
|
1251
2211
|
}
|
|
1252
2212
|
const firstGrowth = !this.reconstructedLocalOnlyWrites;
|
|
1253
|
-
log('Draining control re-replication (reason=%s, firstGrowth=%s, pendingPeers=%d, self=%s, deviceToken=%s)',
|
|
1254
|
-
reason, firstGrowth, this.pendingPeerWrites.size, this.
|
|
1255
|
-
|
|
1256
|
-
|
|
2213
|
+
log('Draining control re-replication (reason=%s, firstGrowth=%s, pendingPeers=%d, pendingRevocations=%d, self=%s, deviceToken=%s)',
|
|
2214
|
+
reason, firstGrowth, this.pendingPeerWrites.size, this.pendingRevocations.size,
|
|
2215
|
+
this.pendingSelfPeerWrite, this.pendingSelfDeviceWrite);
|
|
2216
|
+
|
|
2217
|
+
// 1. Revocation tombstones FIRST: a removal that never reached the cohort
|
|
2218
|
+
// leaves a revoked peer live elsewhere, which outranks every re-touch
|
|
2219
|
+
// below. Ordering is not cosmetic — every `CadrePeer` write here (self row
|
|
2220
|
+
// included) can burn tens of seconds in upstream retry livelock after a
|
|
2221
|
+
// collection fork, while the tombstone lives in a different collection
|
|
2222
|
+
// that is merely behind, not forked. No membership-gate refresh around
|
|
2223
|
+
// this step: a re-issue only bumps `ReissuedAt` — LOCAL membership
|
|
2224
|
+
// visibility changed at delete time (deleteCadrePeer already notified),
|
|
2225
|
+
// and receiving-node staleness is the pre-existing pull-on-read /
|
|
2226
|
+
// periodic-refresh property.
|
|
2227
|
+
await this.drainPendingRevocations();
|
|
2228
|
+
|
|
2229
|
+
// 2. Self rows. On the first growth always re-touch (covers a self row carried
|
|
1257
2230
|
// over from a prior process that committed local-only before this start);
|
|
1258
2231
|
// afterwards only when a self-write-while-alone was recorded this session.
|
|
1259
2232
|
// registerSelf is idempotent + single-flight, so this never races a
|
|
@@ -1267,16 +2240,16 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1267
2240
|
this.pendingSelfDeviceWrite = false;
|
|
1268
2241
|
}
|
|
1269
2242
|
|
|
1270
|
-
//
|
|
2243
|
+
// 3. First growth: an owner reconstructs membership rows it may have
|
|
1271
2244
|
// authored that never replicated (covers writes from before this process
|
|
1272
2245
|
// started). Rows already tracked in the in-memory queue are handled by
|
|
1273
|
-
// step
|
|
2246
|
+
// step 4 — skipped here to avoid a double re-touch.
|
|
1274
2247
|
if (firstGrowth) {
|
|
1275
2248
|
this.reconstructedLocalOnlyWrites = true;
|
|
1276
2249
|
await this.reconstructAuthoredMembership();
|
|
1277
2250
|
}
|
|
1278
2251
|
|
|
1279
|
-
//
|
|
2252
|
+
// 4. In-session pending owner authorize writes.
|
|
1280
2253
|
await this.drainPendingPeerWrites();
|
|
1281
2254
|
}
|
|
1282
2255
|
|
|
@@ -1303,21 +2276,85 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1303
2276
|
}
|
|
1304
2277
|
}
|
|
1305
2278
|
|
|
2279
|
+
/**
|
|
2280
|
+
* Re-issue `Revocation` tombstones so a removal that committed while alone
|
|
2281
|
+
* still reaches the cohort. Two regimes, one method:
|
|
2282
|
+
*
|
|
2283
|
+
* - **First successful pass per process** ({@link reissuedHeldRevocations}
|
|
2284
|
+
* false): sweep EVERY locally-held tombstone from `queryRevocations()` — the
|
|
2285
|
+
* only cover for a removal made before this process started, since the
|
|
2286
|
+
* in-memory queue does not survive a restart. Queued stamps are part of that
|
|
2287
|
+
* same batch (exactly once — one transaction, no double bump).
|
|
2288
|
+
* - **Later passes**: only the tombstones queued in-session by
|
|
2289
|
+
* {@link noteGuardedDelete}.
|
|
2290
|
+
*
|
|
2291
|
+
* Owner-gated: a node with no owner key cannot sign a re-issue, so stray queue
|
|
2292
|
+
* entries are dropped (mirrors {@link drainPendingPeerWrites}). Best-effort — a
|
|
2293
|
+
* failure is logged and leaves the queue (and the sweep flag) untouched for the
|
|
2294
|
+
* next growth edge. A successful `reissueRevocations` exec is NOT proof of
|
|
2295
|
+
* broadcast (the connection that fired the growth edge may not be in the
|
|
2296
|
+
* affected block's cluster — see
|
|
2297
|
+
* tickets/backlog/control-rereplication-broadcast-confirmation); the drain
|
|
2298
|
+
* inherits that known gap, and a full disconnect→reconnect (or the next
|
|
2299
|
+
* process's sweep) re-covers it.
|
|
2300
|
+
*
|
|
2301
|
+
* NOTE: the sweep is O(all tombstones ever) row-updates — plus one owner
|
|
2302
|
+
* signature each — in one transaction, once per lifetime. `Revocation` is
|
|
2303
|
+
* append-only and unbounded growth is declared acceptable for a cadre-sized
|
|
2304
|
+
* party. If the tombstone table ever gets large, bound the sweep (e.g. persist
|
|
2305
|
+
* a node-local high-water mark of what has been re-issued while connected)
|
|
2306
|
+
* instead of re-touching everything; note the `Math.max(...)` spread below also
|
|
2307
|
+
* caps out around 10^5 rows (V8 argument limit) before the size becomes merely
|
|
2308
|
+
* a latency problem.
|
|
2309
|
+
*/
|
|
2310
|
+
private async drainPendingRevocations(): Promise<void> {
|
|
2311
|
+
const sweep = !this.reissuedHeldRevocations;
|
|
2312
|
+
if (!sweep && this.pendingRevocations.size === 0) {
|
|
2313
|
+
return;
|
|
2314
|
+
}
|
|
2315
|
+
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
2316
|
+
return;
|
|
2317
|
+
}
|
|
2318
|
+
if (!this.seedBootstrapService?.canAuthorize()) {
|
|
2319
|
+
// A non-owner node cannot sign a re-issue; drop any stray entries.
|
|
2320
|
+
this.pendingRevocations.clear();
|
|
2321
|
+
return;
|
|
2322
|
+
}
|
|
2323
|
+
try {
|
|
2324
|
+
const held = await this.controlDatabase.queryRevocations();
|
|
2325
|
+
const rows = sweep ? held : held.filter((r) => this.pendingRevocations.has(r.stampId));
|
|
2326
|
+
if (rows.length > 0) {
|
|
2327
|
+
const reissuedAt = Math.max(Date.now(), Math.max(...rows.map((r) => r.reissuedAt)) + 1);
|
|
2328
|
+
await this.seedBootstrapService.reissueRevocations(rows, reissuedAt);
|
|
2329
|
+
log('drain: re-issued %d revocation tombstone(s)%s', rows.length, sweep ? ' (first-growth sweep)' : '');
|
|
2330
|
+
}
|
|
2331
|
+
this.reissuedHeldRevocations = true;
|
|
2332
|
+
// Per-stamp clear (not clear()) so an entry queued concurrently mid-drain
|
|
2333
|
+
// survives for the next growth edge.
|
|
2334
|
+
for (const row of rows) {
|
|
2335
|
+
this.pendingRevocations.delete(row.stampId);
|
|
2336
|
+
}
|
|
2337
|
+
} catch (error) {
|
|
2338
|
+
log('drain: revocation re-issue failed; leaving queued for next growth: %o', error);
|
|
2339
|
+
}
|
|
2340
|
+
}
|
|
2341
|
+
|
|
1306
2342
|
/**
|
|
1307
2343
|
* One-shot, first-cohort-growth reconstruction of the write-while-alone queue for
|
|
1308
|
-
* an
|
|
1309
|
-
*
|
|
2344
|
+
* an OWNER node: re-touch every membership row that may be an unreplicated
|
|
2345
|
+
* owner insert. A row is a candidate iff it is not self (handled by
|
|
1310
2346
|
* {@link registerSelf}), is not already tracked in the in-memory queue (handled by
|
|
1311
2347
|
* {@link drainPendingPeerWrites}), and carries no self-`Sig` yet (an
|
|
1312
|
-
*
|
|
2348
|
+
* owner-authored row the peer has not self-published — the only kind safe to
|
|
1313
2349
|
* bump without invalidating a self-signature). O(rows) on the small control
|
|
1314
|
-
* tables; safe to over-apply (a monotonic
|
|
2350
|
+
* tables; safe to over-apply (a monotonic owner bump on an already-replicated
|
|
1315
2351
|
* row is a no-op-equivalent).
|
|
1316
2352
|
*
|
|
1317
|
-
* A node with no
|
|
1318
|
-
* rows for other peers, and rows it merely holds are not its to re-issue.
|
|
1319
|
-
*
|
|
1320
|
-
*
|
|
2353
|
+
* A node with no owner private key skips this entirely — it cannot re-sign
|
|
2354
|
+
* rows for other peers, and rows it merely holds are not its to re-issue.
|
|
2355
|
+
* DELETEs are not reconstructed here — the `Revocation` tombstone sweep in
|
|
2356
|
+
* {@link drainPendingRevocations} covers them (a removed row leaves no
|
|
2357
|
+
* `CadrePeer` trace, but its tombstone is re-issuable).
|
|
1321
2358
|
*/
|
|
1322
2359
|
private async reconstructAuthoredMembership(): Promise<void> {
|
|
1323
2360
|
if (!this.controlDatabase || !this.controlNode) {
|
|
@@ -1328,10 +2365,29 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1328
2365
|
}
|
|
1329
2366
|
const selfPeerId = this.controlNode.peerId.toString();
|
|
1330
2367
|
const members = await this.controlDatabase.queryCadrePeers();
|
|
2368
|
+
// One gate refresh for the whole sweep: every re-touch is a notifying
|
|
2369
|
+
// `CadrePeer` write, and these rows are already in the local snapshot (they
|
|
2370
|
+
// committed here, just local-only), so per-row refreshes would be O(rows)
|
|
2371
|
+
// membership reads for a snapshot that does not change.
|
|
2372
|
+
const touched = await this.deferMembershipGateRefresh(
|
|
2373
|
+
'drain-reconstruct',
|
|
2374
|
+
() => this.reissueAuthoredMembershipRows(selfPeerId, members)
|
|
2375
|
+
);
|
|
2376
|
+
if (touched > 0) {
|
|
2377
|
+
log('drain: reconstructed %d owner-authored membership row(s) on first cohort growth', touched);
|
|
2378
|
+
}
|
|
2379
|
+
}
|
|
2380
|
+
|
|
2381
|
+
/**
|
|
2382
|
+
* Body of {@link reconstructAuthoredMembership}'s sweep: re-issue each candidate
|
|
2383
|
+
* owner-authored row, returning how many were re-touched. A per-row failure is
|
|
2384
|
+
* logged and skipped; a teardown mid-sweep stops it.
|
|
2385
|
+
*/
|
|
2386
|
+
private async reissueAuthoredMembershipRows(selfPeerId: string, members: CadrePeerRow[]): Promise<number> {
|
|
1331
2387
|
let touched = 0;
|
|
1332
2388
|
for (const member of members) {
|
|
1333
2389
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
1334
|
-
return;
|
|
2390
|
+
return touched;
|
|
1335
2391
|
}
|
|
1336
2392
|
if (member.peerId === selfPeerId || this.pendingPeerWrites.has(member.peerId)) {
|
|
1337
2393
|
continue;
|
|
@@ -1348,66 +2404,64 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1348
2404
|
log('drain: reconstruction re-touch of %s failed (continuing): %o', member.peerId, error);
|
|
1349
2405
|
}
|
|
1350
2406
|
}
|
|
1351
|
-
|
|
1352
|
-
log('drain: reconstructed %d authority-authored membership row(s) on first cohort growth', touched);
|
|
1353
|
-
}
|
|
2407
|
+
return touched;
|
|
1354
2408
|
}
|
|
1355
2409
|
|
|
1356
2410
|
/**
|
|
1357
|
-
* Drain the in-session write-while-alone queue: re-issue each pending
|
|
1358
|
-
*
|
|
1359
|
-
*
|
|
1360
|
-
*
|
|
1361
|
-
*
|
|
1362
|
-
*
|
|
1363
|
-
*
|
|
1364
|
-
*
|
|
1365
|
-
*
|
|
1366
|
-
* - `remove` (security-relevant): a removePeer-while-alone physically removed the
|
|
1367
|
-
* row locally, so a re-issued DELETE matches nothing and does NOT propagate —
|
|
1368
|
-
* full delete durability requires a schema tombstone (deferred:
|
|
1369
|
-
* `control-delete-while-alone-tombstone`). We still attempt the re-issue
|
|
1370
|
-
* (harmless, and it propagates in the row-still-present case); the commit-alone
|
|
1371
|
-
* was already logged loudly in {@link noteControlWrite}.
|
|
2411
|
+
* Drain the in-session write-while-alone queue: re-issue each pending owner
|
|
2412
|
+
* authorize as an idempotent monotonic owner UPDATE
|
|
2413
|
+
* ({@link reissuePeerAuthorize}) now that the cohort can broadcast it.
|
|
2414
|
+
* Sequential (no fan-out) to avoid a thundering re-touch; an entry is cleared
|
|
2415
|
+
* only on success, so a failure (or a still-alone re-commit) leaves it queued
|
|
2416
|
+
* for the next growth. An entry is skipped (and cleared) if the row vanished
|
|
2417
|
+
* (raced a delete) or now carries a self-`Sig` (the peer self-published and owns
|
|
2418
|
+
* its republish). Removals are drained by {@link drainPendingRevocations}, not
|
|
2419
|
+
* here.
|
|
1372
2420
|
*/
|
|
1373
2421
|
private async drainPendingPeerWrites(): Promise<void> {
|
|
1374
2422
|
if (this.pendingPeerWrites.size === 0) {
|
|
1375
2423
|
return;
|
|
1376
2424
|
}
|
|
1377
2425
|
if (!this.seedBootstrapService?.canAuthorize()) {
|
|
1378
|
-
// A non-
|
|
2426
|
+
// A non-owner node cannot have authored these writes; drop any stray entries.
|
|
1379
2427
|
this.pendingPeerWrites.clear();
|
|
1380
2428
|
return;
|
|
1381
2429
|
}
|
|
1382
|
-
for
|
|
1383
|
-
|
|
2430
|
+
// One gate refresh for the whole drain, not one per entry: these re-issues
|
|
2431
|
+
// replay writes this node already applied locally (and already refreshed
|
|
2432
|
+
// for), so the snapshot only has to be correct once the drain settles.
|
|
2433
|
+
await this.deferMembershipGateRefresh('drain-reissue', () => this.reissuePendingPeerWrites());
|
|
2434
|
+
}
|
|
2435
|
+
|
|
2436
|
+
/**
|
|
2437
|
+
* Body of {@link drainPendingPeerWrites}' loop. An entry is cleared only on
|
|
2438
|
+
* success (or when there is nothing left to re-issue); a failure is logged and
|
|
2439
|
+
* the entry stays queued for the next cohort growth.
|
|
2440
|
+
*/
|
|
2441
|
+
private async reissuePendingPeerWrites(): Promise<void> {
|
|
2442
|
+
for (const peerId of [...this.pendingPeerWrites.keys()]) {
|
|
2443
|
+
if (!this._running || !this.controlNode || !this.controlDatabase || !this.seedBootstrapService) {
|
|
1384
2444
|
return;
|
|
1385
2445
|
}
|
|
1386
2446
|
try {
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
continue;
|
|
1393
|
-
}
|
|
1394
|
-
await this.reissuePeerAuthorize(peerId, record.updatedAt);
|
|
1395
|
-
} else {
|
|
1396
|
-
// Best-effort delete re-issue (see method doc): propagates only if the row
|
|
1397
|
-
// is somehow still present locally.
|
|
1398
|
-
await this.seedBootstrapService.removePeer(peerId);
|
|
2447
|
+
const record = await this.controlDatabase.queryPeerRecord(peerId);
|
|
2448
|
+
if (!record || record.sig) {
|
|
2449
|
+
// Removed since, or peer self-published — nothing for the owner to re-issue.
|
|
2450
|
+
this.pendingPeerWrites.delete(peerId);
|
|
2451
|
+
continue;
|
|
1399
2452
|
}
|
|
2453
|
+
await this.reissuePeerAuthorize(peerId, record.updatedAt);
|
|
1400
2454
|
this.pendingPeerWrites.delete(peerId);
|
|
1401
2455
|
} catch (error) {
|
|
1402
|
-
log('drain: re-issue of %s (
|
|
2456
|
+
log('drain: re-issue of %s (authorize) failed; leaving queued for next growth: %o', peerId, error);
|
|
1403
2457
|
}
|
|
1404
2458
|
}
|
|
1405
2459
|
}
|
|
1406
2460
|
|
|
1407
2461
|
/**
|
|
1408
|
-
* Re-issue an
|
|
2462
|
+
* Re-issue an owner membership row as a monotonic owner UPDATE: bump
|
|
1409
2463
|
* `UpdatedAt` strictly above the stored value (and the wall clock) and re-sign via
|
|
1410
|
-
* the
|
|
2464
|
+
* the owner branch of `CadrePeer.AuthorizedUpdate`. The row already exists
|
|
1411
2465
|
* locally (it committed there, just local-only), so this is an UPDATE, not the
|
|
1412
2466
|
* original INSERT.
|
|
1413
2467
|
*/
|
|
@@ -1526,11 +2580,11 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1526
2580
|
* for any member — the `AuthorizedUpdate` self-branch verifies the new `Sig`
|
|
1527
2581
|
* against the bound `CadrePeer.PublicKey`). Platform/Token may change here
|
|
1528
2582
|
* (rotation / platform switch / reinstall are all normal self-updates).
|
|
1529
|
-
* - If not, and the node holds an
|
|
2583
|
+
* - If not, and the node holds an owner service: an owner-signed INSERT
|
|
1530
2584
|
* that also carries the self-signature.
|
|
1531
2585
|
* - Otherwise: throws. Like `CadrePeer`, the first `DeviceToken` row requires an
|
|
1532
|
-
*
|
|
1533
|
-
* seeded by an
|
|
2586
|
+
* owner signature; a non-owner peer (e.g. a phone) must have its row
|
|
2587
|
+
* seeded by an owner — typically the server it enrolled with — before it
|
|
1534
2588
|
* can self-refresh. (Establishing that phone→server registration handshake is
|
|
1535
2589
|
* the downstream "RN registration" ticket; this node only owns the cadre-core
|
|
1536
2590
|
* write path.)
|
|
@@ -1541,7 +2595,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1541
2595
|
* @param platform - `'fcm'` (Android/Firebase) or `'apns'` (Apple).
|
|
1542
2596
|
* @param token - the opaque platform device/registration token.
|
|
1543
2597
|
* @throws if the node is not started, exposes no self-signing key, or has no
|
|
1544
|
-
* existing row and no
|
|
2598
|
+
* existing row and no owner service to self-insert.
|
|
1545
2599
|
*/
|
|
1546
2600
|
async registerDeviceToken(platform: PushPlatform, token: string): Promise<void> {
|
|
1547
2601
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
@@ -1571,14 +2625,21 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1571
2625
|
return;
|
|
1572
2626
|
}
|
|
1573
2627
|
if (this.seedBootstrapService) {
|
|
2628
|
+
// NOTE: the `CadrePeer` twin of this read-then-insert needed race recovery because an
|
|
2629
|
+
// owner `authorizePeer` can seat a peer's row for it. Nothing seats a `DeviceToken`
|
|
2630
|
+
// row on a peer's behalf, and this insert has no in-lock existence check, so a lost
|
|
2631
|
+
// race would THROW rather than silently drop the record. If an owner-driven
|
|
2632
|
+
// device-token seeding path is ever added, or two first-publishes can run
|
|
2633
|
+
// concurrently (this method has no single-flight guard, unlike `registerSelf`),
|
|
2634
|
+
// revisit — see `publishSelfRecord`'s fall-through.
|
|
1574
2635
|
await this.seedBootstrapService.insertSelfDeviceToken(record);
|
|
1575
2636
|
if (this.committedAlone()) this.pendingSelfDeviceWrite = true;
|
|
1576
|
-
log('registerDeviceToken: inserted own DeviceToken (
|
|
2637
|
+
log('registerDeviceToken: inserted own DeviceToken (owner-signed, platform=%s, updatedAt=%d)', platform, updatedAt);
|
|
1577
2638
|
return;
|
|
1578
2639
|
}
|
|
1579
2640
|
throw new Error(
|
|
1580
2641
|
`Cannot register device token for ${peerId}: no existing row to self-update and no ` +
|
|
1581
|
-
'
|
|
2642
|
+
'owner service to self-insert. An owner must seed this peer\'s DeviceToken ' +
|
|
1582
2643
|
'row first (mirrors CadrePeer enrollment).'
|
|
1583
2644
|
);
|
|
1584
2645
|
}
|
|
@@ -1593,8 +2654,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1593
2654
|
* 2. `publicKey <-> peerId` binding — the stored key's libp2p identity is the
|
|
1594
2655
|
* requested peerId,
|
|
1595
2656
|
* 3. a `DeviceToken` row exists with a known {@link PushPlatform},
|
|
1596
|
-
* 4.
|
|
1597
|
-
*
|
|
2657
|
+
* 4. the row's `StampId` is NOT retired in `CadreControl.Revocation` — the
|
|
2658
|
+
* read-side half of the clear (see below),
|
|
2659
|
+
* 5. self-signature verifies against the bound `CadrePeer.PublicKey`,
|
|
2660
|
+
* 6. freshness — `updatedAt` is positive and within `opts.maxAgeMs` (default:
|
|
1598
2661
|
* no ceiling, since a push token is valid until it rotates).
|
|
1599
2662
|
*
|
|
1600
2663
|
* A peer that is not a current member, or whose token has no backing `CadrePeer`
|
|
@@ -1628,6 +2691,23 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1628
2691
|
return null;
|
|
1629
2692
|
}
|
|
1630
2693
|
|
|
2694
|
+
// Retired stamp: the read-side mitigation for the write-time race, mirroring what
|
|
2695
|
+
// listAuthorizedMembers does for CadrePeer. Clearing a token retires its StampId
|
|
2696
|
+
// into Revocation, but the schema's NotRevoked CHECK only sees LOCALLY visible
|
|
2697
|
+
// tombstones — a node that converged on a replayed insert before the tombstone can
|
|
2698
|
+
// hold both rows, so the reader must drop the resurrected one.
|
|
2699
|
+
//
|
|
2700
|
+
// NOTE: the freshness ceiling below defaults to INFINITE by design (a suspended
|
|
2701
|
+
// phone must stay push-reachable long after it published), so unlike a peer address
|
|
2702
|
+
// record a stale token never ages out. Stamp retirement is the ONLY thing that
|
|
2703
|
+
// retires a cleared token — do not weaken this check on the assumption that
|
|
2704
|
+
// staleness will eventually cover it.
|
|
2705
|
+
const revokedStamps = await this.controlDatabase.queryRevokedStamps('DeviceToken');
|
|
2706
|
+
if (revokedStamps.has(record.stampId)) {
|
|
2707
|
+
log('resolveDeviceToken: StampId for %s is retired (cleared token, resurrected row)', peerId);
|
|
2708
|
+
return null;
|
|
2709
|
+
}
|
|
2710
|
+
|
|
1631
2711
|
// Self-signature over (peerId, platform, token, updatedAt), verified against the
|
|
1632
2712
|
// CadrePeer.PublicKey bound to this peerId.
|
|
1633
2713
|
if (!verifyDeviceTokenSignature(record, peerRecord.publicKey)) {
|
|
@@ -1649,11 +2729,11 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1649
2729
|
/**
|
|
1650
2730
|
* Delete this node's own `DeviceToken` row (logout / token invalidation). No-op
|
|
1651
2731
|
* when no row exists. Like {@link registerDeviceToken}'s first insert, the delete
|
|
1652
|
-
* is gated on an
|
|
1653
|
-
* AND delete), so it requires this node's
|
|
1654
|
-
* must route the clear through its
|
|
2732
|
+
* is gated on an owner signature (`DeviceToken.AuthorizedInsert` covers insert
|
|
2733
|
+
* AND delete), so it requires this node's owner service; a non-owner peer
|
|
2734
|
+
* must route the clear through its owner (downstream RN registration path).
|
|
1655
2735
|
*
|
|
1656
|
-
* @throws if the node is not started, or a row exists but no
|
|
2736
|
+
* @throws if the node is not started, or a row exists but no owner service is
|
|
1657
2737
|
* available to sign the delete.
|
|
1658
2738
|
*/
|
|
1659
2739
|
async clearDeviceToken(): Promise<void> {
|
|
@@ -1668,8 +2748,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1668
2748
|
}
|
|
1669
2749
|
if (!this.seedBootstrapService) {
|
|
1670
2750
|
throw new Error(
|
|
1671
|
-
`Cannot clear device token for ${peerId}: delete requires an
|
|
1672
|
-
'and no
|
|
2751
|
+
`Cannot clear device token for ${peerId}: delete requires an owner signature ` +
|
|
2752
|
+
'and no owner service is initialized.'
|
|
1673
2753
|
);
|
|
1674
2754
|
}
|
|
1675
2755
|
await this.seedBootstrapService.deleteDeviceToken(peerId);
|
|
@@ -1681,10 +2761,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1681
2761
|
* unregistered during a push-wake fan-out. Unlike {@link clearDeviceToken}
|
|
1682
2762
|
* (self-only — it hardcodes the local peerId), this takes an arbitrary peerId.
|
|
1683
2763
|
*
|
|
1684
|
-
* - When this node holds an
|
|
1685
|
-
* (`deleteDeviceToken` is
|
|
2764
|
+
* - When this node holds an owner seed service, it deletes the row
|
|
2765
|
+
* (`deleteDeviceToken` is owner-gated and accepts any peerId), so the peer
|
|
1686
2766
|
* is not retried until it re-registers.
|
|
1687
|
-
* - When this node is NOT an
|
|
2767
|
+
* - When this node is NOT an owner it cannot delete the row, so it only logs
|
|
1688
2768
|
* that a re-registration is needed. The fan-out's own in-memory dead-token set
|
|
1689
2769
|
* is what actually stops re-pushing to the dead token this process — see
|
|
1690
2770
|
* {@link PushFanoutService}. That set is acceptably lossy across restarts (a
|
|
@@ -1696,13 +2776,13 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1696
2776
|
if (this.seedBootstrapService) {
|
|
1697
2777
|
try {
|
|
1698
2778
|
await this.seedBootstrapService.deleteDeviceToken(peerId);
|
|
1699
|
-
log('expireDeviceToken:
|
|
2779
|
+
log('expireDeviceToken: owner-deleted stale DeviceToken for %s', peerId);
|
|
1700
2780
|
} catch (error) {
|
|
1701
|
-
log('expireDeviceToken:
|
|
2781
|
+
log('expireDeviceToken: owner delete for %s failed: %o', peerId, error);
|
|
1702
2782
|
}
|
|
1703
2783
|
return;
|
|
1704
2784
|
}
|
|
1705
|
-
log('expireDeviceToken: %s token is stale but this node is not an
|
|
2785
|
+
log('expireDeviceToken: %s token is stale but this node is not an owner; re-registration required', peerId);
|
|
1706
2786
|
}
|
|
1707
2787
|
|
|
1708
2788
|
private async handleStrandAdded(strand: StrandRow): Promise<void> {
|
|
@@ -1730,6 +2810,13 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1730
2810
|
strandId: strand.Id,
|
|
1731
2811
|
error: error instanceof Error ? error : new Error(String(error))
|
|
1732
2812
|
});
|
|
2813
|
+
// Rethrow after the event: the only caller is the StrandWatcher's
|
|
2814
|
+
// onStrandAdded callback, which uses the rejection to decide whether the
|
|
2815
|
+
// strand was really added. Swallowing it here would have the watcher mark
|
|
2816
|
+
// the strand as known-and-added, so no later poll would ever retry it.
|
|
2817
|
+
// The watcher catches and logs, so this never escapes as an unhandled
|
|
2818
|
+
// rejection.
|
|
2819
|
+
throw error;
|
|
1733
2820
|
}
|
|
1734
2821
|
}
|
|
1735
2822
|
|
|
@@ -1737,14 +2824,9 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1737
2824
|
log('Handling strand removed: %s', strandId);
|
|
1738
2825
|
|
|
1739
2826
|
try {
|
|
1740
|
-
//
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
// Remove sApp config
|
|
1744
|
-
this.sAppConfigs.delete(strandId);
|
|
1745
|
-
|
|
1746
|
-
await this.strandManager.stopStrand(strandId);
|
|
1747
|
-
this.emit('strand:stopped', { strandId });
|
|
2827
|
+
// Same local teardown the explicit stop performs — deliberately NOT stopStrand
|
|
2828
|
+
// itself, whose `_running` guard would throw on a poll that lands during shutdown.
|
|
2829
|
+
await this.detachStrand(strandId);
|
|
1748
2830
|
} catch (error) {
|
|
1749
2831
|
log('Error stopping strand %s: %o', strandId, error);
|
|
1750
2832
|
this.emit('strand:error', {
|
|
@@ -1814,8 +2896,19 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1814
2896
|
// Clear sApp configs
|
|
1815
2897
|
this.sAppConfigs.clear();
|
|
1816
2898
|
|
|
1817
|
-
//
|
|
2899
|
+
// Drop delegate-admission state: the grants are scoped to the session that
|
|
2900
|
+
// recorded them, so a stop()/start() cycle on this object must not keep
|
|
2901
|
+
// admitting delegates announced under the previous one, nor carry stale
|
|
2902
|
+
// announce timestamps that would suppress the next session's refresh.
|
|
2903
|
+
this.delegateAdmission.clear();
|
|
2904
|
+
this.delegateAnnounceAt.clear();
|
|
2905
|
+
|
|
2906
|
+
// Close control database (this also shuts down the collection factory).
|
|
2907
|
+
// Detach the membership hub first: nothing this teardown does should drive a
|
|
2908
|
+
// gate refresh, and a notification arriving after this point is a no-op.
|
|
1818
2909
|
if (this.controlDatabase) {
|
|
2910
|
+
this.controlDatabase.setMembershipChangeListener(null);
|
|
2911
|
+
this.controlDatabase.setGuardedDeleteListener(null);
|
|
1819
2912
|
await this.controlDatabase.close();
|
|
1820
2913
|
this.controlDatabase = null;
|
|
1821
2914
|
}
|
|
@@ -1908,7 +3001,13 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1908
3001
|
* (returns the live instance unchanged) as a backstop against double-resume.
|
|
1909
3002
|
*/
|
|
1910
3003
|
private async resumeStrandRuntime(strandId: string): Promise<void> {
|
|
1911
|
-
|
|
3004
|
+
// Re-derive the transport peerId (deterministic and cheap — a quiesced
|
|
3005
|
+
// instance has no `libp2pNode` to read it from) so the seed pass announces
|
|
3006
|
+
// the delegate before `resumeStrand` runs `libp2p.start()`.
|
|
3007
|
+
const delegatePeerId = this.identityKey
|
|
3008
|
+
? peerIdFromPrivateKey(await strandTransportKey(this.identityKey, strandId)).toString()
|
|
3009
|
+
: undefined;
|
|
3010
|
+
const seed = await this.resolveCohortSeed(strandId, delegatePeerId);
|
|
1912
3011
|
const mode = selectStrandMode(undefined, seed.hasOtherPeers);
|
|
1913
3012
|
await this.strandManager.resumeStrand(strandId, {
|
|
1914
3013
|
bootstrapNodes: seed.bootstrapNodes,
|
|
@@ -2048,6 +3147,14 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2048
3147
|
/**
|
|
2049
3148
|
* Add a strand with its sApp configuration.
|
|
2050
3149
|
* The hosting application must provide the sApp schema when creating a strand.
|
|
3150
|
+
*
|
|
3151
|
+
* A rejected call leaves nothing running but DOES leave the sApp config
|
|
3152
|
+
* registered, deliberately: both an explicit retry and the {@link StrandWatcher}'s
|
|
3153
|
+
* automatic relaunch need it. So once the strand's row is visible on the control
|
|
3154
|
+
* network (via {@link publishStrand} or another member), a failed launch keeps
|
|
3155
|
+
* being re-attempted in the background — each failure re-emitting `strand:error`
|
|
3156
|
+
* — until it succeeds. {@link detachStrand} (reached via {@link stopStrand}) is
|
|
3157
|
+
* what abandons a strand for good.
|
|
2051
3158
|
*/
|
|
2052
3159
|
async addStrand(config: StrandConfig): Promise<StrandInstance> {
|
|
2053
3160
|
if (!this._running) {
|
|
@@ -2070,20 +3177,20 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2070
3177
|
|
|
2071
3178
|
/**
|
|
2072
3179
|
* Publish a strand row to the shared control database under this node's own
|
|
2073
|
-
*
|
|
3180
|
+
* owner identity, so other cadre members discover it via control-network
|
|
2074
3181
|
* sync (their {@link StrandWatcher} fires `strand:discovered`).
|
|
2075
3182
|
*
|
|
2076
|
-
* This is the
|
|
3183
|
+
* This is the owner-signed `Strand` INSERT that {@link addStrand}
|
|
2077
3184
|
* deliberately omits: `addStrand` only starts the LOCAL strand instance,
|
|
2078
3185
|
* whereas publishing makes the strand visible cadre-wide. A typical creator
|
|
2079
3186
|
* does both (start locally + publish); a discovering peer only does
|
|
2080
3187
|
* `addStrand` (the row already exists).
|
|
2081
3188
|
*
|
|
2082
3189
|
* The insert is signed with the ed25519 key behind this node's PeerId — which
|
|
2083
|
-
* {@link
|
|
2084
|
-
* so peer identity and
|
|
2085
|
-
* enrolled in `
|
|
2086
|
-
* at genesis) or the schema's `Strand.
|
|
3190
|
+
* {@link ed25519KeyPairFromLibp2p} also exposes as the node's owner keypair,
|
|
3191
|
+
* so peer identity and owner key are one and the same. That key must be
|
|
3192
|
+
* enrolled in `OwnerKey` (e.g. via {@link ControlDatabase.ensureOwnerKey}
|
|
3193
|
+
* at genesis) or the schema's `Strand.AuthorizedInsert` constraint rejects the write.
|
|
2087
3194
|
* Failing loudly here is intentional: a silently-unpublished strand would run
|
|
2088
3195
|
* as a local-only island that no peer could ever discover or join.
|
|
2089
3196
|
*
|
|
@@ -2091,34 +3198,111 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2091
3198
|
* {@link addStrand}).
|
|
2092
3199
|
* @param type - `'o'` for open (default) or `'c'` for closed.
|
|
2093
3200
|
* @param memberPrivateKey - Optional membership key for a closed strand.
|
|
2094
|
-
* @throws if the node is not started, exposes no
|
|
2095
|
-
* control DB rejects the (unauthorized) insert.
|
|
3201
|
+
* @throws if the node is not started, exposes no owner signing key, the id is blank, or
|
|
3202
|
+
* the control DB rejects the (unauthorized) insert.
|
|
2096
3203
|
*/
|
|
2097
3204
|
async publishStrand(strandId: string, type: 'o' | 'c' = 'o', memberPrivateKey?: string): Promise<void> {
|
|
2098
|
-
|
|
2099
|
-
|
|
2100
|
-
|
|
2101
|
-
const
|
|
2102
|
-
|
|
2103
|
-
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
|
|
2107
|
-
|
|
3205
|
+
const signingKey = this.requireOwnerSigningKey(`publish strand ${strandId}`);
|
|
3206
|
+
// Trim/reject here so the id that lands matches the one unpublishStrand looks up: it
|
|
3207
|
+
// trims too, and an untrimmed row would be unreachable by the same string.
|
|
3208
|
+
const trimmed = requireNonBlank(strandId, 'strand id');
|
|
3209
|
+
await this.controlDatabase!.insertStrand(
|
|
3210
|
+
trimmed,
|
|
3211
|
+
type,
|
|
3212
|
+
signingKey.publicKeyB64,
|
|
3213
|
+
signMessageWith(signingKey.privateKeyB64),
|
|
3214
|
+
memberPrivateKey
|
|
3215
|
+
);
|
|
3216
|
+
log('Published strand %s (type %s) to control DB under owner %s', trimmed, type, signingKey.publicKeyB64);
|
|
3217
|
+
}
|
|
3218
|
+
|
|
3219
|
+
/**
|
|
3220
|
+
* Remove this party's `Strand` row from the shared control database — the owner-signed
|
|
3221
|
+
* inverse of {@link publishStrand}.
|
|
3222
|
+
*
|
|
3223
|
+
* Party-wide, unlike {@link stopStrand}, which only stops the strand on THIS node: every
|
|
3224
|
+
* cadre node watching the table sees the row vanish on its next poll (default 5 s) and
|
|
3225
|
+
* stops its own instance. This node converges immediately — the method forces a watcher
|
|
3226
|
+
* poll and stops any still-running local instance before resolving. Removing the row
|
|
3227
|
+
* removes OUR party's participation only: other parties in the strand keep their own
|
|
3228
|
+
* rows and the strand network carries on, so no cross-party sign-off is involved — this
|
|
3229
|
+
* is an owner-signed control-plane write like every other, not "destroy the network".
|
|
3230
|
+
*
|
|
3231
|
+
* Irreversible for a closed strand (`Type='c'`): the row carries `MemberPrivateKey`,
|
|
3232
|
+
* this party's membership secret for that network, and it is stored nowhere else.
|
|
3233
|
+
* With it gone the party can never again admit a member to that closed strand, and a
|
|
3234
|
+
* re-published row would carry a DIFFERENT key that does not match the membership
|
|
3235
|
+
* already written into the strand's RBAC layer. The strand id itself is NOT
|
|
3236
|
+
* blacklisted: a fresh owner-signed {@link publishStrand} re-seats it under a new
|
|
3237
|
+
* stamp — only the unsigned consent re-seat is permanently foreclosed (the removal's
|
|
3238
|
+
* `Revocation` tombstone names the id, which the consent branch of
|
|
3239
|
+
* `Strand.AuthorizedInsert` refuses ever after). Outstanding `FormationInvite` rows
|
|
3240
|
+
* bound to the removed strand are unredeemable while the row is absent (the formation
|
|
3241
|
+
* recorder resolves the strand as missing and rejects cleanly).
|
|
3242
|
+
*
|
|
3243
|
+
* Convergence caveats, same class as {@link enrollValidationKey}'s: a sibling that has
|
|
3244
|
+
* not yet synced keeps running the strand until its own watcher observes the missing
|
|
3245
|
+
* row; a sibling whose `strandFilter` never admitted this strand never observes the
|
|
3246
|
+
* removal AT ALL and keeps running its instance indefinitely — opting out of watching a
|
|
3247
|
+
* strand is also opting out of its party-wide removal, so such a node's only stop is its
|
|
3248
|
+
* own local {@link stopStrand}/`unpublishStrand` call; and a removal committed while
|
|
3249
|
+
* ALONE (0 control connections) deletes the row local-only. The accompanying
|
|
3250
|
+
* `Revocation` tombstone IS queued and re-issued on the next cohort-growth edge
|
|
3251
|
+
* ({@link noteGuardedDelete}/{@link drainPendingRevocations}), so the stamp retirement
|
|
3252
|
+
* — and with it the consent re-seat foreclosure — does propagate; but the physical row
|
|
3253
|
+
* deletion cannot be replayed, and `queryStrands` reads raw (no retired-stamp filter),
|
|
3254
|
+
* so siblings that already hold the row keep running the strand until the collection
|
|
3255
|
+
* itself converges (logged loudly; see the delete-while-alone durability note in
|
|
3256
|
+
* docs/architecture.md).
|
|
3257
|
+
*
|
|
3258
|
+
* A no-op (no throw, no tombstone) when the row is already absent — but a
|
|
3259
|
+
* locally-running instance of that id is still stopped.
|
|
3260
|
+
*
|
|
3261
|
+
* NOTE: control-plane only — the strand's local durable storage is retained. Stopping the
|
|
3262
|
+
* instance closes the `StrandDatabase` and its libp2p node but purges no blocks, so a
|
|
3263
|
+
* closed strand's content stays readable on disk to anyone with the data directory. If a
|
|
3264
|
+
* caller ever needs removal to mean "and erase the local copy", that is a separate purge
|
|
3265
|
+
* step, not a widening of this method.
|
|
3266
|
+
*
|
|
3267
|
+
* @param strandId - The `Strand` row id to remove (as passed to {@link publishStrand}).
|
|
3268
|
+
* @throws if the node is not started, exposes no owner signing key, the id is blank, or
|
|
3269
|
+
* the signer is not an enrolled owner (the schema's `Strand.AuthorizedDelete` rejects
|
|
3270
|
+
* the write and the row survives). A rejection does NOT imply the row survived: the
|
|
3271
|
+
* local stop runs after the control-plane delete has already committed, so a failure
|
|
3272
|
+
* there throws over a completed removal.
|
|
3273
|
+
*/
|
|
3274
|
+
async unpublishStrand(strandId: string): Promise<void> {
|
|
3275
|
+
const signingKey = this.requireOwnerSigningKey(`unpublish strand ${strandId}`);
|
|
3276
|
+
const trimmed = requireNonBlank(strandId, 'strand id');
|
|
3277
|
+
const removed = await this.controlDatabase!.deleteStrand(
|
|
3278
|
+
trimmed,
|
|
3279
|
+
signingKey.publicKeyB64,
|
|
3280
|
+
signMessageWith(signingKey.privateKeyB64)
|
|
3281
|
+
);
|
|
3282
|
+
// Gate on `removed`: the absent-row no-op wrote nothing, so warning about an
|
|
3283
|
+
// unreplicated deletion there would send an operator chasing a phantom.
|
|
3284
|
+
if (removed && this.committedAlone()) {
|
|
3285
|
+
log('unpublishStrand(%s) committed while ALONE (0 control connections): the deletion is ' +
|
|
3286
|
+
'local-only. Its Revocation tombstone is queued for re-issue on cohort growth, but the ' +
|
|
3287
|
+
'row deletion itself cannot be replayed, so other nodes may keep running the strand ' +
|
|
3288
|
+
'until the collection converges.', trimmed);
|
|
3289
|
+
}
|
|
3290
|
+
// Converge locally now rather than waiting up to a poll interval. The watcher fires
|
|
3291
|
+
// onStrandRemoved for a strand it tracked; the explicit stop below covers a node whose
|
|
3292
|
+
// strandFilter never admitted this strand (the watcher never knew it, so it will never
|
|
3293
|
+
// fire). StrandInstanceManager.stopStrand no-ops on an unknown id, so the two paths
|
|
3294
|
+
// cannot double-stop into an error.
|
|
3295
|
+
await this.strandWatcher?.forcePoll();
|
|
3296
|
+
if (this.strandManager.getInstance(trimmed)) {
|
|
3297
|
+
await this.stopStrand(trimmed);
|
|
2108
3298
|
}
|
|
2109
|
-
|
|
2110
|
-
// buildAuthorizationMessage); ed25519-sign them directly (no pre-hash) with
|
|
2111
|
-
// the authority private key, returning a base64url signature.
|
|
2112
|
-
const signMessage = (message: Uint8Array): string =>
|
|
2113
|
-
sign(message, signingKey.privateKeyB64, 'ed25519', 'bytes', 'base64url', 'base64url') as string;
|
|
2114
|
-
await this.controlDatabase.insertStrand(strandId, type, signingKey.publicKeyB64, signMessage, memberPrivateKey);
|
|
2115
|
-
log('Published strand %s (type %s) to control DB under authority %s', strandId, type, signingKey.publicKeyB64);
|
|
3299
|
+
log('Unpublished strand %s from control DB under owner %s', trimmed, signingKey.publicKeyB64);
|
|
2116
3300
|
}
|
|
2117
3301
|
|
|
2118
3302
|
/**
|
|
2119
|
-
* Publish an
|
|
3303
|
+
* Publish an owner-signed `FormationInvite` (open-invitation token) to the
|
|
2120
3304
|
* shared control database, so a later {@link formStrand} redemption can be
|
|
2121
|
-
* validated against it (the consent branch of `Strand.
|
|
3305
|
+
* validated against it (the consent branch of `Strand.AuthorizedInsert`).
|
|
2122
3306
|
*
|
|
2123
3307
|
* Counterpart to {@link createOpenInvitation}, which only mints the
|
|
2124
3308
|
* out-of-band {@link OpenInvitation} envelope: persisting the matching
|
|
@@ -2127,8 +3311,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2127
3311
|
* from this row. A host minting a closed-strand invite does both (mint +
|
|
2128
3312
|
* publish), exactly as the integration harness's `createInvitation` does.
|
|
2129
3313
|
*
|
|
2130
|
-
* Signs with the same self-
|
|
2131
|
-
* key behind this node's PeerId, which must be an enrolled `
|
|
3314
|
+
* Signs with the same self-owner key as {@link publishStrand} (the ed25519
|
|
3315
|
+
* key behind this node's PeerId, which must be an enrolled `OwnerKey`).
|
|
2132
3316
|
* Throws loudly if the node isn't started or exposes no signing key.
|
|
2133
3317
|
*
|
|
2134
3318
|
* @param token - Invitation token (the `FormationInvite` primary key); use the
|
|
@@ -2142,21 +3326,124 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2142
3326
|
sAppId: string,
|
|
2143
3327
|
options: { expiresAtMs?: number; totalUses?: number; validationUrl?: string; strandId?: string } = {}
|
|
2144
3328
|
): Promise<void> {
|
|
3329
|
+
const signingKey = this.requireOwnerSigningKey(`publish formation invite ${token}`);
|
|
3330
|
+
await this.controlDatabase!.insertFormationInvite(
|
|
3331
|
+
token,
|
|
3332
|
+
sAppId,
|
|
3333
|
+
signingKey.publicKeyB64,
|
|
3334
|
+
signMessageWith(signingKey.privateKeyB64),
|
|
3335
|
+
options
|
|
3336
|
+
);
|
|
3337
|
+
// A host may publish an invite whose token was minted elsewhere; registering
|
|
3338
|
+
// it locally opens the connection gate's formation exemption immediately
|
|
3339
|
+
// rather than waiting for the durable row to become readable.
|
|
3340
|
+
this.strandSolicitationService?.registerMintedInvitation(
|
|
3341
|
+
token,
|
|
3342
|
+
options.expiresAtMs ?? Number.POSITIVE_INFINITY
|
|
3343
|
+
);
|
|
3344
|
+
log('Published formation invite %s (sApp %s) under owner %s', token, sAppId, signingKey.publicKeyB64);
|
|
3345
|
+
}
|
|
3346
|
+
|
|
3347
|
+
/**
|
|
3348
|
+
* Enroll an approver public key allowed to sign off on `ValidationUrl` redemptions.
|
|
3349
|
+
*
|
|
3350
|
+
* A `FormationInvite` carrying a `ValidationUrl` is only redeemable when the approval
|
|
3351
|
+
* that comes back from that URL is signed by a key present in `CadreControl.ValidationKey`
|
|
3352
|
+
* — this is how a party says which outside approver it trusts. Without an enrollment,
|
|
3353
|
+
* every such invitation is unredeemable.
|
|
3354
|
+
*
|
|
3355
|
+
* Signs with the same self-owner key as {@link publishStrand} (the ed25519 key behind
|
|
3356
|
+
* this node's PeerId, which must be an enrolled `OwnerKey`). Throws loudly if the node
|
|
3357
|
+
* isn't started, exposes no signing key, or the key is blank — a blank key would reach
|
|
3358
|
+
* the database and fail the signature CHECK as an opaque constraint error.
|
|
3359
|
+
*
|
|
3360
|
+
* Enrolling this node's OWN owner key is permitted and harmless: the domain/action tags
|
|
3361
|
+
* baked into each authorization digest mean an owner-key signature can never satisfy the
|
|
3362
|
+
* approval rule and an approval can never satisfy an owner rule. No guard is needed.
|
|
3363
|
+
*
|
|
3364
|
+
* NOTE: enrollment is replicated control state, so a key enrolled here is visible to a
|
|
3365
|
+
* sibling cadre node only once control replication converges. A redemption that arrives
|
|
3366
|
+
* at a node which has not caught up is refused as not-enrolled — the same convergence
|
|
3367
|
+
* gap the schema records on `Strand.StampId`. Enroll before circulating the invitations
|
|
3368
|
+
* that depend on the key.
|
|
3369
|
+
*
|
|
3370
|
+
* @param key - Approver public key to enroll (base64url ed25519 public key).
|
|
3371
|
+
*/
|
|
3372
|
+
async enrollValidationKey(key: string): Promise<void> {
|
|
3373
|
+
const signingKey = this.requireOwnerSigningKey('enroll a validation key');
|
|
3374
|
+
const trimmed = requireEd25519PublicKeyB64(key, 'validation key');
|
|
3375
|
+
await this.controlDatabase!.insertValidationKey(
|
|
3376
|
+
trimmed,
|
|
3377
|
+
signingKey.publicKeyB64,
|
|
3378
|
+
signMessageWith(signingKey.privateKeyB64)
|
|
3379
|
+
);
|
|
3380
|
+
log('Enrolled validation key %s under owner %s', trimmed, signingKey.publicKeyB64);
|
|
3381
|
+
}
|
|
3382
|
+
|
|
3383
|
+
/**
|
|
3384
|
+
* Remove an approver public key.
|
|
3385
|
+
*
|
|
3386
|
+
* Narrows who may approve FUTURE redemptions ONLY. The schema's approval CHECKs run at
|
|
3387
|
+
* write time, so a join that was already approved by this key stays valid — removal is
|
|
3388
|
+
* not retroactive and does not re-examine committed rows.
|
|
3389
|
+
*
|
|
3390
|
+
* Rotation is therefore add-then-remove, in that order: removing the only enrolled key
|
|
3391
|
+
* while `ValidationUrl` invitations are outstanding makes every one of them unredeemable
|
|
3392
|
+
* until a new key is enrolled.
|
|
3393
|
+
*
|
|
3394
|
+
* Removing a key that is not enrolled is a silent no-op (no throw, no `Revocation`
|
|
3395
|
+
* tombstone) — see {@link ControlDatabase.deleteValidationKey}.
|
|
3396
|
+
*
|
|
3397
|
+
* @param key - Approver public key to remove (base64url ed25519 public key).
|
|
3398
|
+
*/
|
|
3399
|
+
async removeValidationKey(key: string): Promise<void> {
|
|
3400
|
+
const signingKey = this.requireOwnerSigningKey('remove a validation key');
|
|
3401
|
+
const trimmed = requireNonBlank(key, 'validation key');
|
|
3402
|
+
await this.controlDatabase!.deleteValidationKey(
|
|
3403
|
+
trimmed,
|
|
3404
|
+
signingKey.publicKeyB64,
|
|
3405
|
+
signMessageWith(signingKey.privateKeyB64)
|
|
3406
|
+
);
|
|
3407
|
+
log('Removed validation key %s under owner %s', trimmed, signingKey.publicKeyB64);
|
|
3408
|
+
}
|
|
3409
|
+
|
|
3410
|
+
/**
|
|
3411
|
+
* The approver keys currently enrolled, sorted. Read-only, so unlike
|
|
3412
|
+
* {@link enrollValidationKey} / {@link removeValidationKey} it needs no owner signing
|
|
3413
|
+
* key — only a started node.
|
|
3414
|
+
*/
|
|
3415
|
+
async listValidationKeys(): Promise<string[]> {
|
|
3416
|
+
if (!this._running || !this.controlDatabase) {
|
|
3417
|
+
throw new Error('CadreNode must be started before listing validation keys');
|
|
3418
|
+
}
|
|
3419
|
+
return await this.controlDatabase.queryValidationKeys();
|
|
3420
|
+
}
|
|
3421
|
+
|
|
3422
|
+
/**
|
|
3423
|
+
* Resolve the owner keypair every owner-signed control write needs — {@link publishStrand},
|
|
3424
|
+
* {@link unpublishStrand}, {@link publishFormationInvite}, {@link enrollValidationKey},
|
|
3425
|
+
* {@link removeValidationKey}
|
|
3426
|
+
* — failing loudly in one two-part shape (not started / no signing key, naming owner
|
|
3427
|
+
* genesis as the fix). Narrows `controlDatabase` for the caller: a non-null return means
|
|
3428
|
+
* `this.controlDatabase` is non-null too.
|
|
3429
|
+
*
|
|
3430
|
+
* @param action - Infinitive phrase naming the attempted write, e.g. `'enroll a
|
|
3431
|
+
* validation key'`; interpolated into both messages.
|
|
3432
|
+
*/
|
|
3433
|
+
private requireOwnerSigningKey(action: string): { privateKeyB64: string; publicKeyB64: string } {
|
|
2145
3434
|
if (!this._running || !this.controlDatabase) {
|
|
2146
|
-
throw new Error(
|
|
3435
|
+
throw new Error(`CadreNode must be started before attempting to ${action}`);
|
|
2147
3436
|
}
|
|
2148
3437
|
const signingKey = this.getSelfSigningKey();
|
|
2149
3438
|
if (!signingKey) {
|
|
2150
3439
|
throw new Error(
|
|
2151
|
-
`Cannot
|
|
2152
|
-
'(node identity is unavailable or does not match the node PeerId). Run
|
|
2153
|
-
'genesis (
|
|
3440
|
+
`Cannot ${action}: no owner signing key available ` +
|
|
3441
|
+
'(node identity is unavailable or does not match the node PeerId). Run owner ' +
|
|
3442
|
+
'genesis (ensureOwnerKey + initializeSeedBootstrap) before writing owner-signed ' +
|
|
3443
|
+
'control state.'
|
|
2154
3444
|
);
|
|
2155
3445
|
}
|
|
2156
|
-
|
|
2157
|
-
sign(message, signingKey.privateKeyB64, 'ed25519', 'bytes', 'base64url', 'base64url') as string;
|
|
2158
|
-
await this.controlDatabase.insertFormationInvite(token, sAppId, signingKey.publicKeyB64, signMessage, options);
|
|
2159
|
-
log('Published formation invite %s (sApp %s) under authority %s', token, sAppId, signingKey.publicKeyB64);
|
|
3446
|
+
return signingKey;
|
|
2160
3447
|
}
|
|
2161
3448
|
|
|
2162
3449
|
/**
|
|
@@ -2164,6 +3451,15 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2164
3451
|
* control-discovered (`handleStrandAdded`) entry points. Resolves the cohort
|
|
2165
3452
|
* seed, selects the mode, starts the strand, and registers it with the
|
|
2166
3453
|
* hibernation manager before emitting `strand:started`.
|
|
3454
|
+
*
|
|
3455
|
+
* Idempotent no-op when the strand manager already tracks `strand.Id` — the
|
|
3456
|
+
* ordinary founding sequence is `addStrand` (starts it locally) followed by
|
|
3457
|
+
* `publishStrand` (makes it visible), so this node's own `StrandWatcher`
|
|
3458
|
+
* rediscovers a row it already started and calls this again via
|
|
3459
|
+
* `handleStrandAdded`. Without the guard that re-entry would resolve a fresh
|
|
3460
|
+
* (already-connected) cohort seed and re-emit `strand:started` for an
|
|
3461
|
+
* instance that never stopped. The guard runs before the cohort-seed RPC
|
|
3462
|
+
* fan-out, so a rediscovery costs nothing beyond the map lookup.
|
|
2167
3463
|
*/
|
|
2168
3464
|
private async launchStrand(
|
|
2169
3465
|
strand: StrandRow,
|
|
@@ -2171,7 +3467,37 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2171
3467
|
explicitMode?: StrandMode,
|
|
2172
3468
|
founder?: boolean
|
|
2173
3469
|
): Promise<StrandInstance> {
|
|
2174
|
-
const
|
|
3470
|
+
const existing = this.strandManager.getInstance(strand.Id);
|
|
3471
|
+
if (existing) {
|
|
3472
|
+
log('launchStrand: strand %s already tracked locally — skipping re-launch', strand.Id);
|
|
3473
|
+
return existing;
|
|
3474
|
+
}
|
|
3475
|
+
|
|
3476
|
+
// Each strand node gets its own transport identity, derived from the cadre
|
|
3477
|
+
// identity key + strandId (see strand-transport-key.ts). Sharing the
|
|
3478
|
+
// control node's key here gave every node one peerId, which collides at a
|
|
3479
|
+
// shared circuit relay (github.com/gotchoices/sereus/issues/1). The
|
|
3480
|
+
// retained launch config carries this derived key, so hibernate → wake
|
|
3481
|
+
// (resumeStrand) reuses the same peerId.
|
|
3482
|
+
//
|
|
3483
|
+
// NOTE: derivation requires an Ed25519 identity key, so a node configured
|
|
3484
|
+
// with some other key type now fails strand launch outright (surfaced as
|
|
3485
|
+
// `strand:error` / a rejected addStrand) where it previously started the
|
|
3486
|
+
// strand on that key. Ed25519 is already required for every control-DB
|
|
3487
|
+
// signing path, so nothing reachable today hits this; if a non-Ed25519
|
|
3488
|
+
// identity is ever supported, fall back to `undefined` here — libp2p then
|
|
3489
|
+
// generates a random per-strand key, which still avoids the collision but
|
|
3490
|
+
// gives up peerId stability across restarts.
|
|
3491
|
+
const transportKey = this.identityKey
|
|
3492
|
+
? await strandTransportKey(this.identityKey, strand.Id)
|
|
3493
|
+
: undefined;
|
|
3494
|
+
|
|
3495
|
+
// Derived BEFORE seed resolution so the seed pass doubles as the delegate
|
|
3496
|
+
// announcement and every grant is recorded before `startStrand` runs
|
|
3497
|
+
// `libp2p.start()` (the responder records the grant before replying; the
|
|
3498
|
+
// client awaits the replies).
|
|
3499
|
+
const delegatePeerId = transportKey ? peerIdFromPrivateKey(transportKey).toString() : undefined;
|
|
3500
|
+
const seed = await this.resolveCohortSeed(strand.Id, delegatePeerId);
|
|
2175
3501
|
const mode = selectStrandMode(explicitMode, seed.hasOtherPeers);
|
|
2176
3502
|
|
|
2177
3503
|
const instance = await this.strandManager.startStrand({
|
|
@@ -2181,10 +3507,12 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2181
3507
|
network: this.config.network,
|
|
2182
3508
|
profile: this.config.profile,
|
|
2183
3509
|
defaultLatencyHint: this.config.hibernation?.defaultLatencyHint ?? 'interactive',
|
|
2184
|
-
privateKey:
|
|
3510
|
+
privateKey: transportKey,
|
|
2185
3511
|
bootstrapNodes: seed.bootstrapNodes,
|
|
2186
3512
|
mode,
|
|
2187
3513
|
requireSignedSchemas: this.config.requireSignedSchemas,
|
|
3514
|
+
clusterSize: this.config.strandClusterSize,
|
|
3515
|
+
backfill: this.config.strandBackfill,
|
|
2188
3516
|
founder
|
|
2189
3517
|
});
|
|
2190
3518
|
|
|
@@ -2206,8 +3534,19 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2206
3534
|
* empty (`[]`) — mode still follows membership, and the empty seed self-heals
|
|
2207
3535
|
* on the next resume / check-in pass. Returns an empty seed when the control DB
|
|
2208
3536
|
* or node is absent (not yet started / torn down).
|
|
3537
|
+
*
|
|
3538
|
+
* When `delegatePeerId` is given (a strand launch/resume is imminent), the
|
|
3539
|
+
* pass doubles as the delegate ANNOUNCEMENT: our circuit relays are merged
|
|
3540
|
+
* into the RPC targets and every request carries the delegate peerId, so each
|
|
3541
|
+
* receiver records an admission grant BEFORE `libp2p.start()` (re-)dials the
|
|
3542
|
+
* relay reservation — the gate denial there is fatal, not degraded. A
|
|
3543
|
+
* party-member relay answers the RPC (its control node admits us as a
|
|
3544
|
+
* member); a dedicated ops/ relay does not speak the protocol and the
|
|
3545
|
+
* per-peer failure folds to `[]` — harmless, it has no membership gate and
|
|
3546
|
+
* needs no grant. The relay's direct addr rides along as the dial fallback
|
|
3547
|
+
* for a relay we are not yet connected to.
|
|
2209
3548
|
*/
|
|
2210
|
-
private async resolveCohortSeed(strandId: string): Promise<CohortSeed> {
|
|
3549
|
+
private async resolveCohortSeed(strandId: string, delegatePeerId?: string): Promise<CohortSeed> {
|
|
2211
3550
|
if (!this.controlDatabase || !this.controlNode) {
|
|
2212
3551
|
return { bootstrapNodes: [], hasOtherPeers: false };
|
|
2213
3552
|
}
|
|
@@ -2218,13 +3557,118 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2218
3557
|
// RPC only siblings we already have a control connection to — they can answer
|
|
2219
3558
|
// now, and `dialProtocol` by peerId reuses the open connection.
|
|
2220
3559
|
const connected = new Set(this.controlNode.getConnections().map((c) => c.remotePeer.toString()));
|
|
2221
|
-
const targets = otherPeerIds
|
|
3560
|
+
const targets: StrandAddrPeer[] = otherPeerIds
|
|
3561
|
+
.filter((id) => connected.has(id))
|
|
3562
|
+
.map((peerId) => ({ peerId }));
|
|
3563
|
+
const relays = delegatePeerId === undefined ? [] : this.circuitRelayTargets();
|
|
3564
|
+
for (const relay of relays) {
|
|
3565
|
+
if (!targets.some((t) => t.peerId === relay.relayPeerId)) {
|
|
3566
|
+
targets.push(relayStrandAddrPeer(relay));
|
|
3567
|
+
}
|
|
3568
|
+
}
|
|
2222
3569
|
const bootstrapNodes = targets.length
|
|
2223
|
-
? await collectStrandAddrs(this.controlNode, targets
|
|
3570
|
+
? await collectStrandAddrs(this.controlNode, targets, strandId, { delegatePeerId })
|
|
2224
3571
|
: [];
|
|
3572
|
+
// Throttle state for the RELAY targets only — refreshDelegateGrants never
|
|
3573
|
+
// looks up a sibling key, so recording one would only be dead weight.
|
|
3574
|
+
this.recordDelegateAnnounces(relays.map((r) => r.relayPeerId), strandId);
|
|
2225
3575
|
return { bootstrapNodes, hasOtherPeers };
|
|
2226
3576
|
}
|
|
2227
3577
|
|
|
3578
|
+
/**
|
|
3579
|
+
* The circuit relays this node's strand nodes would reserve through: the
|
|
3580
|
+
* union of the RESOLVED listen addrs' circuits — a hand-written
|
|
3581
|
+
* `network.listenAddrs` circuit entry or a `network.relayAddrs` entry, which
|
|
3582
|
+
* the strand node inherits verbatim either way — and the control node's own
|
|
3583
|
+
* live `/p2p-circuit` multiaddrs (a reservation this node discovered rather
|
|
3584
|
+
* than configured).
|
|
3585
|
+
*
|
|
3586
|
+
* NOTE: a relay the STRAND node discovers on its own (autorelay — in neither
|
|
3587
|
+
* source) gets no announcement, and a membership-gated one will deny it; fine
|
|
3588
|
+
* now, every realistic topology feeds one of the two sources.
|
|
3589
|
+
*/
|
|
3590
|
+
private circuitRelayTargets(): CircuitRelayTarget[] {
|
|
3591
|
+
return extractCircuitRelayTargets([
|
|
3592
|
+
...(resolveListenAddrs(this.config.network) ?? []),
|
|
3593
|
+
...(this.controlNode?.getMultiaddrs().map(String) ?? [])
|
|
3594
|
+
]);
|
|
3595
|
+
}
|
|
3596
|
+
|
|
3597
|
+
/**
|
|
3598
|
+
* Record the announce timestamps {@link refreshDelegateGrants} throttles on,
|
|
3599
|
+
* for every relay a delegate-carrying announce pass dialed.
|
|
3600
|
+
*
|
|
3601
|
+
* Recorded OPTIMISTICALLY at announce time: `collectStrandAddrs` folds
|
|
3602
|
+
* per-peer failure to `[]` and reports no per-peer success, and threading
|
|
3603
|
+
* success out would change its API for little gain. A failed INITIAL announce
|
|
3604
|
+
* is fatal-at-start anyway (the relay denies the reservation,
|
|
3605
|
+
* `libp2p.start()` throws, and wake/check-in re-resume retries with a fresh
|
|
3606
|
+
* announce); a failed REFRESH retries within `DELEGATE_GRANT_TTL_MS / 2`
|
|
3607
|
+
* (15 min), still inside the 30 min TTL.
|
|
3608
|
+
*/
|
|
3609
|
+
private recordDelegateAnnounces(relayPeerIds: readonly string[], strandId: string, now = Date.now()): void {
|
|
3610
|
+
for (const relayPeerId of relayPeerIds) {
|
|
3611
|
+
this.delegateAnnounceAt.set(peerStrandKey(relayPeerId, strandId), now);
|
|
3612
|
+
}
|
|
3613
|
+
}
|
|
3614
|
+
|
|
3615
|
+
/**
|
|
3616
|
+
* Re-announce every running strand's delegate peerId to this node's circuit
|
|
3617
|
+
* relays, throttled to once per `DELEGATE_GRANT_TTL_MS / 2` per
|
|
3618
|
+
* (relay, strand). A grant must outlive the reservation: a dropped relay
|
|
3619
|
+
* connection makes the strand's circuit-relay transport re-dial and face the
|
|
3620
|
+
* connection gate again, so the relay must still hold a live grant then.
|
|
3621
|
+
* RELAY targets only — siblings get their announce on every launch/resume
|
|
3622
|
+
* seed pass, and a grant only matters where a reservation can be re-dialed.
|
|
3623
|
+
*
|
|
3624
|
+
* Strands announce CONCURRENTLY: this runs ahead of the reconcile pass's
|
|
3625
|
+
* sibling enumeration, and one unreachable relay costs a dial timeout per
|
|
3626
|
+
* target, which must not stack up per strand.
|
|
3627
|
+
*/
|
|
3628
|
+
private async refreshDelegateGrants(now = Date.now()): Promise<void> {
|
|
3629
|
+
if (!this.controlNode) {
|
|
3630
|
+
return;
|
|
3631
|
+
}
|
|
3632
|
+
// A running strand's delegate peerId is simply its live node's peerId — no
|
|
3633
|
+
// re-derivation here.
|
|
3634
|
+
const running = new Map<string, string>();
|
|
3635
|
+
for (const [strandId, instance] of this.strandManager.getInstances()) {
|
|
3636
|
+
if (instance.libp2pNode) {
|
|
3637
|
+
running.set(strandId, instance.libp2pNode.peerId.toString());
|
|
3638
|
+
}
|
|
3639
|
+
}
|
|
3640
|
+
pruneStoppedStrandAnnounces(this.delegateAnnounceAt, new Set(running.keys()));
|
|
3641
|
+
if (running.size === 0) {
|
|
3642
|
+
return;
|
|
3643
|
+
}
|
|
3644
|
+
const relays = this.circuitRelayTargets();
|
|
3645
|
+
if (relays.length === 0) {
|
|
3646
|
+
return;
|
|
3647
|
+
}
|
|
3648
|
+
await Promise.all([...running].map(
|
|
3649
|
+
([strandId, delegatePeerId]) => this.announceDelegateToDueRelays(strandId, delegatePeerId, relays, now)
|
|
3650
|
+
));
|
|
3651
|
+
}
|
|
3652
|
+
|
|
3653
|
+
/** One strand's share of {@link refreshDelegateGrants}: announce to the relays whose grant is due. */
|
|
3654
|
+
private async announceDelegateToDueRelays(
|
|
3655
|
+
strandId: string,
|
|
3656
|
+
delegatePeerId: string,
|
|
3657
|
+
relays: readonly CircuitRelayTarget[],
|
|
3658
|
+
now: number
|
|
3659
|
+
): Promise<void> {
|
|
3660
|
+
const controlNode = this.controlNode;
|
|
3661
|
+
if (!controlNode) {
|
|
3662
|
+
return;
|
|
3663
|
+
}
|
|
3664
|
+
const due = dueRelayAnnounces(this.delegateAnnounceAt, relays, strandId, now);
|
|
3665
|
+
if (due.length === 0) {
|
|
3666
|
+
return;
|
|
3667
|
+
}
|
|
3668
|
+
await collectStrandAddrs(controlNode, due.map(relayStrandAddrPeer), strandId, { delegatePeerId });
|
|
3669
|
+
this.recordDelegateAnnounces(due.map((relay) => relay.relayPeerId), strandId, now);
|
|
3670
|
+
}
|
|
3671
|
+
|
|
2228
3672
|
/**
|
|
2229
3673
|
* The local strand instance's dialable strand-network multiaddrs for the
|
|
2230
3674
|
* strand-addr RPC, ordered signaling-first (reusing the control node's
|
|
@@ -2243,19 +3687,39 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2243
3687
|
}
|
|
2244
3688
|
|
|
2245
3689
|
/**
|
|
2246
|
-
*
|
|
3690
|
+
* Stop a strand on THIS node only: untrack it from hibernation, drop its sApp config,
|
|
3691
|
+
* stop the local instance, and emit `strand:stopped`. The shared `Strand` row is left
|
|
3692
|
+
* intact, so on the next node restart (or watcher poll) the strand is rediscovered and
|
|
3693
|
+
* surfaces as `strand:discovered` again. Party-wide removal is {@link unpublishStrand}.
|
|
2247
3694
|
*/
|
|
2248
|
-
async
|
|
3695
|
+
async stopStrand(strandId: string): Promise<void> {
|
|
2249
3696
|
if (!this._running) {
|
|
2250
3697
|
throw new Error('CadreNode not running');
|
|
2251
3698
|
}
|
|
2252
3699
|
|
|
2253
|
-
|
|
2254
|
-
|
|
3700
|
+
await this.detachStrand(strandId);
|
|
3701
|
+
}
|
|
2255
3702
|
|
|
2256
|
-
|
|
3703
|
+
/**
|
|
3704
|
+
* Local teardown for one strand, shared by the caller-driven {@link stopStrand} and the
|
|
3705
|
+
* watcher-driven `handleStrandRemoved`: untrack hibernation, drop the sApp config, stop
|
|
3706
|
+
* the instance, emit `strand:stopped`. Touches no control-plane row — which side of the
|
|
3707
|
+
* removal the node is on is the caller's concern, not this method's.
|
|
3708
|
+
*
|
|
3709
|
+
* The stop + emit are skipped when the strand manager holds no instance for `strandId` —
|
|
3710
|
+
* e.g. a party owner that published a strand's row but never ran it locally, or an
|
|
3711
|
+
* explicit {@link stopStrand} for an id this node never started. `hibernationManager`
|
|
3712
|
+
* untrack and the `sAppConfigs` delete stay unconditional (both are no-ops when there is
|
|
3713
|
+
* nothing to remove), so a launch that failed before an instance was ever tracked still
|
|
3714
|
+
* gets its stray sApp config cleared.
|
|
3715
|
+
*/
|
|
3716
|
+
private async detachStrand(strandId: string): Promise<void> {
|
|
3717
|
+
this.hibernationManager.untrackStrand(strandId);
|
|
2257
3718
|
this.sAppConfigs.delete(strandId);
|
|
2258
|
-
|
|
3719
|
+
if (!this.strandManager.hasStrand(strandId)) {
|
|
3720
|
+
log('detachStrand: strand %s not tracked locally — no-op (no stop, no strand:stopped)', strandId);
|
|
3721
|
+
return;
|
|
3722
|
+
}
|
|
2259
3723
|
await this.strandManager.stopStrand(strandId);
|
|
2260
3724
|
this.emit('strand:stopped', { strandId });
|
|
2261
3725
|
}
|
|
@@ -2449,6 +3913,76 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2449
3913
|
return this.controlNode;
|
|
2450
3914
|
}
|
|
2451
3915
|
|
|
3916
|
+
/**
|
|
3917
|
+
* Start keeping a relay reservation: dial the given relay(s) from the control
|
|
3918
|
+
* node, ask the first one that answers for a reservation slot, wait until the
|
|
3919
|
+
* resulting `/p2p-circuit` address makes this node dialable — and keep a
|
|
3920
|
+
* supervisor running that re-drives whenever the reservation is later lost.
|
|
3921
|
+
*
|
|
3922
|
+
* For nodes that listen on the bare `/p2p-circuit` SEARCH addr (a browser tab)
|
|
3923
|
+
* rather than a configured `network.relayAddrs` circuit listener — see
|
|
3924
|
+
* `relay-reservation.ts` for why the two are alternatives and why configuring
|
|
3925
|
+
* both is a regression. Opt-in: nothing calls this for the CLI/host nodes.
|
|
3926
|
+
*
|
|
3927
|
+
* Resolves once the FIRST attempt has settled, exactly as it did when it drove
|
|
3928
|
+
* once; the retries continue in the background until {@link stop} or the next
|
|
3929
|
+
* `reserveRelays` call. Fail-soft — never throws. An unreachable relay, a
|
|
3930
|
+
* missing control node, or a timeout all resolve to a non-`reserved` status, so
|
|
3931
|
+
* a caller can await this during startup without a dead relay aborting the node.
|
|
3932
|
+
*
|
|
3933
|
+
* Passing an empty list stops the supervisor and resets the posture to `none`
|
|
3934
|
+
* without dialing or waiting.
|
|
3935
|
+
*/
|
|
3936
|
+
async reserveRelays(
|
|
3937
|
+
addrs: string[],
|
|
3938
|
+
opts?: RelayReservationSupervisorOptions
|
|
3939
|
+
): Promise<RelayReservationState> {
|
|
3940
|
+
// Stop first, unconditionally: a second call with a different list must not
|
|
3941
|
+
// leave two loops fighting over one pending reservation slot.
|
|
3942
|
+
this.relayReserveSupervisor?.stop();
|
|
3943
|
+
this.relayReserveSupervisor = null;
|
|
3944
|
+
this.relayReserveAddrs = [...addrs];
|
|
3945
|
+
this.relayReserveError = null;
|
|
3946
|
+
if (addrs.length === 0) {
|
|
3947
|
+
return this.getRelayReservationState();
|
|
3948
|
+
}
|
|
3949
|
+
if (!this.controlNode) {
|
|
3950
|
+
// No node to supervise, so this is the one failure the node records itself.
|
|
3951
|
+
this.relayReserveError = 'control node unavailable';
|
|
3952
|
+
return this.getRelayReservationState();
|
|
3953
|
+
}
|
|
3954
|
+
const supervisor = superviseRelayReservation(this.controlNode, addrs, opts);
|
|
3955
|
+
this.relayReserveSupervisor = supervisor;
|
|
3956
|
+
await supervisor.firstAttempt;
|
|
3957
|
+
// Read through `this`, not `supervisor`: an overlapping call may have replaced
|
|
3958
|
+
// it while the first attempt was in flight, and the newest list is the truth.
|
|
3959
|
+
return this.getRelayReservationState();
|
|
3960
|
+
}
|
|
3961
|
+
|
|
3962
|
+
/**
|
|
3963
|
+
* The node's CURRENT relay-reservation posture, recomputed from the control
|
|
3964
|
+
* node's live multiaddrs on every call.
|
|
3965
|
+
*
|
|
3966
|
+
* Deliberately not memoised: a reservation can be lost after
|
|
3967
|
+
* {@link reserveRelays} succeeds (the relay restarts, the connection drops) and
|
|
3968
|
+
* a cached `reserved` would let a caller mint invitations carrying circuit
|
|
3969
|
+
* addresses that no longer route.
|
|
3970
|
+
*
|
|
3971
|
+
* A lost reservation now recovers on its own — the supervisor {@link reserveRelays}
|
|
3972
|
+
* starts re-drives on a backoff, reporting `retrying` meanwhile. `error` here
|
|
3973
|
+
* means nothing is going to try again: no supervisor, or no control node.
|
|
3974
|
+
*/
|
|
3975
|
+
getRelayReservationState(): RelayReservationState {
|
|
3976
|
+
const supervisor = this.relayReserveSupervisor;
|
|
3977
|
+
return resolveRelayReservationState(
|
|
3978
|
+
this.controlNode,
|
|
3979
|
+
this.relayReserveAddrs,
|
|
3980
|
+
supervisor ? supervisor.lastError : this.relayReserveError,
|
|
3981
|
+
supervisor?.driving ?? false,
|
|
3982
|
+
supervisor?.retryAtMs ?? null
|
|
3983
|
+
);
|
|
3984
|
+
}
|
|
3985
|
+
|
|
2452
3986
|
/**
|
|
2453
3987
|
* Get the control database (for advanced queries)
|
|
2454
3988
|
*/
|
|
@@ -2456,6 +3990,22 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2456
3990
|
return this.controlDatabase;
|
|
2457
3991
|
}
|
|
2458
3992
|
|
|
3993
|
+
/**
|
|
3994
|
+
* Open control-network connections right now. A lower-bound proxy for replication
|
|
3995
|
+
* reach: 0 connections ⇒ a control write commits local-only. The private
|
|
3996
|
+
* `committedAlone` write-while-alone seam is defined in terms of this.
|
|
3997
|
+
*
|
|
3998
|
+
* It approximates the precise signal (the block's cluster size), and a caller that
|
|
3999
|
+
* samples it AFTER a write samples a slightly wider window than `committedAlone`
|
|
4000
|
+
* does inside the write itself. It exists so an embedder can warn an owner before a
|
|
4001
|
+
* control write that this machine currently sees none of its siblings, and report
|
|
4002
|
+
* after one that the write may not have travelled — a warning shown unconditionally
|
|
4003
|
+
* is a warning people learn to ignore.
|
|
4004
|
+
*/
|
|
4005
|
+
getControlConnectionCount(): number {
|
|
4006
|
+
return this.controlNode?.getConnections().length ?? 0;
|
|
4007
|
+
}
|
|
4008
|
+
|
|
2459
4009
|
/**
|
|
2460
4010
|
* Force a poll of the strand watcher (for testing)
|
|
2461
4011
|
*/
|
|
@@ -2475,28 +4025,48 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2475
4025
|
// ============================================================================
|
|
2476
4026
|
|
|
2477
4027
|
/**
|
|
2478
|
-
* Initialize the seed bootstrap service with an
|
|
4028
|
+
* Initialize the seed bootstrap service with an owner key.
|
|
2479
4029
|
* Must be called before using seed-related methods that require signing.
|
|
2480
4030
|
*
|
|
2481
|
-
* @param
|
|
4031
|
+
* @param ownerPrivateKey - The owner's private key (base64url encoded)
|
|
2482
4032
|
*/
|
|
2483
|
-
initializeSeedBootstrap(
|
|
4033
|
+
initializeSeedBootstrap(ownerPrivateKey: string): void {
|
|
2484
4034
|
if (!this.controlNode || !this.controlDatabase) {
|
|
2485
4035
|
throw new Error('CadreNode must be started before initializing seed bootstrap');
|
|
2486
4036
|
}
|
|
2487
4037
|
|
|
4038
|
+
// A node wiring seed-bootstrap with an owner PRIVATE key is declaring that
|
|
4039
|
+
// key an authority of its own party (founder genesis / self-signing owner)
|
|
4040
|
+
// — anchor it in the node-local store. The in-memory set updates
|
|
4041
|
+
// synchronously (see TrustedOwnerStore.trust); only the file persist is
|
|
4042
|
+
// fire-and-forget, logged on failure.
|
|
4043
|
+
// NOTE: non-founder members that also wire seed-bootstrap with their own
|
|
4044
|
+
// derived key (e.g. the phone joiner path in runOwnerGenesis, which needs
|
|
4045
|
+
// it to self-publish its CadrePeer row) self-anchor a key that is not a
|
|
4046
|
+
// party authority. Harmless while such a node never mints an invite: the
|
|
4047
|
+
// store does not replicate, so a node trusting itself grants nothing to
|
|
4048
|
+
// others. But `createInvite` now hands out the anchor's contents as the
|
|
4049
|
+
// invitee's pins, so the moment a non-founder member mints an invite it
|
|
4050
|
+
// exports its own non-authority key as a cadre owner key. If that becomes
|
|
4051
|
+
// reachable (today only cadre-cli/cadre-host owners mint invites), gate this
|
|
4052
|
+
// self-anchor on the actual OwnerKey genesis insert instead.
|
|
4053
|
+
if (this.trustedOwnerStore) {
|
|
4054
|
+
void this.trustedOwnerStore
|
|
4055
|
+
.trust(ed25519PublicKeyFromPrivate(ownerPrivateKey), 'genesis')
|
|
4056
|
+
.catch((error) => log('Trusted-owner genesis anchor persist failed: %o', error));
|
|
4057
|
+
}
|
|
4058
|
+
|
|
2488
4059
|
this.seedBootstrapService = new SeedBootstrapService({
|
|
2489
4060
|
partyId: this.config.controlNetwork.partyId,
|
|
2490
|
-
|
|
4061
|
+
ownerPrivateKey,
|
|
2491
4062
|
inviteAddressResolver: () => this.resolveInviteAddresses(),
|
|
2492
4063
|
trustPolicy: this.config.seedTrustPolicy,
|
|
4064
|
+
// Seed trust anchors on the node-local store (seeded just above with this
|
|
4065
|
+
// node's own genesis key), never on the replicated OwnerKey table.
|
|
4066
|
+
...(this.trustedOwnerStore ? { trustedOwners: this.trustedOwnerStore } : {}),
|
|
2493
4067
|
});
|
|
2494
4068
|
|
|
2495
|
-
this.seedBootstrapService.setEventCallbacks(
|
|
2496
|
-
onSeedReceived: (partyId, peerId) => this.emit('seed:received', { partyId, peerId }),
|
|
2497
|
-
onSeedApplied: (partyId, peersAdded) => this.emit('seed:applied', { partyId, peersAdded }),
|
|
2498
|
-
onSeedError: (partyId, error) => this.emit('seed:error', { partyId, error }),
|
|
2499
|
-
});
|
|
4069
|
+
this.seedBootstrapService.setEventCallbacks(this.seedEventCallbacks());
|
|
2500
4070
|
|
|
2501
4071
|
this.seedBootstrapService.initialize(this.controlNode, this.controlDatabase);
|
|
2502
4072
|
log('Seed bootstrap service initialized');
|
|
@@ -2527,7 +4097,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2527
4097
|
}
|
|
2528
4098
|
|
|
2529
4099
|
/**
|
|
2530
|
-
* Enumerate the cadre's `CadrePeer` membership
|
|
4100
|
+
* Enumerate the cadre's `CadrePeer` membership — the ADDRESSABLE surface (see
|
|
4101
|
+
* {@link isMember}). Rows whose stamp is retired in `CadreControl.Revocation`
|
|
4102
|
+
* never reach here: {@link ControlDatabase.queryCadrePeers} excludes them, so a
|
|
4103
|
+
* revoked peer is not addressable either.
|
|
2531
4104
|
*/
|
|
2532
4105
|
async listMembers(): Promise<Array<{ peerId: string; multiaddr: string | null }>> {
|
|
2533
4106
|
if (!this.controlDatabase) {
|
|
@@ -2538,12 +4111,109 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2538
4111
|
|
|
2539
4112
|
/**
|
|
2540
4113
|
* Probe whether a given peer is a `CadrePeer` member.
|
|
4114
|
+
*
|
|
4115
|
+
* This is the ADDRESSABLE surface ("do I have a dialable address record for this
|
|
4116
|
+
* peer") — it includes this node's own self-published row. Address resolution and
|
|
4117
|
+
* push fan-out use this. A peer whose stamp is retired in `CadreControl.Revocation`
|
|
4118
|
+
* is excluded here too (the filter lives in
|
|
4119
|
+
* {@link ControlDatabase.queryCadrePeers}): revocation removes a peer from the
|
|
4120
|
+
* addressable surface, not only the trust-facing one, so a revoked peer is no
|
|
4121
|
+
* longer dialed, RPC'd, or handed out as an address. The trust-facing gate is
|
|
4122
|
+
* {@link isAuthorizedMember}.
|
|
2541
4123
|
*/
|
|
2542
4124
|
async isMember(peerId: string): Promise<boolean> {
|
|
2543
4125
|
const members = await this.listMembers();
|
|
2544
4126
|
return members.some(m => m.peerId === peerId);
|
|
2545
4127
|
}
|
|
2546
4128
|
|
|
4129
|
+
/**
|
|
4130
|
+
* Enumerate the party's AUTHORIZED members — the trust-facing set, distinct from
|
|
4131
|
+
* the addressable set ({@link listMembers}). A peer is authorized iff ALL hold:
|
|
4132
|
+
*
|
|
4133
|
+
* 1. it is not this node itself — a node publishes its own `CadrePeer` address
|
|
4134
|
+
* row so its dialable address rides in seeds, but "self" is not a peer this
|
|
4135
|
+
* node authorized;
|
|
4136
|
+
* 2. its `CadrePeer` row carries a complete voucher (`StampId`, `VouchOwner`,
|
|
4137
|
+
* `VouchSig` all non-null);
|
|
4138
|
+
* 3. `VouchOwner` is in the NODE-LOCAL trusted-owner anchor
|
|
4139
|
+
* ({@link getTrustedOwnerStore}) — never the replicated `OwnerKey` table,
|
|
4140
|
+
* which any stranger can genesis-pollute; and
|
|
4141
|
+
* 4. `VouchSig` verifies as that owner's signature over the row's voucher
|
|
4142
|
+
* digest ({@link verifyCadrePeerVoucher}), so the anchored owner really
|
|
4143
|
+
* vouched THIS peer id under THIS row's nonce; and
|
|
4144
|
+
* 5. the row's `StampId` is NOT retired in `CadreControl.Revocation` — enforced
|
|
4145
|
+
* upstream in {@link ControlDatabase.queryCadrePeers}, which drops retired rows
|
|
4146
|
+
* before ANY reader (this predicate included) sees them, so a row resurrected
|
|
4147
|
+
* by replaying the captured admission approval on a node that had not yet
|
|
4148
|
+
* converged on the tombstone (the write-time `NotRevoked` CHECK only sees
|
|
4149
|
+
* local rows) is still inert to every reader that has the tombstone.
|
|
4150
|
+
*
|
|
4151
|
+
* Fail-closed at every step: a missing anchor (pre-start), an empty anchor (a
|
|
4152
|
+
* not-yet-enrolled node authorizes no one), a null/partial voucher, an
|
|
4153
|
+
* unanchored `VouchOwner`, or a bad signature all yield "not authorized" —
|
|
4154
|
+
* having an address row is NOT membership. The control-network wake and
|
|
4155
|
+
* strand-address gates consult this set, NOT the addressable one.
|
|
4156
|
+
*
|
|
4157
|
+
* NOTE (rotation): if a party owner rotates keys and only the NEW key is pinned
|
|
4158
|
+
* in the anchor, rows the OLD key vouched fail check 3 until re-vouched — a
|
|
4159
|
+
* legit member goes un-authorized on readers that only pin the new key. Full
|
|
4160
|
+
* rotation handling (re-vouch on rotate) is the
|
|
4161
|
+
* `flip-strand-membership-rotation-known-gap` work, not this predicate's.
|
|
4162
|
+
*/
|
|
4163
|
+
async listAuthorizedMembers(): Promise<Array<{ peerId: string; multiaddr: string | null }>> {
|
|
4164
|
+
if (!this.controlDatabase) {
|
|
4165
|
+
throw new Error('CadreNode must be started before listing members');
|
|
4166
|
+
}
|
|
4167
|
+
const selfPeerId = this.peerId?.toString();
|
|
4168
|
+
const rows = await this.controlDatabase.queryCadrePeers();
|
|
4169
|
+
const authorized = rows
|
|
4170
|
+
.filter(row => row.peerId !== selfPeerId && this.hasAnchoredVoucher(row))
|
|
4171
|
+
.map(({ peerId, multiaddr }) => ({ peerId, multiaddr }));
|
|
4172
|
+
// A node whose anchor was never seeded (no invite pin, no operator pin, not a
|
|
4173
|
+
// founder) refuses every wake and strand-addr request, which from the outside
|
|
4174
|
+
// looks like an unexplained "non-member" rejection. Say so once per call rather
|
|
4175
|
+
// than leaving the operator to infer it from silence.
|
|
4176
|
+
if (authorized.length === 0 && rows.length > 0 && (this.trustedOwnerStore?.all().size ?? 0) === 0) {
|
|
4177
|
+
log('listAuthorizedMembers: %d CadrePeer row(s) but the node-local trusted-owner anchor is EMPTY — authorizing no one (this node has no invite/operator owner-key pin)', rows.length);
|
|
4178
|
+
}
|
|
4179
|
+
return authorized;
|
|
4180
|
+
}
|
|
4181
|
+
|
|
4182
|
+
/**
|
|
4183
|
+
* Checks 2–4 of the authorized-membership predicate (see
|
|
4184
|
+
* {@link listAuthorizedMembers}) for one `CadrePeer` row: complete voucher,
|
|
4185
|
+
* `VouchOwner` in the node-local anchor, signature valid over the row's
|
|
4186
|
+
* (PeerId, StampId) voucher digest.
|
|
4187
|
+
*
|
|
4188
|
+
* NOTE: verifies the ed25519 signature on every call (no memo of already-
|
|
4189
|
+
* verified (peerId, stampId, vouchSig) triples). Cadres are a handful of
|
|
4190
|
+
* devices and the gates run per inbound request, so this is cheap today; if
|
|
4191
|
+
* membership or gate traffic ever grows, cache verified triples keyed on the
|
|
4192
|
+
* row's `StampId` (which rotates with every re-vouch).
|
|
4193
|
+
*/
|
|
4194
|
+
private hasAnchoredVoucher(row: CadrePeerVoucherFields): boolean {
|
|
4195
|
+
if (row.stampId === null || row.vouchOwner === null || row.vouchSig === null) {
|
|
4196
|
+
return false;
|
|
4197
|
+
}
|
|
4198
|
+
if (!this.trustedOwnerStore?.has(row.vouchOwner)) {
|
|
4199
|
+
return false;
|
|
4200
|
+
}
|
|
4201
|
+
return verifyCadrePeerVoucher(row.peerId, row.stampId, row.vouchOwner, row.vouchSig);
|
|
4202
|
+
}
|
|
4203
|
+
|
|
4204
|
+
/**
|
|
4205
|
+
* Probe whether a given peer is an AUTHORIZED party member (see
|
|
4206
|
+
* {@link listAuthorizedMembers}) — the gate the control-network wake and
|
|
4207
|
+
* strand-address responders consult, NOT {@link isMember} (the addressable
|
|
4208
|
+
* surface). Deliberately scans the full membership (one `CadrePeer` query)
|
|
4209
|
+
* rather than adding a single-row read path: cadres are small, and one code
|
|
4210
|
+
* path keeps the predicate impossible to drift from the list.
|
|
4211
|
+
*/
|
|
4212
|
+
async isAuthorizedMember(peerId: string): Promise<boolean> {
|
|
4213
|
+
const members = await this.listAuthorizedMembers();
|
|
4214
|
+
return members.some(m => m.peerId === peerId);
|
|
4215
|
+
}
|
|
4216
|
+
|
|
2547
4217
|
/**
|
|
2548
4218
|
* Push-wake a hibernating cadre peer over the control network.
|
|
2549
4219
|
*
|
|
@@ -2573,8 +4243,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2573
4243
|
|
|
2574
4244
|
/**
|
|
2575
4245
|
* Enable the seed listener for receiving seeds via the /sereus/seed/1.0.0 protocol.
|
|
2576
|
-
* This is for drone nodes that need to receive seeds without being an
|
|
2577
|
-
* Does not require an
|
|
4246
|
+
* This is for drone nodes that need to receive seeds without being an owner.
|
|
4247
|
+
* Does not require an owner key.
|
|
2578
4248
|
*/
|
|
2579
4249
|
enableSeedListener(): void {
|
|
2580
4250
|
if (!this.controlNode || !this.controlDatabase) {
|
|
@@ -2589,16 +4259,16 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2589
4259
|
|
|
2590
4260
|
this.seedBootstrapService = new SeedBootstrapService({
|
|
2591
4261
|
partyId: this.config.controlNetwork.partyId,
|
|
2592
|
-
// No
|
|
4262
|
+
// No owner key - this node only receives seeds
|
|
2593
4263
|
inviteAddressResolver: () => this.resolveInviteAddresses(),
|
|
2594
4264
|
trustPolicy: this.config.seedTrustPolicy,
|
|
4265
|
+
// A listener-only node accepts a wire-delivered seed solely against this
|
|
4266
|
+
// anchor (there is no per-call override on the inbound handler): with no
|
|
4267
|
+
// genesis/invite/operator pin it authorizes nobody, which is the point.
|
|
4268
|
+
...(this.trustedOwnerStore ? { trustedOwners: this.trustedOwnerStore } : {}),
|
|
2595
4269
|
});
|
|
2596
4270
|
|
|
2597
|
-
this.seedBootstrapService.setEventCallbacks(
|
|
2598
|
-
onSeedReceived: (partyId, peerId) => this.emit('seed:received', { partyId, peerId }),
|
|
2599
|
-
onSeedApplied: (partyId, peersAdded) => this.emit('seed:applied', { partyId, peersAdded }),
|
|
2600
|
-
onSeedError: (partyId, error) => this.emit('seed:error', { partyId, error }),
|
|
2601
|
-
});
|
|
4271
|
+
this.seedBootstrapService.setEventCallbacks(this.seedEventCallbacks());
|
|
2602
4272
|
|
|
2603
4273
|
this.seedBootstrapService.initialize(this.controlNode, this.controlDatabase);
|
|
2604
4274
|
log('Seed listener enabled');
|
|
@@ -2611,9 +4281,126 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2611
4281
|
return this.seedBootstrapService;
|
|
2612
4282
|
}
|
|
2613
4283
|
|
|
4284
|
+
/**
|
|
4285
|
+
* The event callbacks every {@link SeedBootstrapService} this node owns is
|
|
4286
|
+
* wired with — shared by the owner-capable service ({@link initializeSeedBootstrap})
|
|
4287
|
+
* and the listener-only one ({@link enableSeedListener}) so the two cannot drift.
|
|
4288
|
+
*
|
|
4289
|
+
* A seed applied by the INBOUND protocol handler writes no `CadrePeer` row of
|
|
4290
|
+
* its own (it merges the libp2p peer store and dials owners), so the automatic
|
|
4291
|
+
* write-driven refresh never fires for it — yet applying it can ANCHOR a new
|
|
4292
|
+
* owner key, which flips rows already present from unauthorized to authorized.
|
|
4293
|
+
* Hence the explicit refresh here; without it a peer the freshly anchored owner
|
|
4294
|
+
* vouched for stays denied until the next timed cohort reconcile.
|
|
4295
|
+
*/
|
|
4296
|
+
private seedEventCallbacks(): SeedEventCallbacks {
|
|
4297
|
+
return {
|
|
4298
|
+
onSeedReceived: (partyId, peerId) => this.emit('seed:received', { partyId, peerId }),
|
|
4299
|
+
onSeedApplied: (partyId, peersAdded, seed) => {
|
|
4300
|
+
this.emit('seed:applied', { partyId, peersAdded });
|
|
4301
|
+
// A wire-delivered seed never passes through the `applySeed` wrapper, so
|
|
4302
|
+
// this is where its owner peers become cold-start bootstrap targets.
|
|
4303
|
+
this.recordSeedBootstrapPeers(seed);
|
|
4304
|
+
void this.refreshMembershipGate('seed-applied');
|
|
4305
|
+
},
|
|
4306
|
+
onSeedError: (partyId, error) => this.emit('seed:error', { partyId, error }),
|
|
4307
|
+
};
|
|
4308
|
+
}
|
|
4309
|
+
|
|
4310
|
+
/**
|
|
4311
|
+
* Re-materialize the authorized-peer snapshot that the fail-closed per-stream
|
|
4312
|
+
* control-DB gate ({@link authorizeInboundControlStream}) judges against.
|
|
4313
|
+
*
|
|
4314
|
+
* NO `CadrePeer` writer needs to call this: the control database notifies it
|
|
4315
|
+
* after every committed member-row write (`ControlDatabase.mutateCadrePeer`,
|
|
4316
|
+
* wired in {@link start}), which is exactly what makes the refresh automatic
|
|
4317
|
+
* rather than a caller obligation. It stays public for the changes that write
|
|
4318
|
+
* NO row locally and so raise no notification:
|
|
4319
|
+
*
|
|
4320
|
+
* - a membership row that arrived by REPLICATION (otherwise picked up on the
|
|
4321
|
+
* next timed cohort reconcile — bounded staleness by design), and
|
|
4322
|
+
* - a newly anchored trusted owner key ({@link applySeed}), which flips rows
|
|
4323
|
+
* ALREADY present from unauthorized to authorized without touching them.
|
|
4324
|
+
*
|
|
4325
|
+
* Coalescing: marks the snapshot stale and resolves once a refresh that began
|
|
4326
|
+
* after this call has completed, so a caller that awaits it always observes
|
|
4327
|
+
* its own change. Concurrent callers share one read. Idempotent and
|
|
4328
|
+
* best-effort (a failed read keeps the previous snapshot); never rejects.
|
|
4329
|
+
*/
|
|
4330
|
+
async refreshMembershipGate(reason = 'external-write'): Promise<void> {
|
|
4331
|
+
this.membershipGateDirty = true;
|
|
4332
|
+
if (this.membershipGateDeferDepth > 0) {
|
|
4333
|
+
// Flushed once when the enclosing scope exits.
|
|
4334
|
+
return;
|
|
4335
|
+
}
|
|
4336
|
+
while (this.membershipGateDirty) {
|
|
4337
|
+
// Re-checked after the await: the drain may have taken its last look at
|
|
4338
|
+
// the flag before we set it, in which case we need a fresh one.
|
|
4339
|
+
this.membershipGateDrain ??= this.drainMembershipGate(reason);
|
|
4340
|
+
await this.membershipGateDrain;
|
|
4341
|
+
}
|
|
4342
|
+
}
|
|
4343
|
+
|
|
4344
|
+
/**
|
|
4345
|
+
* The single in-flight refresh loop behind {@link refreshMembershipGate}: read
|
|
4346
|
+
* until the stale flag stays clear, then release the slot.
|
|
4347
|
+
*
|
|
4348
|
+
* NOTE: correct only because {@link membershipGateDirty} is guaranteed TRUE at
|
|
4349
|
+
* entry — its sole caller sets it synchronously, with no await in between. Were
|
|
4350
|
+
* this ever invoked with the flag already clear, the body would never suspend,
|
|
4351
|
+
* so the `finally` would null the slot BEFORE the caller's `??=` filled it, and
|
|
4352
|
+
* the slot would be left holding an already-settled drain that every later
|
|
4353
|
+
* refresh reuses — an unbreakable await/re-check spin. If a second caller is
|
|
4354
|
+
* ever added, have it mark the flag first (or assert it here).
|
|
4355
|
+
*/
|
|
4356
|
+
private async drainMembershipGate(reason: string): Promise<void> {
|
|
4357
|
+
try {
|
|
4358
|
+
while (this.membershipGateDirty) {
|
|
4359
|
+
// Cleared BEFORE the read (not after) so a write landing mid-read marks
|
|
4360
|
+
// the snapshot stale again and earns another pass, and so a throw still
|
|
4361
|
+
// terminates the loop.
|
|
4362
|
+
this.membershipGateDirty = false;
|
|
4363
|
+
await this.refreshAuthorizedControlPeers(reason);
|
|
4364
|
+
}
|
|
4365
|
+
} catch (error) {
|
|
4366
|
+
// Upholds `refreshMembershipGate`'s never-rejects contract. The read helper
|
|
4367
|
+
// already swallows its own failures, so this is defensive — but a rejection
|
|
4368
|
+
// here would otherwise surface out of every awaiting writer.
|
|
4369
|
+
log('membership gate drain (%s) failed — keeping previous snapshot: %o', reason, error);
|
|
4370
|
+
} finally {
|
|
4371
|
+
this.membershipGateDrain = null;
|
|
4372
|
+
}
|
|
4373
|
+
}
|
|
4374
|
+
|
|
4375
|
+
/**
|
|
4376
|
+
* Collapse a burst of `CadrePeer` writes into ONE gate refresh at scope exit.
|
|
4377
|
+
*
|
|
4378
|
+
* For loops that re-touch many rows (the write-while-alone drains), where a
|
|
4379
|
+
* per-row refresh would mean one full membership read per row for a snapshot
|
|
4380
|
+
* that only has to be correct once the loop settles.
|
|
4381
|
+
*
|
|
4382
|
+
* The depth counter is instance-level, so it also suppresses a genuinely
|
|
4383
|
+
* CONCURRENT external write's refresh for the life of the scope: that writer's
|
|
4384
|
+
* promise resolves before its peer is admitted, and admission lands at scope
|
|
4385
|
+
* exit instead. Acceptable — scope exit is milliseconds away (bounded by the
|
|
4386
|
+
* drain), versus the ~15 s reconcile interval that was the alternative before
|
|
4387
|
+
* any of this existed — but it is the reason these scopes stay short and rare.
|
|
4388
|
+
*/
|
|
4389
|
+
private async deferMembershipGateRefresh<T>(reason: string, body: () => Promise<T>): Promise<T> {
|
|
4390
|
+
this.membershipGateDeferDepth++;
|
|
4391
|
+
try {
|
|
4392
|
+
return await body();
|
|
4393
|
+
} finally {
|
|
4394
|
+
this.membershipGateDeferDepth--;
|
|
4395
|
+
if (this.membershipGateDeferDepth === 0 && this.membershipGateDirty) {
|
|
4396
|
+
await this.refreshMembershipGate(reason);
|
|
4397
|
+
}
|
|
4398
|
+
}
|
|
4399
|
+
}
|
|
4400
|
+
|
|
2614
4401
|
/**
|
|
2615
4402
|
* Authorize a new peer to join the cadre.
|
|
2616
|
-
* Signs the peer ID with the
|
|
4403
|
+
* Signs the peer ID with the owner key and inserts into CadrePeer table.
|
|
2617
4404
|
*
|
|
2618
4405
|
* @param peerId - The peer ID to authorize
|
|
2619
4406
|
* @param multiaddrs - Optional multiaddrs for the peer
|
|
@@ -2622,6 +4409,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2622
4409
|
if (!this.seedBootstrapService) {
|
|
2623
4410
|
throw new Error('Seed bootstrap service not initialized. Call initializeSeedBootstrap() first.');
|
|
2624
4411
|
}
|
|
4412
|
+
// The insert notifies the control DB's membership hub, so the per-stream gate
|
|
4413
|
+
// has already admitted the just-vouched peer by the time this resolves.
|
|
2625
4414
|
await this.seedBootstrapService.authorizePeer({ peerId, multiaddrs });
|
|
2626
4415
|
// Queue for re-replication if this committed local-only (no connected cohort).
|
|
2627
4416
|
this.noteControlWrite(peerId, 'authorize');
|
|
@@ -2629,7 +4418,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2629
4418
|
|
|
2630
4419
|
/**
|
|
2631
4420
|
* Remove a previously-authorized peer from the cadre.
|
|
2632
|
-
* Signs the peer ID with the
|
|
4421
|
+
* Signs the peer ID with the owner key and deletes the CadrePeer row.
|
|
2633
4422
|
*
|
|
2634
4423
|
* @param peerId - The peer ID to remove
|
|
2635
4424
|
*/
|
|
@@ -2637,6 +4426,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2637
4426
|
if (!this.seedBootstrapService) {
|
|
2638
4427
|
throw new Error('Seed bootstrap service not initialized. Call initializeSeedBootstrap() first.');
|
|
2639
4428
|
}
|
|
4429
|
+
// The delete notifies the membership hub, so the removed peer is out of the
|
|
4430
|
+
// per-stream gate by the time this resolves.
|
|
2640
4431
|
await this.seedBootstrapService.removePeer(peerId);
|
|
2641
4432
|
// Track + loudly flag a delete that committed local-only (security-relevant).
|
|
2642
4433
|
this.noteControlWrite(peerId, 'remove');
|
|
@@ -2644,7 +4435,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2644
4435
|
|
|
2645
4436
|
/**
|
|
2646
4437
|
* Create a seed from the current control network state.
|
|
2647
|
-
* The seed contains peer information and is signed by an
|
|
4438
|
+
* The seed contains peer information and is signed by an owner.
|
|
2648
4439
|
*/
|
|
2649
4440
|
async createSeed(): Promise<ControlNetworkSeed> {
|
|
2650
4441
|
if (!this.seedBootstrapService) {
|
|
@@ -2659,7 +4450,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2659
4450
|
* Validates the seed signature, then evaluates a trust anchor for the signer
|
|
2660
4451
|
* key (see `SeedTrustPolicy`). An enrollment caller can pass a per-seed
|
|
2661
4452
|
* `trustPolicy` override — e.g. a `pinnedKeyTrustPolicy` built from a
|
|
2662
|
-
* `CadreInvite.
|
|
4453
|
+
* `CadreInvite.ownerKeys` — so a cold-start node can accept its first
|
|
2663
4454
|
* seed without reconfiguring the service.
|
|
2664
4455
|
*/
|
|
2665
4456
|
async applySeed(
|
|
@@ -2667,7 +4458,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2667
4458
|
options?: { trustPolicy?: SeedTrustPolicy }
|
|
2668
4459
|
): Promise<ApplySeedResult> {
|
|
2669
4460
|
if (!this.seedBootstrapService) {
|
|
2670
|
-
// Create a temporary service for applying seeds (doesn't need
|
|
4461
|
+
// Create a temporary service for applying seeds (doesn't need owner key).
|
|
2671
4462
|
// partyId is the attacker-influenced seed.partyId — it only labels logs; the
|
|
2672
4463
|
// trust decision rests solely on signerKey vs the anchor set (configured
|
|
2673
4464
|
// default below, or the per-call options.trustPolicy override).
|
|
@@ -2681,13 +4472,43 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2681
4472
|
const tempService = new SeedBootstrapService({
|
|
2682
4473
|
partyId: seed.partyId,
|
|
2683
4474
|
trustPolicy: this.config.seedTrustPolicy,
|
|
4475
|
+
// The anchor is node-scoped, not service-scoped: a throwaway service
|
|
4476
|
+
// must consult (and persist an accepted signer into) the SAME store the
|
|
4477
|
+
// persistent one would, or a cold-start enrollment via this path would
|
|
4478
|
+
// anchor nothing and re-prompt for the pin on every later seed.
|
|
4479
|
+
...(this.trustedOwnerStore ? { trustedOwners: this.trustedOwnerStore } : {}),
|
|
2684
4480
|
});
|
|
2685
4481
|
if (this.controlNode && this.controlDatabase) {
|
|
2686
4482
|
tempService.initialize(this.controlNode, this.controlDatabase, { registerHandler: false });
|
|
2687
4483
|
}
|
|
2688
|
-
|
|
4484
|
+
const tempResult = await tempService.applySeed(seed, options);
|
|
4485
|
+
this.noteAppliedSeed(tempResult, seed);
|
|
4486
|
+
// Not a row write (so nothing notifies): applying a seed can anchor a new
|
|
4487
|
+
// owner key, which flips rows ALREADY present into the authorized set.
|
|
4488
|
+
await this.refreshMembershipGate('applySeed');
|
|
4489
|
+
return tempResult;
|
|
4490
|
+
}
|
|
4491
|
+
const result = await this.seedBootstrapService.applySeed(seed, options);
|
|
4492
|
+
this.noteAppliedSeed(result, seed);
|
|
4493
|
+
await this.refreshMembershipGate('applySeed');
|
|
4494
|
+
return result;
|
|
4495
|
+
}
|
|
4496
|
+
|
|
4497
|
+
/**
|
|
4498
|
+
* Post-process an {@link applySeed} outcome: retain the seed's owner peers as
|
|
4499
|
+
* cold-start bootstrap targets, and surface a seed that was accepted but whose
|
|
4500
|
+
* every owner dial failed — the node is now seeded yet unconnected, and the
|
|
4501
|
+
* cold-start reconcile branch is what gets it out of that state.
|
|
4502
|
+
*/
|
|
4503
|
+
private noteAppliedSeed(result: ApplySeedResult, seed: ControlNetworkSeed): void {
|
|
4504
|
+
if (!result.success) {
|
|
4505
|
+
return;
|
|
4506
|
+
}
|
|
4507
|
+
this.recordSeedBootstrapPeers(seed);
|
|
4508
|
+
if (result.ownerDialsAttempted > 0 && result.ownerDialsFailed === result.ownerDialsAttempted) {
|
|
4509
|
+
log('applySeed: seeded but no owner reachable (%d/%d dial(s) failed) — cold-start reconcile will retry',
|
|
4510
|
+
result.ownerDialsFailed, result.ownerDialsAttempted);
|
|
2689
4511
|
}
|
|
2690
|
-
return await this.seedBootstrapService.applySeed(seed, options);
|
|
2691
4512
|
}
|
|
2692
4513
|
|
|
2693
4514
|
/**
|
|
@@ -2745,6 +4566,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2745
4566
|
if (!this.seedBootstrapService) {
|
|
2746
4567
|
throw new Error('Seed bootstrap service not initialized. Call initializeSeedBootstrap() first.');
|
|
2747
4568
|
}
|
|
4569
|
+
// The drone's `CadrePeer` insert notifies the membership hub on the way to the
|
|
4570
|
+
// seed, so the per-stream gate already admits it here.
|
|
2748
4571
|
return await this.seedBootstrapService.addDrone(options);
|
|
2749
4572
|
}
|
|
2750
4573
|
|
|
@@ -2756,7 +4579,12 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2756
4579
|
if (!this.seedBootstrapService) {
|
|
2757
4580
|
throw new Error('Seed bootstrap service not initialized. Call initializeSeedBootstrap() first.');
|
|
2758
4581
|
}
|
|
2759
|
-
|
|
4582
|
+
const result = await this.seedBootstrapService.createInvite(token, expiresIn);
|
|
4583
|
+
// The invitee dials this node BEFORE it is authorized (dialInvite →
|
|
4584
|
+
// acceptPhone), so hold the inbound connection gate open for strangers for
|
|
4585
|
+
// the invite's validity (or a bounded default when it never expires).
|
|
4586
|
+
this.openEnrollmentWindow(result.invite.expiresAt ?? Date.now() + DEFAULT_ENROLLMENT_WINDOW_MS);
|
|
4587
|
+
return result;
|
|
2760
4588
|
}
|
|
2761
4589
|
|
|
2762
4590
|
/**
|
|
@@ -2767,6 +4595,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2767
4595
|
if (!this.seedBootstrapService) {
|
|
2768
4596
|
throw new Error('Seed bootstrap service not initialized. Call initializeSeedBootstrap() first.');
|
|
2769
4597
|
}
|
|
4598
|
+
// The just-accepted phone opens control-DB streams next; its `CadrePeer` insert
|
|
4599
|
+
// notifies the membership hub, so it is already admitted when this resolves.
|
|
2770
4600
|
await this.seedBootstrapService.acceptPhone(options, issuedInvite);
|
|
2771
4601
|
}
|
|
2772
4602
|
|
|
@@ -2778,6 +4608,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2778
4608
|
if (!this.seedBootstrapService) {
|
|
2779
4609
|
throw new Error('Seed bootstrap service not initialized. Call initializeSeedBootstrap() first.');
|
|
2780
4610
|
}
|
|
4611
|
+
// The service authorizes the phone (a `CadrePeer` insert, which notifies the
|
|
4612
|
+
// membership hub) on the way to the seed, so the per-stream gate admits it
|
|
4613
|
+
// before this resolves — not on the next reconcile, by which time the phone's
|
|
4614
|
+
// own control-DB schema load may have died denied.
|
|
2781
4615
|
return await this.seedBootstrapService.addPhoneWithRelay(phonePeerId);
|
|
2782
4616
|
}
|
|
2783
4617
|
|
|
@@ -2799,7 +4633,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2799
4633
|
}
|
|
2800
4634
|
|
|
2801
4635
|
/**
|
|
2802
|
-
* Dial an
|
|
4636
|
+
* Dial an owner from an invite (for phone joining via invite).
|
|
2803
4637
|
*/
|
|
2804
4638
|
async dialInvite(invite: CadreInvite): Promise<void> {
|
|
2805
4639
|
if (!this.seedBootstrapService) {
|