@serfab/cadre-core 1.9.0 → 1.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cadre-node.d.ts +148 -44
- package/dist/cadre-node.js +363 -139
- package/dist/cadre-node.js.map +1 -1
- package/dist/control-authorization.d.ts +2 -2
- package/dist/control-authorization.js +1 -0
- package/dist/control-authorization.js.map +1 -1
- package/dist/control-cohort.d.ts +1 -1
- package/dist/control-cohort.js +1 -1
- package/dist/control-database.d.ts +105 -14
- package/dist/control-database.js +242 -73
- package/dist/control-database.js.map +1 -1
- package/dist/control-formation-recorder.d.ts +6 -6
- package/dist/control-formation-recorder.js +7 -8
- package/dist/control-formation-recorder.js.map +1 -1
- package/dist/control-schema.d.ts +1 -1
- package/dist/control-schema.js +1066 -979
- package/dist/control-schema.js.map +1 -1
- package/dist/control-write-retry.d.ts +21 -15
- package/dist/control-write-retry.js +21 -15
- package/dist/control-write-retry.js.map +1 -1
- package/dist/diagnostics/connection-path.d.ts +1 -18
- package/dist/diagnostics/connection-path.js +1 -20
- package/dist/diagnostics/connection-path.js.map +1 -1
- package/dist/identity-key.d.ts +0 -30
- package/dist/identity-key.js +7 -40
- package/dist/identity-key.js.map +1 -1
- package/dist/index.d.ts +8 -5
- package/dist/index.js +12 -6
- package/dist/index.js.map +1 -1
- package/dist/invitation-bootstrap.d.ts +32 -0
- package/dist/invitation-bootstrap.js +30 -0
- package/dist/invitation-bootstrap.js.map +1 -0
- package/dist/link-budget.d.ts +204 -119
- package/dist/link-budget.js +208 -120
- package/dist/link-budget.js.map +1 -1
- package/dist/membership-connection-gater.d.ts +3 -3
- package/dist/membership-connection-gater.js +3 -3
- package/dist/peer-dial.d.ts +18 -14
- package/dist/peer-dial.js +18 -14
- package/dist/peer-dial.js.map +1 -1
- package/dist/peer-join-backfill.d.ts +3 -3
- package/dist/pending-join-runner.d.ts +186 -0
- package/dist/pending-join-runner.js +649 -0
- package/dist/pending-join-runner.js.map +1 -0
- package/dist/push-fanout.d.ts +1 -1
- package/dist/push-fanout.js +1 -1
- package/dist/relay-stun.d.ts +24 -0
- package/dist/relay-stun.js +51 -0
- package/dist/relay-stun.js.map +1 -0
- package/dist/seed-bootstrap.d.ts +2 -2
- package/dist/seed-bootstrap.js +5 -5
- package/dist/strand-addr-protocol.js +2 -2
- package/dist/strand-formation-deadlines.d.ts +5 -5
- package/dist/strand-formation-deadlines.js +5 -5
- package/dist/strand-formation-manager.d.ts +18 -4
- package/dist/strand-formation-manager.js +48 -33
- package/dist/strand-formation-manager.js.map +1 -1
- package/dist/strand-formation-protocol.d.ts +71 -14
- package/dist/strand-formation-protocol.js +222 -47
- package/dist/strand-formation-protocol.js.map +1 -1
- package/dist/strand-formation-rejection.d.ts +94 -0
- package/dist/strand-formation-rejection.js +77 -0
- package/dist/strand-formation-rejection.js.map +1 -0
- package/dist/strand-instance-manager.js +4 -1
- package/dist/strand-instance-manager.js.map +1 -1
- package/dist/strand-membership-writer.js +32 -32
- package/dist/strand-solicitation.d.ts +11 -3
- package/dist/strand-solicitation.js +9 -0
- package/dist/strand-solicitation.js.map +1 -1
- package/dist/types.d.ts +103 -17
- package/dist/types.js +5 -9
- package/dist/types.js.map +1 -1
- package/package.json +22 -22
- package/src/cadre-node.ts +8654 -8400
- package/src/control-authorization.ts +2 -1
- package/src/control-cohort.ts +128 -128
- package/src/control-database.ts +303 -82
- package/src/control-formation-recorder.ts +353 -354
- package/src/control-read-retry.ts +216 -216
- package/src/control-schema.ts +1078 -991
- package/src/control-write-retry.ts +567 -561
- package/src/diagnostics/connection-path.ts +1 -29
- package/src/formation-approval.ts +610 -610
- package/src/hibernation-manager.ts +638 -638
- package/src/identity-key.ts +7 -62
- package/src/index.ts +644 -627
- package/src/invitation-bootstrap.ts +40 -0
- package/src/joined-strand-store.ts +498 -498
- package/src/link-budget.ts +684 -583
- package/src/membership-connection-gater.ts +570 -570
- package/src/node-local-snapshot.ts +298 -298
- package/src/peer-dial.ts +315 -311
- package/src/peer-join-backfill.ts +710 -710
- package/src/pending-join-runner.ts +735 -0
- package/src/push-fanout.ts +1 -1
- package/src/relay-reservation.ts +1121 -1121
- package/src/relay-server.ts +98 -98
- package/src/relay-stun.ts +58 -0
- package/src/seed-bootstrap.ts +1448 -1448
- package/src/strand-addr-protocol.ts +547 -547
- package/src/strand-database.ts +287 -287
- package/src/strand-first-sync-gate.ts +391 -391
- package/src/strand-formation-deadlines.ts +227 -227
- package/src/strand-formation-manager.ts +725 -700
- package/src/strand-formation-protocol.ts +1036 -802
- package/src/strand-formation-rejection.ts +129 -0
- package/src/strand-fret-addrs.ts +145 -145
- package/src/strand-instance-manager.ts +1789 -1786
- package/src/strand-membership-writer.ts +1974 -1974
- package/src/strand-network-state-file.ts +40 -40
- package/src/strand-network-state.ts +243 -243
- package/src/strand-revocation-enforcer.ts +654 -654
- package/src/strand-solicitation.ts +532 -519
- package/src/strand-wake-protocol.ts +401 -401
- package/src/types.ts +2222 -2140
- package/dist/diagnostics/webrtc-turn-tracker.d.ts +0 -50
- package/dist/diagnostics/webrtc-turn-tracker.js +0 -196
- package/dist/diagnostics/webrtc-turn-tracker.js.map +0 -1
- package/src/diagnostics/webrtc-turn-tracker.ts +0 -222
|
@@ -1,401 +1,401 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Control-network push-wake transport.
|
|
3
|
-
*
|
|
4
|
-
* Lets a same-cadre peer — typically an always-on server that participates in a
|
|
5
|
-
* strand and sees new activity — signal a hibernating cadre peer to bring that
|
|
6
|
-
* strand online, pull pending activity, and re-hibernate. Push-wake travels the
|
|
7
|
-
* **control network** (the per-party network connecting this party's own cadre
|
|
8
|
-
* nodes), which is the only network a hibernating peer keeps connected.
|
|
9
|
-
*
|
|
10
|
-
* Modeled directly on `seed-bootstrap.ts`: a dedicated libp2p protocol id,
|
|
11
|
-
* 4-byte big-endian length-prefixed JSON frames, `node.handle` for the receiver,
|
|
12
|
-
* `node.dialProtocol` for the sender, and the shared `ControlStream` primitives
|
|
13
|
-
* from `control-stream.ts`. The exchange is a single request → single ack on one
|
|
14
|
-
* stream (like seed delivery), so each side reads to EOF (under a read timeout)
|
|
15
|
-
* and decodes one frame via the shared {@link decodeLengthPrefixedFrame} guard.
|
|
16
|
-
*
|
|
17
|
-
* **Authorization (v1):** a wake is low-risk — it only causes the receiver to
|
|
18
|
-
* spend resources coming online for a strand it already participates in — so the
|
|
19
|
-
* receiver carries no per-request signature and instead defers entirely to the
|
|
20
|
-
* injected `isMember` predicate. `CadreNode` injects its AUTHORIZED-membership
|
|
21
|
-
* predicate there (`isAuthorizedMember`: the sender's `CadrePeer` row must carry
|
|
22
|
-
* a voucher that verifies against an owner key in the receiver's node-local
|
|
23
|
-
* trusted-owner anchor), so a peer that merely published rows into the replicated
|
|
24
|
-
* control DB is refused. This module stays agnostic: it enforces whatever
|
|
25
|
-
* predicate it is given.
|
|
26
|
-
*/
|
|
27
|
-
|
|
28
|
-
import debug from 'debug';
|
|
29
|
-
import type { Libp2p, Connection } from '@libp2p/interface';
|
|
30
|
-
import type { Multiaddr } from '@multiformats/multiaddr';
|
|
31
|
-
import type { StrandInstance, WakeRequest, WakeAck } from './types.js';
|
|
32
|
-
import { decodeLengthPrefixedFrame } from './seed-bootstrap.js';
|
|
33
|
-
import { type ControlStream, exchangeFrame, readStreamToEnd, replyAndClose } from './control-stream.js';
|
|
34
|
-
import { tryAddrsInTurn } from './peer-dial.js';
|
|
35
|
-
import { relayedRequestBudgetMs } from './link-budget.js';
|
|
36
|
-
|
|
37
|
-
const log = debug('sereus:cadre:strand-wake');
|
|
38
|
-
|
|
39
|
-
/** Protocol id for control-network push-wake (parallel to `/sereus/seed/1.0.0`). */
|
|
40
|
-
export const WAKE_PROTOCOL = '/sereus/strand-wake/1.0.0';
|
|
41
|
-
|
|
42
|
-
/**
|
|
43
|
-
* Maximum wake frame size. Wake messages are tiny (a strand id + short reason),
|
|
44
|
-
* so this is a defensive cap — far below the 1MB seed ceiling — that bounds the
|
|
45
|
-
* bytes a peer can make the receiver buffer per stream.
|
|
46
|
-
*/
|
|
47
|
-
const MAX_WAKE_SIZE = 64 * 1024;
|
|
48
|
-
|
|
49
|
-
/**
|
|
50
|
-
* Default deadline for ONE wake attempt — dial, request, ack — at the default declared link
|
|
51
|
-
* (ms): `relayedRequestBudgetMs` in `link-budget.ts`, because the target may be reachable
|
|
52
|
-
* only through a relay and the attempt has to open that connection before the exchange.
|
|
53
|
-
* {@link dialWake} derives it from {@link DialWakeOptions.linkRoundTripMs} instead when given.
|
|
54
|
-
*
|
|
55
|
-
* It holds only link work because the receiver acks as soon as it has DECIDED, before the
|
|
56
|
-
* wake itself runs ({@link StrandWakeService.processWakeRequest}).
|
|
57
|
-
*
|
|
58
|
-
* NOTE: the receiver's membership check (two live control reads) runs inside this deadline and
|
|
59
|
-
* is not counted. In steady state those reads touch only held blocks and do not consult the
|
|
60
|
-
* cohort. If a wake or address request is seen timing out while the receiver's membership read
|
|
61
|
-
* is consulting, count one membership decision in this deadline or answer the check from the
|
|
62
|
-
* materialized authorized-peer snapshot. See docs/cadre-consistency.md → "Deadlines Over
|
|
63
|
-
* Optimystic's Reads and Commits".
|
|
64
|
-
*/
|
|
65
|
-
export const DEFAULT_WAKE_TIMEOUT_MS = relayedRequestBudgetMs();
|
|
66
|
-
|
|
67
|
-
/**
|
|
68
|
-
* Whole attempts one {@link dialWake} call budgets for: a stale signaling address, then the
|
|
69
|
-
* direct address that works.
|
|
70
|
-
*/
|
|
71
|
-
export const WAKE_DIAL_ATTEMPTS = 2;
|
|
72
|
-
|
|
73
|
-
/**
|
|
74
|
-
* Default budget for a WHOLE {@link dialWake} call at the default declared link, in ms —
|
|
75
|
-
* every candidate address together, not each one: {@link WAKE_DIAL_ATTEMPTS} attempt
|
|
76
|
-
* deadlines.
|
|
77
|
-
*
|
|
78
|
-
* Without it the cost of a wake is (candidate count × {@link
|
|
79
|
-
* DEFAULT_WAKE_TIMEOUT_MS}), a number nothing chooses or bounds. That is not
|
|
80
|
-
* hypothetical: an address behind a dropped NAT mapping, or any host that
|
|
81
|
-
* blackholes rather than sending a RST, burns its full attempt timeout instead
|
|
82
|
-
* of failing in milliseconds, so a five-address peer would cost five attempts.
|
|
83
|
-
*
|
|
84
|
-
* This makes the TARGET PEER the unit rather than the address — the same
|
|
85
|
-
* decision, for the same reason, as
|
|
86
|
-
* `DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS` in `peer-dial.ts`, though not the same
|
|
87
|
-
* count: that one is sized to fit several dead addresses ahead of a working one,
|
|
88
|
-
* while a wake target is expected to be awake and reachable on its first or second
|
|
89
|
-
* address. Two whole attempts so a reachable peer whose signaling address is stale
|
|
90
|
-
* still gets a genuine try at its direct one; a peer needing longer than that is
|
|
91
|
-
* not "asleep and reachable", which is the only case a wake is for. The last
|
|
92
|
-
* attempt inside the budget gets whatever remains of it, so the call returns at
|
|
93
|
-
* the budget, not past it.
|
|
94
|
-
*
|
|
95
|
-
* Override per call with {@link DialWakeOptions.budgetMs} — tests that drive
|
|
96
|
-
* dead addresses on purpose set it low so a dial's duration is a chosen number
|
|
97
|
-
* rather than a transitive libp2p default stretched by machine load.
|
|
98
|
-
*/
|
|
99
|
-
export const DEFAULT_WAKE_DIAL_BUDGET_MS = WAKE_DIAL_ATTEMPTS * DEFAULT_WAKE_TIMEOUT_MS;
|
|
100
|
-
|
|
101
|
-
/** Default time the receiver waits for an inbound wake frame before aborting (ms). */
|
|
102
|
-
// eslint-disable-next-line no-restricted-syntax -- link-independent: a receiver cap on one small request frame on a stream the peer already opened; it bounds a peer that opens a stream and never sends, not the dial
|
|
103
|
-
const DEFAULT_WAKE_READ_TIMEOUT_MS = 10_000;
|
|
104
|
-
|
|
105
|
-
/** Default cap on concurrent inbound wake streams a single peer can pin open. */
|
|
106
|
-
const DEFAULT_MAX_CONCURRENT_WAKES = 100;
|
|
107
|
-
|
|
108
|
-
/**
|
|
109
|
-
* Read a libp2p stream to EOF and decode the single length-prefixed JSON frame
|
|
110
|
-
* it carries. Bounded by `timeoutMs` (a never-half-closing peer is aborted, not
|
|
111
|
-
* awaited forever) and capped at {@link MAX_WAKE_SIZE}, reusing the shared
|
|
112
|
-
* {@link decodeLengthPrefixedFrame} guard for the prefix/length checks.
|
|
113
|
-
*/
|
|
114
|
-
async function readFrame<T>(stream: ControlStream, timeoutMs: number): Promise<T> {
|
|
115
|
-
const data = await readStreamToEnd(stream, { maxBytes: MAX_WAKE_SIZE, timeoutMs, label: 'Wake' });
|
|
116
|
-
const body = decodeLengthPrefixedFrame(data, MAX_WAKE_SIZE);
|
|
117
|
-
return JSON.parse(new TextDecoder().decode(body)) as T;
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
/**
|
|
121
|
-
* Dependencies the {@link StrandWakeService} receiver needs from its host
|
|
122
|
-
* (`CadreNode`), injected so the service is testable without a full node.
|
|
123
|
-
*/
|
|
124
|
-
export interface StrandWakeServiceOptions {
|
|
125
|
-
/** Membership gate: is the remote peer a `CadrePeer` member of this cadre? */
|
|
126
|
-
isMember(remotePeerId: string): Promise<boolean>;
|
|
127
|
-
/** Look up a local strand instance by id (undefined if not participated in). */
|
|
128
|
-
getStrand(strandId: string): StrandInstance | undefined;
|
|
129
|
-
/**
|
|
130
|
-
* Trigger the local wake path for a hibernating/idle strand. Wired to
|
|
131
|
-
* `CadreNode.wakeStrand` (→ `HibernationManager` → `resumeStrand`); `resumeStrand`
|
|
132
|
-
* joins any resume already in flight, so a push-wake racing a check-in shares its
|
|
133
|
-
* runtime build.
|
|
134
|
-
*
|
|
135
|
-
* Started, not awaited: the ack goes back before the strand is up, and a wake
|
|
136
|
-
* that fails afterwards is logged here rather than reported to the sender.
|
|
137
|
-
*/
|
|
138
|
-
wake(strandId: string): Promise<void>;
|
|
139
|
-
/**
|
|
140
|
-
* Time to wait for the inbound wake frame before aborting the read (ms).
|
|
141
|
-
* Defaults to {@link DEFAULT_WAKE_READ_TIMEOUT_MS}. Bounds a buggy/compromised
|
|
142
|
-
* own-cadre node that opens a stream and never half-closes its write end.
|
|
143
|
-
*/
|
|
144
|
-
readTimeoutMs?: number;
|
|
145
|
-
/**
|
|
146
|
-
* Cap on concurrent inbound wake streams (defaults to
|
|
147
|
-
* {@link DEFAULT_MAX_CONCURRENT_WAKES}). Over the cap, a non-accepting ack is
|
|
148
|
-
* returned without invoking the wake path.
|
|
149
|
-
*
|
|
150
|
-
* It counts streams, not wakes: a wake started after its ack is no longer
|
|
151
|
-
* counted. Those are bounded by the strands this node participates in instead,
|
|
152
|
-
* since an unknown strand is refused before any wake starts and the wake path
|
|
153
|
-
* coalesces per strand.
|
|
154
|
-
*/
|
|
155
|
-
maxConcurrent?: number;
|
|
156
|
-
}
|
|
157
|
-
|
|
158
|
-
/**
|
|
159
|
-
* Receiver side of the push-wake protocol. Registers a `WAKE_PROTOCOL` handler
|
|
160
|
-
* on the control node and, for each inbound {@link WakeRequest}, gates on cadre
|
|
161
|
-
* membership, replies with a {@link WakeAck}, and starts resuming the named
|
|
162
|
-
* strand if it is hibernating/idle and we participate in it.
|
|
163
|
-
*/
|
|
164
|
-
export class StrandWakeService {
|
|
165
|
-
private readonly options: StrandWakeServiceOptions;
|
|
166
|
-
private readonly readTimeoutMs: number;
|
|
167
|
-
private readonly maxConcurrent: number;
|
|
168
|
-
private node: Libp2p | null = null;
|
|
169
|
-
/** In-flight inbound wake streams, used to enforce {@link maxConcurrent}. */
|
|
170
|
-
private activeStreams = 0;
|
|
171
|
-
|
|
172
|
-
constructor(options: StrandWakeServiceOptions) {
|
|
173
|
-
this.options = options;
|
|
174
|
-
this.readTimeoutMs = options.readTimeoutMs ?? DEFAULT_WAKE_READ_TIMEOUT_MS;
|
|
175
|
-
this.maxConcurrent = options.maxConcurrent ?? DEFAULT_MAX_CONCURRENT_WAKES;
|
|
176
|
-
}
|
|
177
|
-
|
|
178
|
-
/** Number of in-flight inbound wake streams. */
|
|
179
|
-
get activeCount(): number {
|
|
180
|
-
return this.activeStreams;
|
|
181
|
-
}
|
|
182
|
-
|
|
183
|
-
/**
|
|
184
|
-
* Register the wake protocol handler on the control node. Rejects when libp2p
|
|
185
|
-
* refuses the registration; the node reference is kept only once the handler
|
|
186
|
-
* is in place.
|
|
187
|
-
*/
|
|
188
|
-
async initialize(node: Libp2p): Promise<void> {
|
|
189
|
-
// `runOnLimitedConnection: true` is REQUIRED for the relay path: a NAT'd
|
|
190
|
-
// receiver is reached over a circuit-relay connection, which libp2p marks
|
|
191
|
-
// "limited" (the relay caps its data/duration). Without this the receiver
|
|
192
|
-
// would refuse the inbound wake stream on exactly the connection the
|
|
193
|
-
// protocol is designed to use (see the relay note on `dialWake`).
|
|
194
|
-
await node.handle(WAKE_PROTOCOL, async (rawStream: unknown, rawConnection: unknown) => {
|
|
195
|
-
const remotePeerId = (rawConnection as Connection).remotePeer.toString();
|
|
196
|
-
await this.handleStream(rawStream as ControlStream, remotePeerId);
|
|
197
|
-
}, { runOnLimitedConnection: true });
|
|
198
|
-
this.node = node;
|
|
199
|
-
log('StrandWakeService registered handler: %s', WAKE_PROTOCOL);
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
/** Unregister the handler and release the node reference. */
|
|
203
|
-
async shutdown(): Promise<void> {
|
|
204
|
-
if (this.node) {
|
|
205
|
-
await this.node.unhandle(WAKE_PROTOCOL);
|
|
206
|
-
this.node = null;
|
|
207
|
-
log('StrandWakeService shutdown');
|
|
208
|
-
}
|
|
209
|
-
}
|
|
210
|
-
|
|
211
|
-
/**
|
|
212
|
-
* Read the inbound request, decide the wake, and write the ack.
|
|
213
|
-
*
|
|
214
|
-
* Three hardening layers, all reported as a non-accepting ack rather than a
|
|
215
|
-
* dropped/hung stream: a concurrency cap (over {@link maxConcurrent}, reply
|
|
216
|
-
* without touching the wake path), a read timeout (a peer that never
|
|
217
|
-
* half-closes is aborted inside {@link readFrame}/`readStreamToEnd`), and the
|
|
218
|
-
* existing malformed/oversized-frame guard.
|
|
219
|
-
*/
|
|
220
|
-
private async handleStream(stream: ControlStream, remotePeerId: string): Promise<void> {
|
|
221
|
-
log('Incoming wake request from: %s', remotePeerId);
|
|
222
|
-
|
|
223
|
-
if (this.activeStreams >= this.maxConcurrent) {
|
|
224
|
-
log('Rejecting wake from %s: %d concurrent streams at cap %d', remotePeerId, this.activeStreams, this.maxConcurrent);
|
|
225
|
-
await replyAndClose(stream, { accepted: false, reason: 'Too many concurrent wake requests' } satisfies WakeAck, 'Wake');
|
|
226
|
-
return;
|
|
227
|
-
}
|
|
228
|
-
|
|
229
|
-
this.activeStreams++;
|
|
230
|
-
try {
|
|
231
|
-
await replyAndClose(stream, await this.answerStream(stream, remotePeerId), 'Wake');
|
|
232
|
-
} finally {
|
|
233
|
-
this.activeStreams--;
|
|
234
|
-
}
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
/** Read and decide one inbound request; any failure becomes a non-accepting ack. */
|
|
238
|
-
private async answerStream(stream: ControlStream, remotePeerId: string): Promise<WakeAck> {
|
|
239
|
-
try {
|
|
240
|
-
const request = await readFrame<WakeRequest>(stream, this.readTimeoutMs);
|
|
241
|
-
return await this.processWakeRequest(request, remotePeerId);
|
|
242
|
-
} catch (err) {
|
|
243
|
-
log('Error handling wake request from %s: %o', remotePeerId, err);
|
|
244
|
-
return { accepted: false, reason: err instanceof Error ? err.message : 'Unknown error' };
|
|
245
|
-
}
|
|
246
|
-
}
|
|
247
|
-
|
|
248
|
-
/**
|
|
249
|
-
* Decide the wake for a decoded request and start it. Exposed (not private) so
|
|
250
|
-
* the decision matrix can be unit-tested directly.
|
|
251
|
-
*
|
|
252
|
-
* - Non-member sender → rejected (`accepted: false`).
|
|
253
|
-
* - Unknown / not-participated strand → rejected.
|
|
254
|
-
* - Hibernating or idle strand → `accepted` with that status, and a wake started.
|
|
255
|
-
* - Already-live strand → no-op, `accepted` with current status.
|
|
256
|
-
*
|
|
257
|
-
* The wake is not awaited, so the ack means "a wake was started", not "the
|
|
258
|
-
* strand is up". Awaiting it would put the whole resume — a sibling address
|
|
259
|
-
* collection, a strand node build, a relay reservation drive — inside the
|
|
260
|
-
* sender's attempt deadline, coupling that deadline to this node's own budgets.
|
|
261
|
-
*/
|
|
262
|
-
async processWakeRequest(request: WakeRequest, remotePeerId: string): Promise<WakeAck> {
|
|
263
|
-
// The injected membership predicate is the whole v1 authorization (CadreNode
|
|
264
|
-
// supplies the voucher-anchored one); only a peer it admits may ask us to wake.
|
|
265
|
-
if (!(await this.options.isMember(remotePeerId))) {
|
|
266
|
-
log('Rejecting wake from non-member %s', remotePeerId);
|
|
267
|
-
return { accepted: false, reason: 'Sender is not a cadre member' };
|
|
268
|
-
}
|
|
269
|
-
|
|
270
|
-
const instance = this.options.getStrand(request.strandId);
|
|
271
|
-
if (!instance) {
|
|
272
|
-
log('Rejecting wake for unknown/unparticipated strand %s', request.strandId);
|
|
273
|
-
return { accepted: false, reason: 'Strand not found or not participated in' };
|
|
274
|
-
}
|
|
275
|
-
|
|
276
|
-
// Read before the wake starts: the wake path mutates the shared instance.
|
|
277
|
-
const status = instance.status;
|
|
278
|
-
if (status === 'hibernating' || status === 'idle') {
|
|
279
|
-
log('Waking strand %s on push from %s (reason=%s)', request.strandId, remotePeerId, request.reason ?? 'unspecified');
|
|
280
|
-
this.startWake(request.strandId);
|
|
281
|
-
} else {
|
|
282
|
-
log('Strand %s already %s; push-wake is a no-op', request.strandId, status);
|
|
283
|
-
}
|
|
284
|
-
|
|
285
|
-
return { accepted: true, status };
|
|
286
|
-
}
|
|
287
|
-
|
|
288
|
-
/**
|
|
289
|
-
* Start a wake without awaiting it — the same fire-and-forget an activity-driven
|
|
290
|
-
* local wake uses (`HibernationManager`), which also coalesces the two.
|
|
291
|
-
*/
|
|
292
|
-
private startWake(strandId: string): void {
|
|
293
|
-
void this.options.wake(strandId).catch((err: unknown) => {
|
|
294
|
-
log('Push-wake of strand %s failed after it was accepted: %o', strandId, err);
|
|
295
|
-
});
|
|
296
|
-
}
|
|
297
|
-
}
|
|
298
|
-
|
|
299
|
-
/** Options for {@link dialWake}. */
|
|
300
|
-
export interface DialWakeOptions {
|
|
301
|
-
/**
|
|
302
|
-
* The sender's declared link round trip (`NetworkConfig.linkRoundTripMs`), from which
|
|
303
|
-
* both deadlines below are derived when not given. Unset means the declared default.
|
|
304
|
-
*/
|
|
305
|
-
linkRoundTripMs?: number;
|
|
306
|
-
/** Per-ATTEMPT timeout in ms (default: derived as {@link DEFAULT_WAKE_TIMEOUT_MS} is). */
|
|
307
|
-
timeoutMs?: number;
|
|
308
|
-
/**
|
|
309
|
-
* Budget for the whole call — every candidate together (default:
|
|
310
|
-
* {@link WAKE_DIAL_ATTEMPTS} × the per-attempt timeout, as {@link DEFAULT_WAKE_DIAL_BUDGET_MS}).
|
|
311
|
-
*/
|
|
312
|
-
budgetMs?: number;
|
|
313
|
-
/** Override the protocol id (defaults to {@link WAKE_PROTOCOL}). */
|
|
314
|
-
protocolId?: string;
|
|
315
|
-
}
|
|
316
|
-
|
|
317
|
-
/**
|
|
318
|
-
* Sender side: dial a target's control-network address(es), send a
|
|
319
|
-
* {@link WakeRequest} over `WAKE_PROTOCOL`, and return the peer's {@link WakeAck}.
|
|
320
|
-
*
|
|
321
|
-
* Tries each candidate address in order (signaling/relay first, as produced by
|
|
322
|
-
* `CadreNode.resolvePeerAddrs`) until one dials, so a NAT'd peer is reachable via
|
|
323
|
-
* its circuit-relay address. Each attempt is bounded by `timeoutMs` and the call
|
|
324
|
-
* as a whole by `budgetMs` (see {@link DEFAULT_WAKE_DIAL_BUDGET_MS}); a candidate
|
|
325
|
-
* the budget leaves no room for is reported as untried rather than silently
|
|
326
|
-
* dropped.
|
|
327
|
-
*
|
|
328
|
-
* Throws if no address is dialable — with an error naming EVERY candidate and
|
|
329
|
-
* why it failed, not merely the last one ({@link tryAddrsInTurn}). That
|
|
330
|
-
* distinction is not cosmetic: with only the last candidate's message the
|
|
331
|
-
* failure that mattered (usually the signaling address, tried first) is
|
|
332
|
-
* invisible outside a debug log, and the surfaced message points at whichever
|
|
333
|
-
* address happened to be tried last.
|
|
334
|
-
*
|
|
335
|
-
* The candidate loop is deliberately explicit rather than one
|
|
336
|
-
* `dialProtocol(addrs)` call. libp2p sorts any multi-address dial with
|
|
337
|
-
* `defaultAddressSorter`, whose `circuitRelayAddressesLast` pass would demote
|
|
338
|
-
* exactly the signaling address this ordering puts first — silently inverting
|
|
339
|
-
* it, with no per-dial sorter override to opt out of. For the same reason this
|
|
340
|
-
* does not use `dialPeerAddrs`, which puts relayed addresses last.
|
|
341
|
-
*
|
|
342
|
-
* NOTE: unlike `dialPeerAddrs`, this does not drop addresses that relay through this node, so a
|
|
343
|
-
* relay waking a peer that holds a reservation on it tries that address and gets `Can not dial
|
|
344
|
-
* self` (fast, then the next candidate). If wakes are ever sent from a relay to its own
|
|
345
|
-
* reservation holders routinely, share `dialPeerAddrs`'s filter here.
|
|
346
|
-
*/
|
|
347
|
-
export async function dialWake(
|
|
348
|
-
node: Libp2p,
|
|
349
|
-
addrs: Multiaddr[],
|
|
350
|
-
request: WakeRequest,
|
|
351
|
-
options: DialWakeOptions = {}
|
|
352
|
-
): Promise<WakeAck> {
|
|
353
|
-
if (addrs.length === 0) {
|
|
354
|
-
throw new Error('No dialable address for wake target');
|
|
355
|
-
}
|
|
356
|
-
const protocolId = options.protocolId ?? WAKE_PROTOCOL;
|
|
357
|
-
const perAddressMs = options.timeoutMs ?? relayedRequestBudgetMs(options.linkRoundTripMs);
|
|
358
|
-
const budget = {
|
|
359
|
-
perAddressMs,
|
|
360
|
-
totalMs: options.budgetMs ?? WAKE_DIAL_ATTEMPTS * perAddressMs,
|
|
361
|
-
};
|
|
362
|
-
// The attempt's signal aborts the in-flight dialProtocol and resets the live
|
|
363
|
-
// stream, so neither the connect nor the ack-read leaks.
|
|
364
|
-
return await tryAddrsInTurn(addrs, budget, 'Wake dial', (addr, signal, attemptMs) =>
|
|
365
|
-
sendWake(node, addr, protocolId, request, attemptMs, signal));
|
|
366
|
-
}
|
|
367
|
-
|
|
368
|
-
/**
|
|
369
|
-
* Open one stream, send the request, half-close, and read the ack.
|
|
370
|
-
*
|
|
371
|
-
* `signal` is the per-attempt deadline from {@link dialWake}: it goes to
|
|
372
|
-
* `dialProtocol` so a timeout during connect aborts the dial, and into
|
|
373
|
-
* {@link exchangeFrame} so a timeout after the stream is open resets it —
|
|
374
|
-
* releasing the otherwise unbounded ack-read.
|
|
375
|
-
*/
|
|
376
|
-
async function sendWake(
|
|
377
|
-
node: Libp2p,
|
|
378
|
-
addr: Multiaddr,
|
|
379
|
-
protocolId: string,
|
|
380
|
-
request: WakeRequest,
|
|
381
|
-
timeoutMs: number,
|
|
382
|
-
signal: AbortSignal,
|
|
383
|
-
): Promise<WakeAck> {
|
|
384
|
-
// `runOnLimitedConnection: true`: the target may be reachable only over a
|
|
385
|
-
// circuit-relay connection (the signaling-first addr), which libp2p marks
|
|
386
|
-
// "limited". The wake exchange is a single tiny request→ack well within the
|
|
387
|
-
// relay's data/duration cap, so opening it on the limited connection is safe
|
|
388
|
-
// and is the whole point of dialing the relay address.
|
|
389
|
-
const rawStream = await node.dialProtocol(addr, protocolId, { runOnLimitedConnection: true, signal });
|
|
390
|
-
|
|
391
|
-
const ack = await exchangeFrame(
|
|
392
|
-
rawStream as unknown as ControlStream,
|
|
393
|
-
signal,
|
|
394
|
-
request,
|
|
395
|
-
(stream) => readFrame<WakeAck>(stream, timeoutMs),
|
|
396
|
-
'Wake dial aborted by timeout',
|
|
397
|
-
);
|
|
398
|
-
|
|
399
|
-
log('Wake ack from %s: accepted=%s status=%s', addr.toString(), ack.accepted, ack.status ?? '-');
|
|
400
|
-
return ack;
|
|
401
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* Control-network push-wake transport.
|
|
3
|
+
*
|
|
4
|
+
* Lets a same-cadre peer — typically an always-on server that participates in a
|
|
5
|
+
* strand and sees new activity — signal a hibernating cadre peer to bring that
|
|
6
|
+
* strand online, pull pending activity, and re-hibernate. Push-wake travels the
|
|
7
|
+
* **control network** (the per-party network connecting this party's own cadre
|
|
8
|
+
* nodes), which is the only network a hibernating peer keeps connected.
|
|
9
|
+
*
|
|
10
|
+
* Modeled directly on `seed-bootstrap.ts`: a dedicated libp2p protocol id,
|
|
11
|
+
* 4-byte big-endian length-prefixed JSON frames, `node.handle` for the receiver,
|
|
12
|
+
* `node.dialProtocol` for the sender, and the shared `ControlStream` primitives
|
|
13
|
+
* from `control-stream.ts`. The exchange is a single request → single ack on one
|
|
14
|
+
* stream (like seed delivery), so each side reads to EOF (under a read timeout)
|
|
15
|
+
* and decodes one frame via the shared {@link decodeLengthPrefixedFrame} guard.
|
|
16
|
+
*
|
|
17
|
+
* **Authorization (v1):** a wake is low-risk — it only causes the receiver to
|
|
18
|
+
* spend resources coming online for a strand it already participates in — so the
|
|
19
|
+
* receiver carries no per-request signature and instead defers entirely to the
|
|
20
|
+
* injected `isMember` predicate. `CadreNode` injects its AUTHORIZED-membership
|
|
21
|
+
* predicate there (`isAuthorizedMember`: the sender's `CadrePeer` row must carry
|
|
22
|
+
* a voucher that verifies against an owner key in the receiver's node-local
|
|
23
|
+
* trusted-owner anchor), so a peer that merely published rows into the replicated
|
|
24
|
+
* control DB is refused. This module stays agnostic: it enforces whatever
|
|
25
|
+
* predicate it is given.
|
|
26
|
+
*/
|
|
27
|
+
|
|
28
|
+
import debug from 'debug';
|
|
29
|
+
import type { Libp2p, Connection } from '@libp2p/interface';
|
|
30
|
+
import type { Multiaddr } from '@multiformats/multiaddr';
|
|
31
|
+
import type { StrandInstance, WakeRequest, WakeAck } from './types.js';
|
|
32
|
+
import { decodeLengthPrefixedFrame } from './seed-bootstrap.js';
|
|
33
|
+
import { type ControlStream, exchangeFrame, readStreamToEnd, replyAndClose } from './control-stream.js';
|
|
34
|
+
import { tryAddrsInTurn } from './peer-dial.js';
|
|
35
|
+
import { relayedRequestBudgetMs } from './link-budget.js';
|
|
36
|
+
|
|
37
|
+
const log = debug('sereus:cadre:strand-wake');
|
|
38
|
+
|
|
39
|
+
/** Protocol id for control-network push-wake (parallel to `/sereus/seed/1.0.0`). */
|
|
40
|
+
export const WAKE_PROTOCOL = '/sereus/strand-wake/1.0.0';
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Maximum wake frame size. Wake messages are tiny (a strand id + short reason),
|
|
44
|
+
* so this is a defensive cap — far below the 1MB seed ceiling — that bounds the
|
|
45
|
+
* bytes a peer can make the receiver buffer per stream.
|
|
46
|
+
*/
|
|
47
|
+
const MAX_WAKE_SIZE = 64 * 1024;
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Default deadline for ONE wake attempt — dial, request, ack — at the default declared link
|
|
51
|
+
* (ms): `relayedRequestBudgetMs` in `link-budget.ts`, because the target may be reachable
|
|
52
|
+
* only through a relay and the attempt has to open that connection before the exchange.
|
|
53
|
+
* {@link dialWake} derives it from {@link DialWakeOptions.linkRoundTripMs} instead when given.
|
|
54
|
+
*
|
|
55
|
+
* It holds only link work because the receiver acks as soon as it has DECIDED, before the
|
|
56
|
+
* wake itself runs ({@link StrandWakeService.processWakeRequest}).
|
|
57
|
+
*
|
|
58
|
+
* NOTE: the receiver's membership check (two live control reads) runs inside this deadline and
|
|
59
|
+
* is not counted. In steady state those reads touch only held blocks and do not consult the
|
|
60
|
+
* cohort. If a wake or address request is seen timing out while the receiver's membership read
|
|
61
|
+
* is consulting, count one membership decision in this deadline or answer the check from the
|
|
62
|
+
* materialized authorized-peer snapshot. See docs/cadre-consistency.md → "Deadlines Over
|
|
63
|
+
* Optimystic's Reads and Commits".
|
|
64
|
+
*/
|
|
65
|
+
export const DEFAULT_WAKE_TIMEOUT_MS = relayedRequestBudgetMs();
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Whole attempts one {@link dialWake} call budgets for: a stale signaling address, then the
|
|
69
|
+
* direct address that works.
|
|
70
|
+
*/
|
|
71
|
+
export const WAKE_DIAL_ATTEMPTS = 2;
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Default budget for a WHOLE {@link dialWake} call at the default declared link, in ms —
|
|
75
|
+
* every candidate address together, not each one: {@link WAKE_DIAL_ATTEMPTS} attempt
|
|
76
|
+
* deadlines.
|
|
77
|
+
*
|
|
78
|
+
* Without it the cost of a wake is (candidate count × {@link
|
|
79
|
+
* DEFAULT_WAKE_TIMEOUT_MS}), a number nothing chooses or bounds. That is not
|
|
80
|
+
* hypothetical: an address behind a dropped NAT mapping, or any host that
|
|
81
|
+
* blackholes rather than sending a RST, burns its full attempt timeout instead
|
|
82
|
+
* of failing in milliseconds, so a five-address peer would cost five attempts.
|
|
83
|
+
*
|
|
84
|
+
* This makes the TARGET PEER the unit rather than the address — the same
|
|
85
|
+
* decision, for the same reason, as
|
|
86
|
+
* `DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS` in `peer-dial.ts`, though not the same
|
|
87
|
+
* count: that one is sized to fit several dead addresses ahead of a working one,
|
|
88
|
+
* while a wake target is expected to be awake and reachable on its first or second
|
|
89
|
+
* address. Two whole attempts so a reachable peer whose signaling address is stale
|
|
90
|
+
* still gets a genuine try at its direct one; a peer needing longer than that is
|
|
91
|
+
* not "asleep and reachable", which is the only case a wake is for. The last
|
|
92
|
+
* attempt inside the budget gets whatever remains of it, so the call returns at
|
|
93
|
+
* the budget, not past it.
|
|
94
|
+
*
|
|
95
|
+
* Override per call with {@link DialWakeOptions.budgetMs} — tests that drive
|
|
96
|
+
* dead addresses on purpose set it low so a dial's duration is a chosen number
|
|
97
|
+
* rather than a transitive libp2p default stretched by machine load.
|
|
98
|
+
*/
|
|
99
|
+
export const DEFAULT_WAKE_DIAL_BUDGET_MS = WAKE_DIAL_ATTEMPTS * DEFAULT_WAKE_TIMEOUT_MS;
|
|
100
|
+
|
|
101
|
+
/** Default time the receiver waits for an inbound wake frame before aborting (ms). */
|
|
102
|
+
// eslint-disable-next-line no-restricted-syntax -- link-independent: a receiver cap on one small request frame on a stream the peer already opened; it bounds a peer that opens a stream and never sends, not the dial
|
|
103
|
+
const DEFAULT_WAKE_READ_TIMEOUT_MS = 10_000;
|
|
104
|
+
|
|
105
|
+
/** Default cap on concurrent inbound wake streams a single peer can pin open. */
|
|
106
|
+
const DEFAULT_MAX_CONCURRENT_WAKES = 100;
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Read a libp2p stream to EOF and decode the single length-prefixed JSON frame
|
|
110
|
+
* it carries. Bounded by `timeoutMs` (a never-half-closing peer is aborted, not
|
|
111
|
+
* awaited forever) and capped at {@link MAX_WAKE_SIZE}, reusing the shared
|
|
112
|
+
* {@link decodeLengthPrefixedFrame} guard for the prefix/length checks.
|
|
113
|
+
*/
|
|
114
|
+
async function readFrame<T>(stream: ControlStream, timeoutMs: number): Promise<T> {
|
|
115
|
+
const data = await readStreamToEnd(stream, { maxBytes: MAX_WAKE_SIZE, timeoutMs, label: 'Wake' });
|
|
116
|
+
const body = decodeLengthPrefixedFrame(data, MAX_WAKE_SIZE);
|
|
117
|
+
return JSON.parse(new TextDecoder().decode(body)) as T;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Dependencies the {@link StrandWakeService} receiver needs from its host
|
|
122
|
+
* (`CadreNode`), injected so the service is testable without a full node.
|
|
123
|
+
*/
|
|
124
|
+
export interface StrandWakeServiceOptions {
|
|
125
|
+
/** Membership gate: is the remote peer a `CadrePeer` member of this cadre? */
|
|
126
|
+
isMember(remotePeerId: string): Promise<boolean>;
|
|
127
|
+
/** Look up a local strand instance by id (undefined if not participated in). */
|
|
128
|
+
getStrand(strandId: string): StrandInstance | undefined;
|
|
129
|
+
/**
|
|
130
|
+
* Trigger the local wake path for a hibernating/idle strand. Wired to
|
|
131
|
+
* `CadreNode.wakeStrand` (→ `HibernationManager` → `resumeStrand`); `resumeStrand`
|
|
132
|
+
* joins any resume already in flight, so a push-wake racing a check-in shares its
|
|
133
|
+
* runtime build.
|
|
134
|
+
*
|
|
135
|
+
* Started, not awaited: the ack goes back before the strand is up, and a wake
|
|
136
|
+
* that fails afterwards is logged here rather than reported to the sender.
|
|
137
|
+
*/
|
|
138
|
+
wake(strandId: string): Promise<void>;
|
|
139
|
+
/**
|
|
140
|
+
* Time to wait for the inbound wake frame before aborting the read (ms).
|
|
141
|
+
* Defaults to {@link DEFAULT_WAKE_READ_TIMEOUT_MS}. Bounds a buggy/compromised
|
|
142
|
+
* own-cadre node that opens a stream and never half-closes its write end.
|
|
143
|
+
*/
|
|
144
|
+
readTimeoutMs?: number;
|
|
145
|
+
/**
|
|
146
|
+
* Cap on concurrent inbound wake streams (defaults to
|
|
147
|
+
* {@link DEFAULT_MAX_CONCURRENT_WAKES}). Over the cap, a non-accepting ack is
|
|
148
|
+
* returned without invoking the wake path.
|
|
149
|
+
*
|
|
150
|
+
* It counts streams, not wakes: a wake started after its ack is no longer
|
|
151
|
+
* counted. Those are bounded by the strands this node participates in instead,
|
|
152
|
+
* since an unknown strand is refused before any wake starts and the wake path
|
|
153
|
+
* coalesces per strand.
|
|
154
|
+
*/
|
|
155
|
+
maxConcurrent?: number;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* Receiver side of the push-wake protocol. Registers a `WAKE_PROTOCOL` handler
|
|
160
|
+
* on the control node and, for each inbound {@link WakeRequest}, gates on cadre
|
|
161
|
+
* membership, replies with a {@link WakeAck}, and starts resuming the named
|
|
162
|
+
* strand if it is hibernating/idle and we participate in it.
|
|
163
|
+
*/
|
|
164
|
+
export class StrandWakeService {
|
|
165
|
+
private readonly options: StrandWakeServiceOptions;
|
|
166
|
+
private readonly readTimeoutMs: number;
|
|
167
|
+
private readonly maxConcurrent: number;
|
|
168
|
+
private node: Libp2p | null = null;
|
|
169
|
+
/** In-flight inbound wake streams, used to enforce {@link maxConcurrent}. */
|
|
170
|
+
private activeStreams = 0;
|
|
171
|
+
|
|
172
|
+
constructor(options: StrandWakeServiceOptions) {
|
|
173
|
+
this.options = options;
|
|
174
|
+
this.readTimeoutMs = options.readTimeoutMs ?? DEFAULT_WAKE_READ_TIMEOUT_MS;
|
|
175
|
+
this.maxConcurrent = options.maxConcurrent ?? DEFAULT_MAX_CONCURRENT_WAKES;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/** Number of in-flight inbound wake streams. */
|
|
179
|
+
get activeCount(): number {
|
|
180
|
+
return this.activeStreams;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/**
|
|
184
|
+
* Register the wake protocol handler on the control node. Rejects when libp2p
|
|
185
|
+
* refuses the registration; the node reference is kept only once the handler
|
|
186
|
+
* is in place.
|
|
187
|
+
*/
|
|
188
|
+
async initialize(node: Libp2p): Promise<void> {
|
|
189
|
+
// `runOnLimitedConnection: true` is REQUIRED for the relay path: a NAT'd
|
|
190
|
+
// receiver is reached over a circuit-relay connection, which libp2p marks
|
|
191
|
+
// "limited" (the relay caps its data/duration). Without this the receiver
|
|
192
|
+
// would refuse the inbound wake stream on exactly the connection the
|
|
193
|
+
// protocol is designed to use (see the relay note on `dialWake`).
|
|
194
|
+
await node.handle(WAKE_PROTOCOL, async (rawStream: unknown, rawConnection: unknown) => {
|
|
195
|
+
const remotePeerId = (rawConnection as Connection).remotePeer.toString();
|
|
196
|
+
await this.handleStream(rawStream as ControlStream, remotePeerId);
|
|
197
|
+
}, { runOnLimitedConnection: true });
|
|
198
|
+
this.node = node;
|
|
199
|
+
log('StrandWakeService registered handler: %s', WAKE_PROTOCOL);
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/** Unregister the handler and release the node reference. */
|
|
203
|
+
async shutdown(): Promise<void> {
|
|
204
|
+
if (this.node) {
|
|
205
|
+
await this.node.unhandle(WAKE_PROTOCOL);
|
|
206
|
+
this.node = null;
|
|
207
|
+
log('StrandWakeService shutdown');
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* Read the inbound request, decide the wake, and write the ack.
|
|
213
|
+
*
|
|
214
|
+
* Three hardening layers, all reported as a non-accepting ack rather than a
|
|
215
|
+
* dropped/hung stream: a concurrency cap (over {@link maxConcurrent}, reply
|
|
216
|
+
* without touching the wake path), a read timeout (a peer that never
|
|
217
|
+
* half-closes is aborted inside {@link readFrame}/`readStreamToEnd`), and the
|
|
218
|
+
* existing malformed/oversized-frame guard.
|
|
219
|
+
*/
|
|
220
|
+
private async handleStream(stream: ControlStream, remotePeerId: string): Promise<void> {
|
|
221
|
+
log('Incoming wake request from: %s', remotePeerId);
|
|
222
|
+
|
|
223
|
+
if (this.activeStreams >= this.maxConcurrent) {
|
|
224
|
+
log('Rejecting wake from %s: %d concurrent streams at cap %d', remotePeerId, this.activeStreams, this.maxConcurrent);
|
|
225
|
+
await replyAndClose(stream, { accepted: false, reason: 'Too many concurrent wake requests' } satisfies WakeAck, 'Wake');
|
|
226
|
+
return;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
this.activeStreams++;
|
|
230
|
+
try {
|
|
231
|
+
await replyAndClose(stream, await this.answerStream(stream, remotePeerId), 'Wake');
|
|
232
|
+
} finally {
|
|
233
|
+
this.activeStreams--;
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/** Read and decide one inbound request; any failure becomes a non-accepting ack. */
|
|
238
|
+
private async answerStream(stream: ControlStream, remotePeerId: string): Promise<WakeAck> {
|
|
239
|
+
try {
|
|
240
|
+
const request = await readFrame<WakeRequest>(stream, this.readTimeoutMs);
|
|
241
|
+
return await this.processWakeRequest(request, remotePeerId);
|
|
242
|
+
} catch (err) {
|
|
243
|
+
log('Error handling wake request from %s: %o', remotePeerId, err);
|
|
244
|
+
return { accepted: false, reason: err instanceof Error ? err.message : 'Unknown error' };
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* Decide the wake for a decoded request and start it. Exposed (not private) so
|
|
250
|
+
* the decision matrix can be unit-tested directly.
|
|
251
|
+
*
|
|
252
|
+
* - Non-member sender → rejected (`accepted: false`).
|
|
253
|
+
* - Unknown / not-participated strand → rejected.
|
|
254
|
+
* - Hibernating or idle strand → `accepted` with that status, and a wake started.
|
|
255
|
+
* - Already-live strand → no-op, `accepted` with current status.
|
|
256
|
+
*
|
|
257
|
+
* The wake is not awaited, so the ack means "a wake was started", not "the
|
|
258
|
+
* strand is up". Awaiting it would put the whole resume — a sibling address
|
|
259
|
+
* collection, a strand node build, a relay reservation drive — inside the
|
|
260
|
+
* sender's attempt deadline, coupling that deadline to this node's own budgets.
|
|
261
|
+
*/
|
|
262
|
+
async processWakeRequest(request: WakeRequest, remotePeerId: string): Promise<WakeAck> {
|
|
263
|
+
// The injected membership predicate is the whole v1 authorization (CadreNode
|
|
264
|
+
// supplies the voucher-anchored one); only a peer it admits may ask us to wake.
|
|
265
|
+
if (!(await this.options.isMember(remotePeerId))) {
|
|
266
|
+
log('Rejecting wake from non-member %s', remotePeerId);
|
|
267
|
+
return { accepted: false, reason: 'Sender is not a cadre member' };
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
const instance = this.options.getStrand(request.strandId);
|
|
271
|
+
if (!instance) {
|
|
272
|
+
log('Rejecting wake for unknown/unparticipated strand %s', request.strandId);
|
|
273
|
+
return { accepted: false, reason: 'Strand not found or not participated in' };
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
// Read before the wake starts: the wake path mutates the shared instance.
|
|
277
|
+
const status = instance.status;
|
|
278
|
+
if (status === 'hibernating' || status === 'idle') {
|
|
279
|
+
log('Waking strand %s on push from %s (reason=%s)', request.strandId, remotePeerId, request.reason ?? 'unspecified');
|
|
280
|
+
this.startWake(request.strandId);
|
|
281
|
+
} else {
|
|
282
|
+
log('Strand %s already %s; push-wake is a no-op', request.strandId, status);
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
return { accepted: true, status };
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
/**
|
|
289
|
+
* Start a wake without awaiting it — the same fire-and-forget an activity-driven
|
|
290
|
+
* local wake uses (`HibernationManager`), which also coalesces the two.
|
|
291
|
+
*/
|
|
292
|
+
private startWake(strandId: string): void {
|
|
293
|
+
void this.options.wake(strandId).catch((err: unknown) => {
|
|
294
|
+
log('Push-wake of strand %s failed after it was accepted: %o', strandId, err);
|
|
295
|
+
});
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
/** Options for {@link dialWake}. */
|
|
300
|
+
export interface DialWakeOptions {
|
|
301
|
+
/**
|
|
302
|
+
* The sender's declared link round trip (`NetworkConfig.linkRoundTripMs`), from which
|
|
303
|
+
* both deadlines below are derived when not given. Unset means the declared default.
|
|
304
|
+
*/
|
|
305
|
+
linkRoundTripMs?: number;
|
|
306
|
+
/** Per-ATTEMPT timeout in ms (default: derived as {@link DEFAULT_WAKE_TIMEOUT_MS} is). */
|
|
307
|
+
timeoutMs?: number;
|
|
308
|
+
/**
|
|
309
|
+
* Budget for the whole call — every candidate together (default:
|
|
310
|
+
* {@link WAKE_DIAL_ATTEMPTS} × the per-attempt timeout, as {@link DEFAULT_WAKE_DIAL_BUDGET_MS}).
|
|
311
|
+
*/
|
|
312
|
+
budgetMs?: number;
|
|
313
|
+
/** Override the protocol id (defaults to {@link WAKE_PROTOCOL}). */
|
|
314
|
+
protocolId?: string;
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/**
|
|
318
|
+
* Sender side: dial a target's control-network address(es), send a
|
|
319
|
+
* {@link WakeRequest} over `WAKE_PROTOCOL`, and return the peer's {@link WakeAck}.
|
|
320
|
+
*
|
|
321
|
+
* Tries each candidate address in order (signaling/relay first, as produced by
|
|
322
|
+
* `CadreNode.resolvePeerAddrs`) until one dials, so a NAT'd peer is reachable via
|
|
323
|
+
* its circuit-relay address. Each attempt is bounded by `timeoutMs` and the call
|
|
324
|
+
* as a whole by `budgetMs` (see {@link DEFAULT_WAKE_DIAL_BUDGET_MS}); a candidate
|
|
325
|
+
* the budget leaves no room for is reported as untried rather than silently
|
|
326
|
+
* dropped.
|
|
327
|
+
*
|
|
328
|
+
* Throws if no address is dialable — with an error naming EVERY candidate and
|
|
329
|
+
* why it failed, not merely the last one ({@link tryAddrsInTurn}). That
|
|
330
|
+
* distinction is not cosmetic: with only the last candidate's message the
|
|
331
|
+
* failure that mattered (usually the signaling address, tried first) is
|
|
332
|
+
* invisible outside a debug log, and the surfaced message points at whichever
|
|
333
|
+
* address happened to be tried last.
|
|
334
|
+
*
|
|
335
|
+
* The candidate loop is deliberately explicit rather than one
|
|
336
|
+
* `dialProtocol(addrs)` call. libp2p sorts any multi-address dial with
|
|
337
|
+
* `defaultAddressSorter`, whose `circuitRelayAddressesLast` pass would demote
|
|
338
|
+
* exactly the signaling address this ordering puts first — silently inverting
|
|
339
|
+
* it, with no per-dial sorter override to opt out of. For the same reason this
|
|
340
|
+
* does not use `dialPeerAddrs`, which puts relayed addresses last.
|
|
341
|
+
*
|
|
342
|
+
* NOTE: unlike `dialPeerAddrs`, this does not drop addresses that relay through this node, so a
|
|
343
|
+
* relay waking a peer that holds a reservation on it tries that address and gets `Can not dial
|
|
344
|
+
* self` (fast, then the next candidate). If wakes are ever sent from a relay to its own
|
|
345
|
+
* reservation holders routinely, share `dialPeerAddrs`'s filter here.
|
|
346
|
+
*/
|
|
347
|
+
export async function dialWake(
|
|
348
|
+
node: Libp2p,
|
|
349
|
+
addrs: Multiaddr[],
|
|
350
|
+
request: WakeRequest,
|
|
351
|
+
options: DialWakeOptions = {}
|
|
352
|
+
): Promise<WakeAck> {
|
|
353
|
+
if (addrs.length === 0) {
|
|
354
|
+
throw new Error('No dialable address for wake target');
|
|
355
|
+
}
|
|
356
|
+
const protocolId = options.protocolId ?? WAKE_PROTOCOL;
|
|
357
|
+
const perAddressMs = options.timeoutMs ?? relayedRequestBudgetMs(options.linkRoundTripMs);
|
|
358
|
+
const budget = {
|
|
359
|
+
perAddressMs,
|
|
360
|
+
totalMs: options.budgetMs ?? WAKE_DIAL_ATTEMPTS * perAddressMs,
|
|
361
|
+
};
|
|
362
|
+
// The attempt's signal aborts the in-flight dialProtocol and resets the live
|
|
363
|
+
// stream, so neither the connect nor the ack-read leaks.
|
|
364
|
+
return await tryAddrsInTurn(addrs, budget, 'Wake dial', (addr, signal, attemptMs) =>
|
|
365
|
+
sendWake(node, addr, protocolId, request, attemptMs, signal));
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
/**
|
|
369
|
+
* Open one stream, send the request, half-close, and read the ack.
|
|
370
|
+
*
|
|
371
|
+
* `signal` is the per-attempt deadline from {@link dialWake}: it goes to
|
|
372
|
+
* `dialProtocol` so a timeout during connect aborts the dial, and into
|
|
373
|
+
* {@link exchangeFrame} so a timeout after the stream is open resets it —
|
|
374
|
+
* releasing the otherwise unbounded ack-read.
|
|
375
|
+
*/
|
|
376
|
+
async function sendWake(
|
|
377
|
+
node: Libp2p,
|
|
378
|
+
addr: Multiaddr,
|
|
379
|
+
protocolId: string,
|
|
380
|
+
request: WakeRequest,
|
|
381
|
+
timeoutMs: number,
|
|
382
|
+
signal: AbortSignal,
|
|
383
|
+
): Promise<WakeAck> {
|
|
384
|
+
// `runOnLimitedConnection: true`: the target may be reachable only over a
|
|
385
|
+
// circuit-relay connection (the signaling-first addr), which libp2p marks
|
|
386
|
+
// "limited". The wake exchange is a single tiny request→ack well within the
|
|
387
|
+
// relay's data/duration cap, so opening it on the limited connection is safe
|
|
388
|
+
// and is the whole point of dialing the relay address.
|
|
389
|
+
const rawStream = await node.dialProtocol(addr, protocolId, { runOnLimitedConnection: true, signal });
|
|
390
|
+
|
|
391
|
+
const ack = await exchangeFrame(
|
|
392
|
+
rawStream as unknown as ControlStream,
|
|
393
|
+
signal,
|
|
394
|
+
request,
|
|
395
|
+
(stream) => readFrame<WakeAck>(stream, timeoutMs),
|
|
396
|
+
'Wake dial aborted by timeout',
|
|
397
|
+
);
|
|
398
|
+
|
|
399
|
+
log('Wake ack from %s: accepted=%s status=%s', addr.toString(), ack.accepted, ack.status ?? '-');
|
|
400
|
+
return ack;
|
|
401
|
+
}
|