@serfab/cadre-core 1.3.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/dist/cadre-node.d.ts +70 -34
- package/dist/cadre-node.js +197 -60
- package/dist/cadre-node.js.map +1 -1
- package/dist/control-database.d.ts +24 -10
- package/dist/control-database.js +48 -16
- package/dist/control-database.js.map +1 -1
- package/dist/control-formation-recorder.d.ts +4 -4
- package/dist/control-formation-recorder.js +6 -6
- package/dist/control-formation-recorder.js.map +1 -1
- package/dist/control-read-retry.d.ts +12 -0
- package/dist/control-read-retry.js +12 -0
- package/dist/control-read-retry.js.map +1 -1
- package/dist/control-write-retry.d.ts +5 -4
- package/dist/control-write-retry.js +24 -7
- package/dist/control-write-retry.js.map +1 -1
- package/dist/index.d.ts +5 -4
- package/dist/index.js +10 -5
- package/dist/index.js.map +1 -1
- package/dist/link-budget.d.ts +182 -0
- package/dist/link-budget.js +195 -0
- package/dist/link-budget.js.map +1 -0
- package/dist/membership-connection-gater.d.ts +18 -5
- package/dist/membership-connection-gater.js +50 -10
- package/dist/membership-connection-gater.js.map +1 -1
- package/dist/peer-dial.d.ts +36 -17
- package/dist/peer-dial.js +37 -17
- package/dist/peer-dial.js.map +1 -1
- package/dist/peer-join-backfill.d.ts +58 -4
- package/dist/peer-join-backfill.js +165 -20
- package/dist/peer-join-backfill.js.map +1 -1
- package/dist/relay-reservation.d.ts +59 -11
- package/dist/relay-reservation.js +169 -43
- package/dist/relay-reservation.js.map +1 -1
- package/dist/seed-bootstrap.js +2 -1
- package/dist/seed-bootstrap.js.map +1 -1
- package/dist/storage-scope.d.ts +50 -12
- package/dist/storage-scope.js +85 -12
- package/dist/storage-scope.js.map +1 -1
- package/dist/strand-first-sync-gate.d.ts +43 -5
- package/dist/strand-first-sync-gate.js +55 -7
- package/dist/strand-first-sync-gate.js.map +1 -1
- package/dist/strand-formation-manager.js +2 -1
- package/dist/strand-formation-manager.js.map +1 -1
- package/dist/strand-id.d.ts +35 -0
- package/dist/strand-id.js +46 -0
- package/dist/strand-id.js.map +1 -0
- package/dist/strand-instance-manager.d.ts +27 -2
- package/dist/strand-instance-manager.js +75 -15
- package/dist/strand-instance-manager.js.map +1 -1
- package/dist/strand-membership-reconciler.d.ts +200 -45
- package/dist/strand-membership-reconciler.js +302 -86
- package/dist/strand-membership-reconciler.js.map +1 -1
- package/dist/strand-membership-writer.d.ts +3 -2
- package/dist/strand-membership-writer.js +67 -90
- package/dist/strand-membership-writer.js.map +1 -1
- package/dist/strand-network-config.d.ts +3 -3
- package/dist/strand-network-config.js +3 -3
- package/dist/strand-solicitation.js +2 -1
- package/dist/strand-solicitation.js.map +1 -1
- package/dist/strand-watcher.d.ts +45 -0
- package/dist/strand-watcher.js +74 -13
- package/dist/strand-watcher.js.map +1 -1
- package/dist/types.d.ts +190 -24
- package/dist/types.js +73 -4
- package/dist/types.js.map +1 -1
- package/package.json +8 -8
- package/src/cadre-node.ts +7445 -7302
- package/src/control-database.ts +48 -16
- package/src/control-formation-recorder.ts +6 -6
- package/src/control-read-retry.ts +214 -202
- package/src/control-write-retry.ts +24 -7
- package/src/index.ts +29 -2
- package/src/link-budget.ts +217 -0
- package/src/membership-connection-gater.ts +56 -11
- package/src/peer-dial.ts +39 -17
- package/src/peer-join-backfill.ts +192 -23
- package/src/relay-reservation.ts +1119 -951
- package/src/seed-bootstrap.ts +2 -1
- package/src/storage-scope.ts +163 -78
- package/src/strand-first-sync-gate.ts +55 -7
- package/src/strand-formation-manager.ts +647 -646
- package/src/strand-id.ts +48 -0
- package/src/strand-instance-manager.ts +1516 -1440
- package/src/strand-membership-reconciler.ts +334 -92
- package/src/strand-membership-writer.ts +1962 -1987
- package/src/strand-network-config.ts +250 -250
- package/src/strand-solicitation.ts +500 -499
- package/src/strand-watcher.ts +388 -325
- package/src/types.ts +192 -23
package/src/index.ts
CHANGED
|
@@ -119,8 +119,16 @@ export {
|
|
|
119
119
|
|
|
120
120
|
// Storage scope keys — what `CadreNodeConfig.storage.provider` is called with. A
|
|
121
121
|
// strand's key is its strand id; the control database's key carries the party id,
|
|
122
|
-
// so two parties on one device never share a control store.
|
|
123
|
-
|
|
122
|
+
// so two parties on one device never share a control store. Every key stays within
|
|
123
|
+
// `[A-Za-z0-9._-]`: the control key by base64url encoding, a strand's by
|
|
124
|
+
// `assertStrandScopeKey`, which every strand launch runs.
|
|
125
|
+
export {
|
|
126
|
+
controlStorageScope,
|
|
127
|
+
isControlStorageScope,
|
|
128
|
+
isValidStrandScopeKey,
|
|
129
|
+
assertStrandScopeKey,
|
|
130
|
+
InvalidStrandIdError
|
|
131
|
+
} from './storage-scope.js';
|
|
124
132
|
|
|
125
133
|
// Strand database
|
|
126
134
|
export { StrandDatabase, type StrandDatabaseConfig } from './strand-database.js';
|
|
@@ -162,6 +170,7 @@ export {
|
|
|
162
170
|
PeerJoinBackfill,
|
|
163
171
|
DEFAULT_PEER_JOIN_BACKFILL,
|
|
164
172
|
MAX_BLOCK_MESSAGE_BYTES,
|
|
173
|
+
PEER_JOIN_BACKFILL_WARN_AFTER_FAILURES,
|
|
165
174
|
type PeerJoinBackfillConfig,
|
|
166
175
|
type PeerJoinBackfillDeps,
|
|
167
176
|
type PeerJoinBackfillResult,
|
|
@@ -432,11 +441,28 @@ export {
|
|
|
432
441
|
directBeforeRelayed,
|
|
433
442
|
DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS,
|
|
434
443
|
DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS,
|
|
444
|
+
CONTROL_COHORT_DIAL_ADDRESS_ATTEMPTS,
|
|
435
445
|
DEFAULT_PEER_DIAL_BUDGET,
|
|
436
446
|
type PeerDialBudget,
|
|
437
447
|
type AddrDialer
|
|
438
448
|
} from './peer-dial.js';
|
|
439
449
|
|
|
450
|
+
// Dial and reservation budgets counted in link round trips rather than fixed milliseconds, so
|
|
451
|
+
// one declared assumption about the link moves them all — see `NetworkConfig.linkRoundTripMs`.
|
|
452
|
+
export {
|
|
453
|
+
DECLARED_LINK_ROUND_TRIP_MS,
|
|
454
|
+
RELAYED_DIAL_ROUND_TRIPS,
|
|
455
|
+
RELAY_RESERVATION_ROUND_TRIPS,
|
|
456
|
+
CIRCUIT_REQUEST_ROUND_TRIPS,
|
|
457
|
+
PUSH_TRANSFER_ALLOWANCE_MS,
|
|
458
|
+
resolveLinkRoundTripMs,
|
|
459
|
+
relayedDialBudgetMs,
|
|
460
|
+
relayReservationBudgetMs,
|
|
461
|
+
circuitRequestBudgetMs,
|
|
462
|
+
peerJoinPushBudget,
|
|
463
|
+
type PeerJoinPushBudget
|
|
464
|
+
} from './link-budget.js';
|
|
465
|
+
|
|
440
466
|
// Seed trust policy (trust anchor for incoming seeds)
|
|
441
467
|
export {
|
|
442
468
|
anchoredTrustPolicy,
|
|
@@ -493,6 +519,7 @@ export {
|
|
|
493
519
|
DEFAULT_ENROLLMENT_WINDOW_MS,
|
|
494
520
|
ADMISSION_DECISION_TIMEOUT_MS,
|
|
495
521
|
RELAY_ADMISSION_RESERVE_DEADLINE_MS,
|
|
522
|
+
RELAY_ADMISSION_CLOSE_TIMEOUT_MS,
|
|
496
523
|
MAX_UNAUTHORIZED_RELAY_RESERVATIONS,
|
|
497
524
|
UNAUTHORIZED_RESERVATION_TTL_MS,
|
|
498
525
|
UnauthorizedReservationBudget,
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Dial and reservation budgets derived from ONE declared link round trip, instead of fixed
|
|
3
|
+
* millisecond numbers chosen on a fast local network.
|
|
4
|
+
*
|
|
5
|
+
* Reaching another machine through a relay costs a FIXED NUMBER OF EXCHANGES, not a fixed
|
|
6
|
+
* number of milliseconds. So every budget written as milliseconds has a link speed above which
|
|
7
|
+
* it can never open a connection at all — and the slower the link, the more of them fall over,
|
|
8
|
+
* silently, because an abandoned dial looks the same as a peer that is not there. Counting round
|
|
9
|
+
* trips instead is what stops that class of defect coming back: the next person to widen a
|
|
10
|
+
* budget changes ONE declaration, and the next person to add a dial writes down its round-trip
|
|
11
|
+
* count beside the arithmetic.
|
|
12
|
+
*
|
|
13
|
+
* ── The instrument ──
|
|
14
|
+
*
|
|
15
|
+
* Every count and every number below comes from
|
|
16
|
+
* `packages/integration-tests/src/scenarios/relayed-dial-cost-by-latency.integration.ts`, which
|
|
17
|
+
* is committed and opt-in (`RELAY_DIAL_COST=1`). **Its doc comment is the single home of the
|
|
18
|
+
* measurement**; this module holds only what the derivation needs. Re-run it before changing
|
|
19
|
+
* anything here.
|
|
20
|
+
*
|
|
21
|
+
* ── The unit ──
|
|
22
|
+
*
|
|
23
|
+
* ONE LINK ROUND TRIP is a message from one machine reaching the other and a reply coming
|
|
24
|
+
* back, over the path those two machines actually use — through the relay, when that is the
|
|
25
|
+
* only way they can reach each other. In the instrument, which injects a constant ONE-WAY
|
|
26
|
+
* frame delay `d`, one link round trip is `2d`.
|
|
27
|
+
*
|
|
28
|
+
* A leg between a node and the RELAY is half a link round trip, because it crosses one of the
|
|
29
|
+
* two hops rather than both. That is why the reservation drive's cost reads as "four
|
|
30
|
+
* node-to-relay round trips" in one telling and "two link round trips" in another: same
|
|
31
|
+
* quantity, different unit. This module uses the LINK round trip throughout, so there is one
|
|
32
|
+
* unit to reason in.
|
|
33
|
+
*
|
|
34
|
+
* ── The counts, and what they cost ──
|
|
35
|
+
*
|
|
36
|
+
* | operation | link round trips | at the declared default |
|
|
37
|
+
* | ---------------------------------------------------- | ---------------- | ----------------------- |
|
|
38
|
+
* | open a relayed connection to another machine | 4 | 8 000 ms |
|
|
39
|
+
* | dial the relay itself | 1 | 2 000 ms |
|
|
40
|
+
* | request a reservation on an open relay connection | 1 | 2 000 ms |
|
|
41
|
+
* | negotiate a protocol over an established circuit | 1 | 2 000 ms |
|
|
42
|
+
*
|
|
43
|
+
* **Measured** 2026-09-26, one Windows machine, loopback dedicated relay, re-run for this
|
|
44
|
+
* change: a relayed dial took 20-25 ms at no delay, **7 255-7 279 ms at 900 ms one-way**, and
|
|
45
|
+
* **12 066-12 094 ms at 1 500 ms one-way**. Four link round trips of pure delay would be 7 200
|
|
46
|
+
* and 12 000, so the real cost sits about 60-95 ms ABOVE the arithmetic — the handshakes and a
|
|
47
|
+
* phone's pure-JS crypto, which no delay figure contains. That residue is small but it is why
|
|
48
|
+
* {@link DECLARED_LINK_ROUND_TRIP_MS} carries headroom over the band sereus supports rather
|
|
49
|
+
* than matching it exactly: a declaration equal to the measured round trip would derive a
|
|
50
|
+
* budget marginally BELOW the dial it has to contain.
|
|
51
|
+
*
|
|
52
|
+
* ── The ceiling this does NOT lift ──
|
|
53
|
+
*
|
|
54
|
+
* Above roughly 1 250 ms one-way — a 2.5-second link round trip — no relayed connection can be
|
|
55
|
+
* established by this stack whatever cadre declares here. Two budgets of 10 000 ms each inside
|
|
56
|
+
* libp2p bound the same dial and cannot be reached from sereus at all:
|
|
57
|
+
* `connectionManager.dialTimeout` (libp2p's own default, which `@optimystic/db-p2p`'s
|
|
58
|
+
* `libp2p-node-base.ts` neither sets nor exposes) and `connectionManager.inboundUpgradeTimeout`
|
|
59
|
+
* (which it sets to 10 000). Both appear in the same re-run: at 1 500 ms one-way the dial under
|
|
60
|
+
* libp2p's default failed with `The operation was aborted due to timeout`, and with the shipped
|
|
61
|
+
* inbound-upgrade budget the DIALER's own 12 094 ms dial resolved while the LISTENER had
|
|
62
|
+
* already thrown the half-built connection away — so the first stream over it died with
|
|
63
|
+
* `Unexpected EOF - stream closed while reading 0/1 bytes` and the listener reported no peer at
|
|
64
|
+
* all. That is why a too-slow link produces silence rather than an error.
|
|
65
|
+
*
|
|
66
|
+
* Two consequences for a reader choosing a declaration. Below that ceiling, the DEFAULT
|
|
67
|
+
* declaration of 2 000 ms does not reach it: four round trips of 2 000 ms is 8 000 ms, which
|
|
68
|
+
* covers a one-way delay up to 1 000 ms, so a deployment somewhere between 1 000 and 1 250 ms
|
|
69
|
+
* one-way has to declare its own (about 2 200-2 500) to use the band libp2p still allows. And
|
|
70
|
+
* declaring much ABOVE 2 500 buys nothing: the budgets here grow, and
|
|
71
|
+
* the connection still fails inside libp2p. Lifting that needs an upstream change and a
|
|
72
|
+
* decision about how slow a link sereus intends to carry —
|
|
73
|
+
* `tickets/blocked/how-slow-a-relayed-link-does-sereus-carry`. A reader who raised these
|
|
74
|
+
* numbers and still cannot connect at 1 500 ms one-way has met that ceiling, not this module.
|
|
75
|
+
*/
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* The link round trip cadre assumes when a host declares none, in milliseconds. A deployment
|
|
79
|
+
* that knows its own link states it with `NetworkConfig.linkRoundTripMs` instead.
|
|
80
|
+
*
|
|
81
|
+
* **Why 2 000.** It states the slowest link sereus already claims to support, and two other
|
|
82
|
+
* budgets fix that claim independently: `COHORT_READ_DEADLINE_MS`
|
|
83
|
+
* (`quereus-plugin-sereus/src/cluster-size.ts`, 5 000 ms) and
|
|
84
|
+
* `DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS` (`strand-first-sync-gate.ts`, 120 000 ms) were both
|
|
85
|
+
* sized against two parties reaching each other only through a public relay at a round trip
|
|
86
|
+
* near 1.8 s. Declaring 2 000 rounds that band up and leaves about 720 ms over the worst
|
|
87
|
+
* relayed dial measured at it (7 279 ms against a derived 8 000 ms).
|
|
88
|
+
*
|
|
89
|
+
* Raising it is how a deployment that KNOWS its link is slower moves every derived budget at
|
|
90
|
+
* once. The cost is the ordinary cost of a longer deadline: a peer that is genuinely gone holds
|
|
91
|
+
* the operation for that much longer before it is abandoned and retried. Above about 2 500 it
|
|
92
|
+
* buys nothing — see the module doc's ceiling paragraph.
|
|
93
|
+
*/
|
|
94
|
+
export const DECLARED_LINK_ROUND_TRIP_MS = 2000;
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Link round trips one relayed connection setup costs: transport, encryption and multiplexer
|
|
98
|
+
* handshakes to the relay, the circuit open, then both handshakes again end to end through it.
|
|
99
|
+
* Measured at 8 one-way delays, which is 4 link round trips.
|
|
100
|
+
*/
|
|
101
|
+
export const RELAYED_DIAL_ROUND_TRIPS = 4;
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Link round trips one whole reservation drive is budgeted for
|
|
105
|
+
* (`relay-reservation.ts`'s `driveRelayReservation`, whose ONE deadline covers the dial, the
|
|
106
|
+
* reservation request and the wait together).
|
|
107
|
+
*
|
|
108
|
+
* The protocol work measured is 2 — dial the relay (1) plus request the reservation (1). It is
|
|
109
|
+
* budgeted at 4, the relayed-dial count, because the wait afterwards may instead be satisfied
|
|
110
|
+
* by libp2p's own relay discovery, which repeats both legs after identify has run; 4 is
|
|
111
|
+
* therefore the largest thing this one deadline can be asked to contain, and the poll interval
|
|
112
|
+
* (`DEFAULT_RELAY_RESERVE_POLL_MS`) rounds up on top of it.
|
|
113
|
+
*/
|
|
114
|
+
export const RELAY_RESERVATION_ROUND_TRIPS = 4;
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* Link round trips one request-and-answer over an ALREADY-OPEN circuit costs: the protocol
|
|
118
|
+
* negotiation (1, measured at 2 one-way delays) plus the request and its response (1). It does
|
|
119
|
+
* not include a dial — a caller that may have to open the connection budgets
|
|
120
|
+
* {@link RELAYED_DIAL_ROUND_TRIPS} separately, which is exactly what the two-field shape of
|
|
121
|
+
* Optimystic's `dialTimeoutMs` / `responseTimeoutMs` pair is for.
|
|
122
|
+
*/
|
|
123
|
+
export const CIRCUIT_REQUEST_ROUND_TRIPS = 2;
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* What a block-transfer push message's BYTES get to cross, on top of the latency
|
|
127
|
+
* {@link CIRCUIT_REQUEST_ROUND_TRIPS} covers, in milliseconds.
|
|
128
|
+
*
|
|
129
|
+
* This part is bandwidth, not latency, so it does not scale with the declared round trip: a
|
|
130
|
+
* chunk is capped at `PeerJoinBackfillConfig.maxChunkBytes` (1 MiB by default) and crossing it
|
|
131
|
+
* costs whatever the link's throughput costs.
|
|
132
|
+
*
|
|
133
|
+
* NOTE: 6 000 ms is not a measurement of any throughput — it is the residue of the 10 000 ms
|
|
134
|
+
* this deadline shipped as before it was split into a latency part and a transfer part, chosen
|
|
135
|
+
* so that at the default declaration the derived value is the same 10 000 and no fast link got
|
|
136
|
+
* slower. Nobody has measured how long 1 MiB takes to cross a relayed mobile link. If pushes
|
|
137
|
+
* ever time out with the transfer only part done, measure that and raise THIS, not the declared
|
|
138
|
+
* round trip.
|
|
139
|
+
*/
|
|
140
|
+
export const PUSH_TRANSFER_ALLOWANCE_MS = 6000;
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Resolve a host's declared link round trip, falling back to
|
|
144
|
+
* {@link DECLARED_LINK_ROUND_TRIP_MS}.
|
|
145
|
+
*
|
|
146
|
+
* Validated here rather than downstream because nothing downstream would: every consumer
|
|
147
|
+
* multiplies this value into a `setTimeout` deadline, where a zero silently turns a budget into
|
|
148
|
+
* "give up immediately" and a `NaN` turns it into "never time out".
|
|
149
|
+
*
|
|
150
|
+
* Both node bring-up paths call this EAGERLY — `CadreNode.start()`, and
|
|
151
|
+
* `StrandInstanceManager.buildStrandRuntime` behind `addStrand`/`resumeStrand` — so a bad
|
|
152
|
+
* declaration fails the same start that Optimystic's own check on `cohortQueryTimeoutMs` fails.
|
|
153
|
+
* The eager call is what makes that true: every budget derived here is behind a condition (a
|
|
154
|
+
* node with no control storage builds no catch-up, a node with no relay addrs drives no
|
|
155
|
+
* reservation), so waiting for a first consumer would let a broken declaration boot and then
|
|
156
|
+
* throw inside a best-effort path that logs and carries on.
|
|
157
|
+
*/
|
|
158
|
+
export function resolveLinkRoundTripMs(linkRoundTripMs?: number): number {
|
|
159
|
+
if (linkRoundTripMs === undefined) {
|
|
160
|
+
return DECLARED_LINK_ROUND_TRIP_MS;
|
|
161
|
+
}
|
|
162
|
+
if (!Number.isFinite(linkRoundTripMs) || linkRoundTripMs <= 0) {
|
|
163
|
+
throw new Error(
|
|
164
|
+
`network.linkRoundTripMs must be a finite number of milliseconds above zero, not ${String(linkRoundTripMs)}`
|
|
165
|
+
);
|
|
166
|
+
}
|
|
167
|
+
return linkRoundTripMs;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* Deadline for OPENING a connection to another machine that may only be reachable through a
|
|
172
|
+
* relay: {@link RELAYED_DIAL_ROUND_TRIPS} at the declared link round trip.
|
|
173
|
+
*/
|
|
174
|
+
export function relayedDialBudgetMs(linkRoundTripMs?: number): number {
|
|
175
|
+
return RELAYED_DIAL_ROUND_TRIPS * resolveLinkRoundTripMs(linkRoundTripMs);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* Deadline for one whole relay reservation drive: {@link RELAY_RESERVATION_ROUND_TRIPS} at the
|
|
180
|
+
* declared link round trip.
|
|
181
|
+
*/
|
|
182
|
+
export function relayReservationBudgetMs(linkRoundTripMs?: number): number {
|
|
183
|
+
return RELAY_RESERVATION_ROUND_TRIPS * resolveLinkRoundTripMs(linkRoundTripMs);
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* Deadline for one request and its answer over a circuit that is already open —
|
|
188
|
+
* {@link CIRCUIT_REQUEST_ROUND_TRIPS} at the declared link round trip, plus `transferAllowanceMs`
|
|
189
|
+
* for the payload's own bytes.
|
|
190
|
+
*/
|
|
191
|
+
export function circuitRequestBudgetMs(
|
|
192
|
+
linkRoundTripMs?: number,
|
|
193
|
+
transferAllowanceMs = PUSH_TRANSFER_ALLOWANCE_MS
|
|
194
|
+
): number {
|
|
195
|
+
return CIRCUIT_REQUEST_ROUND_TRIPS * resolveLinkRoundTripMs(linkRoundTripMs) + transferAllowanceMs;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/** The two deadlines one peer-join catch-up push needs, both derived from the declared link. */
|
|
199
|
+
export interface PeerJoinPushBudget {
|
|
200
|
+
/** Opening the connection to the peer being caught up — a relayed dial, in the worst case. */
|
|
201
|
+
dialTimeoutMs: number;
|
|
202
|
+
/** The push request and its answer over that connection, including the chunk's own bytes. */
|
|
203
|
+
responseTimeoutMs: number;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
/**
|
|
207
|
+
* The dial and response deadlines a peer-join catch-up push needs at a declared link. One
|
|
208
|
+
* helper rather than two call-site expressions, because the two construction sites
|
|
209
|
+
* (`CadreNode`'s control catch-up and `StrandInstanceManager`'s per-strand one) and
|
|
210
|
+
* `DEFAULT_PEER_JOIN_BACKFILL` must not drift apart.
|
|
211
|
+
*/
|
|
212
|
+
export function peerJoinPushBudget(linkRoundTripMs?: number): PeerJoinPushBudget {
|
|
213
|
+
return {
|
|
214
|
+
dialTimeoutMs: relayedDialBudgetMs(linkRoundTripMs),
|
|
215
|
+
responseTimeoutMs: circuitRequestBudgetMs(linkRoundTripMs)
|
|
216
|
+
};
|
|
217
|
+
}
|
|
@@ -88,9 +88,10 @@
|
|
|
88
88
|
* annex the party's relay capacity.
|
|
89
89
|
* - An `'admit-for-relay'` connection that is NOT reserving is dropped: it has
|
|
90
90
|
* {@link RELAY_ADMISSION_RESERVE_DEADLINE_MS} to get a reservation ADMITTED
|
|
91
|
-
* at that hook, after which the underlying connection
|
|
92
|
-
*
|
|
93
|
-
* conversation" to "in it briefly,
|
|
91
|
+
* at that hook, after which the gate CLOSES the underlying connection so
|
|
92
|
+
* both ends let go of the socket. The guarantee above thus weakens on
|
|
93
|
+
* relay-enabled nodes from "never in the conversation" to "in it briefly,
|
|
94
|
+
* and can speak nothing".
|
|
94
95
|
*
|
|
95
96
|
* The deadline clears on reservation ADMISSION, not on the reservation's own
|
|
96
97
|
* success: this gate cannot observe the server-side `reserve()` outcome, so a
|
|
@@ -158,6 +159,7 @@ import debug from 'debug';
|
|
|
158
159
|
import type { ConnectionGater, PeerId, MultiaddrConnection } from '@libp2p/interface';
|
|
159
160
|
import { SEED_PROTOCOL } from './seed-bootstrap.js';
|
|
160
161
|
import { FORMATION_PROTOCOL } from './strand-formation-protocol.js';
|
|
162
|
+
import { withDeadline } from './control-stream.js';
|
|
161
163
|
|
|
162
164
|
const log = debug('sereus:cadre:connection-gater');
|
|
163
165
|
|
|
@@ -197,7 +199,7 @@ export const ADMISSION_DECISION_TIMEOUT_MS = 2_000;
|
|
|
197
199
|
/**
|
|
198
200
|
* How long an `'admit-for-relay'` connection may exist without a relay
|
|
199
201
|
* reservation being ADMITTED at the `denyInboundRelayReservation` hook, after
|
|
200
|
-
* which the gate
|
|
202
|
+
* which the gate closes the underlying connection. A reserving client asks for
|
|
201
203
|
* its slot immediately after the connection upgrades (`relay-reservation.ts`
|
|
202
204
|
* dials and requests in one drive; a strand node's configured circuit listener
|
|
203
205
|
* does the same from inside its own `listen()`), so a connection idle past this
|
|
@@ -205,6 +207,19 @@ export const ADMISSION_DECISION_TIMEOUT_MS = 2_000;
|
|
|
205
207
|
*/
|
|
206
208
|
export const RELAY_ADMISSION_RESERVE_DEADLINE_MS = 5_000;
|
|
207
209
|
|
|
210
|
+
/**
|
|
211
|
+
* Bound on the graceful close of an expired relay-only connection. It exists
|
|
212
|
+
* because `AbstractMultiaddrConnection.close()` awaits an `idle`/`drain` event
|
|
213
|
+
* when the connection still has unsent bytes, and that wait has no timeout of
|
|
214
|
+
* its own — an unsignalled one never ends. This gate writes nothing to a
|
|
215
|
+
* stranger, so the wait is not reachable today; the bound is here so a timer
|
|
216
|
+
* callback can never hold an unending await.
|
|
217
|
+
*
|
|
218
|
+
* Applied through {@link withDeadline}, not `AbortSignal.timeout`: the latter is
|
|
219
|
+
* not reliably present on React Native/Hermes, which loads this same module.
|
|
220
|
+
*/
|
|
221
|
+
export const RELAY_ADMISSION_CLOSE_TIMEOUT_MS = 2_000;
|
|
222
|
+
|
|
208
223
|
/**
|
|
209
224
|
* Default cap on concurrent relay reservations held by peers the membership
|
|
210
225
|
* check could not place (see {@link UnauthorizedReservationBudget}). Small on
|
|
@@ -351,7 +366,7 @@ export class UnauthorizedReservationBudget {
|
|
|
351
366
|
*
|
|
352
367
|
* An `'admit-for-relay'` verdict admits the connection and arms a
|
|
353
368
|
* `reserveDeadlineMs` timer against it; the timer is disarmed when a
|
|
354
|
-
* reservation for that peer is ADMITTED at the reservation hook, and
|
|
369
|
+
* reservation for that peer is ADMITTED at the reservation hook, and closes the
|
|
355
370
|
* underlying `MultiaddrConnection` when it fires first.
|
|
356
371
|
*
|
|
357
372
|
* NOTE: `base` is spread, so a gater passed as a CLASS INSTANCE would lose its
|
|
@@ -470,7 +485,7 @@ export async function decideWithinDeadline<T>(
|
|
|
470
485
|
}
|
|
471
486
|
}
|
|
472
487
|
|
|
473
|
-
/** One armed not-reserving deadline: the connection it will
|
|
488
|
+
/** One armed not-reserving deadline: the connection it will close, and its timer. */
|
|
474
489
|
interface PendingReserveDeadline {
|
|
475
490
|
maConn: MultiaddrConnection;
|
|
476
491
|
timer: ReturnType<typeof setTimeout>;
|
|
@@ -479,11 +494,11 @@ interface PendingReserveDeadline {
|
|
|
479
494
|
/**
|
|
480
495
|
* The not-reserving deadlines for `'admit-for-relay'` connections, keyed by
|
|
481
496
|
* remote peerId (a Set per peer — one peer can hold several in-flight
|
|
482
|
-
* connections). `arm` starts a timer that
|
|
497
|
+
* connections). `arm` starts a timer that closes the connection; `disarm`
|
|
483
498
|
* (called when a reservation for that peer is admitted) cancels every pending
|
|
484
499
|
* timer for the peer. Timers are unref'd so an armed deadline never holds a
|
|
485
500
|
* process open, and a timer that fires against an already-closed connection is
|
|
486
|
-
* a
|
|
501
|
+
* a no-op (`close()` returns at once on any status but `open`).
|
|
487
502
|
*
|
|
488
503
|
* Disarming is final for the connections it cancelled: a peer that reserved
|
|
489
504
|
* once and then lets its reservation lapse keeps a mute connection (the
|
|
@@ -528,11 +543,41 @@ class PendingReserveDeadlines {
|
|
|
528
543
|
if (entries?.size === 0) {
|
|
529
544
|
this.byPeer.delete(remotePeerId);
|
|
530
545
|
}
|
|
531
|
-
log('Relay-only admission expired for %s — no reservation admitted within %dms,
|
|
546
|
+
log('Relay-only admission expired for %s — no reservation admitted within %dms, closing the connection', remotePeerId, this.deadlineMs);
|
|
547
|
+
void this.drop(remotePeerId, entry.maConn);
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
/**
|
|
551
|
+
* End an expired relay-only connection: a bounded graceful close, with
|
|
552
|
+
* `abort()` only as the escalation. The order cannot be reversed — `abort()`
|
|
553
|
+
* marks the connection `aborted`, and `close()` returns immediately on any
|
|
554
|
+
* status that is not `open`, so aborting first would make the close a silent
|
|
555
|
+
* no-op. Closing a connection the peer already dropped returns at once
|
|
556
|
+
* without throwing, so the fallback stays unreached in the ordinary case.
|
|
557
|
+
*/
|
|
558
|
+
private async drop(remotePeerId: string, maConn: MultiaddrConnection): Promise<void> {
|
|
559
|
+
try {
|
|
560
|
+
await withDeadline(
|
|
561
|
+
RELAY_ADMISSION_CLOSE_TIMEOUT_MS,
|
|
562
|
+
`relay-only close for ${remotePeerId}`,
|
|
563
|
+
(signal) => maConn.close({ signal })
|
|
564
|
+
);
|
|
565
|
+
return;
|
|
566
|
+
} catch (error) {
|
|
567
|
+
log('Closing the expired relay-only connection from %s failed — aborting: %o', remotePeerId, error);
|
|
568
|
+
}
|
|
569
|
+
// NOTE: this fallback cannot actually free a WebSocket. `@libp2p/websockets`'
|
|
570
|
+
// `sendReset()` calls `websocket.close(1006)`, and 1006 is a reserved code
|
|
571
|
+
// RFC 6455 forbids an endpoint from sending, so `ws` throws and
|
|
572
|
+
// `AbstractMessageStream.abort()` swallows it — the local end flips to
|
|
573
|
+
// `aborted` while the socket stays up. Measured against
|
|
574
|
+
// @libp2p/websockets@10.1.3 (see tickets/blocked/report-libp2p-websockets-abort-close-code.md).
|
|
575
|
+
// `MultiaddrConnection` exposes no way down to the raw socket, so this is as
|
|
576
|
+
// far as this layer can go. Revisit — drop the fallback, or go back to a
|
|
577
|
+
// plain `abort()` — once that transport sends a legal reset code.
|
|
532
578
|
try {
|
|
533
|
-
|
|
579
|
+
maConn.abort(new Error(`relay-only admission expired: no relay reservation admitted within ${this.deadlineMs}ms`));
|
|
534
580
|
} catch (error) {
|
|
535
|
-
// Aborting a connection that already closed on its own is not a failure.
|
|
536
581
|
log('Aborting the expired relay-only connection from %s threw: %o', remotePeerId, error);
|
|
537
582
|
}
|
|
538
583
|
}
|
package/src/peer-dial.ts
CHANGED
|
@@ -27,32 +27,47 @@ import debug from 'debug';
|
|
|
27
27
|
import type { Connection } from '@libp2p/interface';
|
|
28
28
|
import type { Multiaddr } from '@multiformats/multiaddr';
|
|
29
29
|
import { withDeadline } from './control-stream.js';
|
|
30
|
+
import { relayedDialBudgetMs } from './link-budget.js';
|
|
30
31
|
|
|
31
32
|
const log = debug('sereus:cadre:peer-dial');
|
|
32
33
|
|
|
33
34
|
/**
|
|
34
|
-
* Default limit on ONE address's dial attempt, in ms
|
|
35
|
+
* Default limit on ONE address's dial attempt, in ms, at the DEFAULT declared link round trip:
|
|
36
|
+
* `RELAYED_DIAL_ROUND_TRIPS` (4) x `DECLARED_LINK_ROUND_TRIP_MS` (2000 ms) = 8000 ms, which is
|
|
37
|
+
* what this constant was already set to as a fixed number.
|
|
35
38
|
*
|
|
36
39
|
* It has to cover the slowest address that should succeed, because an attempt
|
|
37
40
|
* that needs longer fails the same way on every retry. That is a relayed dial on
|
|
38
41
|
* a mobile link: connect to the relay (transport, encryption and multiplexer
|
|
39
42
|
* handshakes, when no relay connection is open yet), open the circuit, then run
|
|
40
|
-
* both handshakes again end to end through the relay
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
*
|
|
43
|
+
* both handshakes again end to end through the relay. `link-budget.ts` is where that
|
|
44
|
+
* reasoning is now a measured count instead of an estimate — four link round trips, measured at
|
|
45
|
+
* 7 279 ms on a link whose round trip is 1.8 s — so the value MOVES with a host's
|
|
46
|
+
* `NetworkConfig.linkRoundTripMs` rather than pinning the band this node can reach to whatever
|
|
47
|
+
* number was typed here. A direct dial that works is far quicker — a phone's WebSocket dial to
|
|
48
|
+
* a node forwarded over `adb reverse` measured 1.6 s end to end.
|
|
49
|
+
*
|
|
50
|
+
* Deriving it rather than leaving it fixed is deliberate: at 8000 ms fixed, the per-address
|
|
51
|
+
* limit — not libp2p's own 10 s dial timeout — was the first thing to cut a relayed dial off,
|
|
52
|
+
* from about 1000 ms of one-way link delay upward. Nothing said so, and a reader raising
|
|
53
|
+
* `DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS` below would not have found it.
|
|
46
54
|
*
|
|
47
55
|
* NOTE: a `/p2p-circuit/webrtc` address also negotiates a WebRTC session after
|
|
48
56
|
* the relay leg. If such dials are ever seen failing at exactly this limit, raise
|
|
49
|
-
*
|
|
50
|
-
* rather than the per-peer limit, which cannot help an address that is itself
|
|
57
|
+
* the declared link round trip (or `network.controlCohort.perAddressDialTimeoutMs` on the
|
|
58
|
+
* affected nodes) rather than the per-peer limit, which cannot help an address that is itself
|
|
51
59
|
* cut off.
|
|
52
60
|
*
|
|
53
61
|
* Override per node with `network.controlCohort.perAddressDialTimeoutMs`.
|
|
54
62
|
*/
|
|
55
|
-
export const DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS =
|
|
63
|
+
export const DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS = relayedDialBudgetMs();
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* How many whole per-address dial attempts {@link DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS}
|
|
67
|
+
* budgets for: the address that works plus the three dead ones a lent cadre-host node is
|
|
68
|
+
* observed to report ahead of it (loopback and LAN WebSocket addresses, plus virtual adapters).
|
|
69
|
+
*/
|
|
70
|
+
export const CONTROL_COHORT_DIAL_ADDRESS_ATTEMPTS = 4;
|
|
56
71
|
|
|
57
72
|
/**
|
|
58
73
|
* Default limit on dialing ONE peer — every candidate address together — in ms.
|
|
@@ -66,18 +81,25 @@ export const DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS = 8_000;
|
|
|
66
81
|
*
|
|
67
82
|
* Sized to fit a typical list with dead addresses ahead of the one that works:
|
|
68
83
|
* a lent cadre-host node reports two to four addresses a phone can dial
|
|
69
|
-
* (loopback and LAN WebSocket addresses, plus any virtual adapters).
|
|
70
|
-
*
|
|
71
|
-
*
|
|
72
|
-
* working address sits behind four or more dropped ones does not fit, and fails
|
|
73
|
-
*
|
|
74
|
-
*
|
|
84
|
+
* (loopback and LAN WebSocket addresses, plus any virtual adapters). So it is
|
|
85
|
+
* {@link CONTROL_COHORT_DIAL_ADDRESS_ATTEMPTS} whole per-address budgets — three silently
|
|
86
|
+
* dropped addresses, then the one that works, each getting the FULL per-address limit. A peer
|
|
87
|
+
* whose working address sits behind four or more dropped ones does not fit, and fails each pass
|
|
88
|
+
* the same way; if that shows up, the remedy is ordering (try the address that last connected
|
|
89
|
+
* first), not a larger number here.
|
|
90
|
+
*
|
|
91
|
+
* Counted as whole per-address budgets rather than the flat 30_000 it was before, because the
|
|
92
|
+
* old number left only 6 s after three dropped addresses — enough for the 1.6 s direct dial the
|
|
93
|
+
* phone measured, but NOT for the relayed dial the per-address limit is sized to cover. A peer
|
|
94
|
+
* whose only working address was a relayed one behind two dead ones therefore could not be
|
|
95
|
+
* dialled at all, and the arithmetic in this comment was what said otherwise.
|
|
75
96
|
*
|
|
76
97
|
* Override per node with `network.controlCohort.dialTimeoutMs` — tests that
|
|
77
98
|
* drive dead addresses on purpose set it low so a pass's duration is a chosen
|
|
78
99
|
* number rather than a transitive libp2p default stretched by machine load.
|
|
79
100
|
*/
|
|
80
|
-
export const DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS =
|
|
101
|
+
export const DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS =
|
|
102
|
+
CONTROL_COHORT_DIAL_ADDRESS_ATTEMPTS * DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS;
|
|
81
103
|
|
|
82
104
|
/** Time limits for dialing one peer from a list of candidate addresses. */
|
|
83
105
|
export interface PeerDialBudget {
|