@serfab/cadre-core 1.4.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cadre-node.d.ts +38 -15
- package/dist/cadre-node.js +81 -29
- package/dist/cadre-node.js.map +1 -1
- package/dist/control-database.d.ts +24 -10
- package/dist/control-database.js +48 -16
- package/dist/control-database.js.map +1 -1
- package/dist/control-read-retry.d.ts +12 -0
- package/dist/control-read-retry.js +12 -0
- package/dist/control-read-retry.js.map +1 -1
- package/dist/control-write-retry.d.ts +5 -4
- package/dist/control-write-retry.js +24 -7
- package/dist/control-write-retry.js.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.js +5 -2
- package/dist/index.js.map +1 -1
- package/dist/link-budget.d.ts +224 -0
- package/dist/link-budget.js +239 -0
- package/dist/link-budget.js.map +1 -0
- package/dist/peer-dial.d.ts +36 -17
- package/dist/peer-dial.js +37 -17
- package/dist/peer-dial.js.map +1 -1
- package/dist/peer-join-backfill.d.ts +58 -4
- package/dist/peer-join-backfill.js +166 -20
- package/dist/peer-join-backfill.js.map +1 -1
- package/dist/relay-reservation.d.ts +59 -11
- package/dist/relay-reservation.js +169 -43
- package/dist/relay-reservation.js.map +1 -1
- package/dist/seed-bootstrap.js +2 -1
- package/dist/seed-bootstrap.js.map +1 -1
- package/dist/strand-first-sync-gate.d.ts +105 -22
- package/dist/strand-first-sync-gate.js +121 -27
- package/dist/strand-first-sync-gate.js.map +1 -1
- package/dist/strand-instance-manager.d.ts +27 -2
- package/dist/strand-instance-manager.js +60 -13
- package/dist/strand-instance-manager.js.map +1 -1
- package/dist/strand-membership-reconciler.d.ts +200 -45
- package/dist/strand-membership-reconciler.js +302 -86
- package/dist/strand-membership-reconciler.js.map +1 -1
- package/dist/strand-membership-writer.d.ts +3 -2
- package/dist/strand-membership-writer.js +67 -90
- package/dist/strand-membership-writer.js.map +1 -1
- package/dist/strand-network-config.d.ts +2 -2
- package/dist/strand-network-config.js +2 -2
- package/dist/types.d.ts +105 -9
- package/dist/types.js +20 -4
- package/dist/types.js.map +1 -1
- package/package.json +8 -8
- package/src/cadre-node.ts +7449 -7397
- package/src/control-database.ts +48 -16
- package/src/control-read-retry.ts +214 -202
- package/src/control-write-retry.ts +24 -7
- package/src/index.ts +582 -563
- package/src/link-budget.ts +264 -0
- package/src/peer-dial.ts +39 -17
- package/src/peer-join-backfill.ts +648 -478
- package/src/relay-reservation.ts +1119 -951
- package/src/seed-bootstrap.ts +2 -1
- package/src/strand-first-sync-gate.ts +352 -258
- package/src/strand-instance-manager.ts +1520 -1458
- package/src/strand-membership-reconciler.ts +334 -92
- package/src/strand-membership-writer.ts +1962 -1987
- package/src/strand-network-config.ts +2 -2
- package/src/types.ts +104 -8
|
@@ -17,9 +17,17 @@
|
|
|
17
17
|
* freshly initialized `StrandDatabase` back from the app — `StrandInstance.database` stays
|
|
18
18
|
* unset and the instance reports `'syncing'` — and probes on a cadence until the Header is
|
|
19
19
|
* readable AND a read of each app table settles, at which point the database is published
|
|
20
|
-
* and the instance goes `'active'`. A machine
|
|
21
|
-
*
|
|
22
|
-
* offline-first writes on a machine that has synced
|
|
20
|
+
* and the instance goes `'active'`. A machine whose store already holds the Header and every
|
|
21
|
+
* app table (a founder, and normally a restart or a hibernation resume) passes the probe on
|
|
22
|
+
* the first try and is never gated, so offline-first writes on a machine that has synced
|
|
23
|
+
* before keep working.
|
|
24
|
+
*
|
|
25
|
+
* "Has synced before" is decided by what the local store actually holds, not by whether this
|
|
26
|
+
* machine once read the strand: a machine that left soon after its first sync can hold less
|
|
27
|
+
* than it read, and is then gated like a joiner. The re-attach measurement on
|
|
28
|
+
* {@link DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS} saw exactly that — a machine back over the store
|
|
29
|
+
* it kept, without the `Strand.Header` collection it had read before it left. Gating it is
|
|
30
|
+
* right: writing into a collection this machine does not hold would fork it.
|
|
23
31
|
*
|
|
24
32
|
* Why the app tables too: the Header is ONE collection. It reaches a joiner ahead of the
|
|
25
33
|
* founder's app-table collections (peer-join backfill and pull-on-read deliver those
|
|
@@ -36,10 +44,12 @@
|
|
|
36
44
|
* every machine; the first write to it still creates its collection — see `docs/strands.md`
|
|
37
45
|
* ("Joining") for that residual.
|
|
38
46
|
*
|
|
39
|
-
* Owned by `StrandInstanceManager`: created in `buildStrandRuntime` for
|
|
40
|
-
* whose first probe
|
|
41
|
-
* resume rebuild re-probes over the same store), and
|
|
42
|
-
*
|
|
47
|
+
* Owned by `StrandInstanceManager`: created in `buildStrandRuntime` for every launch, armed
|
|
48
|
+
* only for a non-founder launch whose first probe fails, stopped and dropped in
|
|
49
|
+
* `releaseRuntime` (so a quiesce → resume rebuild re-probes over the same store), and opened
|
|
50
|
+
* by `publishDatabase` whenever the database is published without its loop — a launch that
|
|
51
|
+
* needed no gate, or a founder request that ran the bootstrap against the still-gated
|
|
52
|
+
* database (`foundExistingStrand`).
|
|
43
53
|
*
|
|
44
54
|
* NOTE: the gate covers app and membership tables, not the schema catalog. Every launch —
|
|
45
55
|
* joiner or founder — writes the catalog collection (`optimystic/schema` plus two
|
|
@@ -57,18 +67,80 @@ import { type TimeoutScheduler } from './timeout-scheduler.js';
|
|
|
57
67
|
/** How often a gated launch re-probes (`Strand.Header`, then every `App` table) while waiting for its first sync. */
|
|
58
68
|
export declare const DEFAULT_STRAND_FIRST_SYNC_POLL_MS = 500;
|
|
59
69
|
/**
|
|
60
|
-
* How long `CadreNode.addStrand` waits for a
|
|
61
|
-
*
|
|
62
|
-
*
|
|
63
|
-
*
|
|
64
|
-
*
|
|
70
|
+
* How long `CadreNode.addStrand` waits for a machine's first sync before rejecting with
|
|
71
|
+
* {@link StrandAwaitingFirstSyncError}. It has to cover the slowest attach sereus carries,
|
|
72
|
+
* which is not a first join: a machine RE-attaching through a relay on a slow link can take
|
|
73
|
+
* longer than one joining for the first time.
|
|
74
|
+
*
|
|
75
|
+
* Measured over a DIRECT connection (2026-09-16): the joiner reads the founder's rows about
|
|
76
|
+
* 1.3 s after `addStrand` would previously have resolved. That is the fast case, and it is not
|
|
77
|
+
* what this budget has to cover.
|
|
78
|
+
*
|
|
79
|
+
* Every harness figure below was taken on one shape: one Windows developer machine, two
|
|
80
|
+
* relay-only `CadreNode`s (`listenAddrs: []`) on a shared loopback dedicated relay, with a
|
|
81
|
+
* 900 ms one-way per-frame outbound delay applied to every websocket by
|
|
82
|
+
* `integration-tests/src/harness/ws-latency.ts` in `pipelined` mode — frames stay overlapped
|
|
83
|
+
* in flight, so the figure is latency and bandwidth stays unlimited; the harness's other mode
|
|
84
|
+
* (`serial`) is a per-socket frame-rate cap and its delays are NOT comparable
|
|
85
|
+
* (`docs/testing.md` → "Link latency"). A round trip therefore costs about 1.8 s. Times run
|
|
86
|
+
* from the `addStrand` call to writable.
|
|
87
|
+
*
|
|
88
|
+
* FRESH JOIN (2026-09-26), a machine holding nothing of the strand yet: 23, 27, 31 and 41 s
|
|
89
|
+
* over four runs at optimystic's 1000 ms cohort read deadline, and 35, 42 and 46 s over three
|
|
90
|
+
* runs at a 5000 ms one; the re-attach scenario below later took 38.7, 38.7 and 56.9 s for
|
|
91
|
+
* the same fresh join. **The 5000 ms band, 35-57 s, is the one in force**: sereus declares
|
|
92
|
+
* 5000 ms (`COHORT_READ_DEADLINE_MS`, `quereus-plugin-sereus/src/cluster-size.ts`), because
|
|
93
|
+
* 1000 ms is shorter than one round trip on that link and leaves every cohort read with no
|
|
94
|
+
* answer to corroborate against. Widening that deadline makes a consult against a peer that
|
|
95
|
+
* cannot answer cost longer, and this phase runs several of those — which is why both bands
|
|
96
|
+
* are recorded here, and why a change to either number has to be weighed against the other.
|
|
97
|
+
*
|
|
98
|
+
* RE-ATTACH (2026-09-26, the opt-in
|
|
99
|
+
* `integration-tests/src/scenarios/strand-reattach-first-sync-measure.integration.ts` — re-run
|
|
100
|
+
* it before changing this number): B attached once on an undelayed link, left with
|
|
101
|
+
* `stopStrand`, A wrote while it was away, the delay was raised, and B called `addStrand`
|
|
102
|
+
* again.
|
|
103
|
+
* - Over an EMPTY store (one minted per launch, cadre's default with no storage provider):
|
|
104
|
+
* gated in all four runs and writable at 38.7-56.8 s, the same as a fresh join.
|
|
105
|
+
* - Over the store it KEPT (a phone with durable storage), bimodal. In 3 of 8 runs the store
|
|
106
|
+
* held everything and the launch came up writable (5.6 s, the launch itself). In the other
|
|
107
|
+
* 5 it lacked the `Strand.Header` collection B had read on its first attach, so B was gated
|
|
108
|
+
* like a joiner and became writable at 64.1-78.7 s — the slowest attach measured, about
|
|
109
|
+
* twice a fresh join. Why syncing over a partial store is slower sits inside optimystic
|
|
110
|
+
* (`tickets/blocked/report-reattach-over-partial-replica-to-optimystic`).
|
|
111
|
+
* - Neither a redial nor catching up on missed writes explains the time: B's strand node
|
|
112
|
+
* connected to A's at 11 s in every arm, fresh join included, and a run with 20 missed
|
|
113
|
+
* writes fell inside the run-to-run spread of those with 1.
|
|
114
|
+
* This budget covers WRITABLE, not caught up: in the gated kept-store runs the row A wrote
|
|
115
|
+
* while B was away became readable only at 140-162 s.
|
|
116
|
+
*
|
|
117
|
+
* Two samples from outside this harness, neither reproduced here and neither discarded: a
|
|
118
|
+
* reporter's re-attach of the same shape (optimystic #22; their harness, storage and delay
|
|
119
|
+
* injector are unknown to us) opened the gate at about 150 s, after the previous 120 s budget
|
|
120
|
+
* had already rejected; and a real Galaxy S7 joining fresh through relay.sereus.org took 178 s.
|
|
121
|
+
*
|
|
122
|
+
* 300 s clears the worst harness sample (79 s) by about 3.8x, the reporter's re-attach by 2x
|
|
123
|
+
* and the device join by about 1.7x. 240 s was rejected: it clears the device join by only
|
|
124
|
+
* 1.35x, and that sample was not a re-attach. History: the original 30 s sat INSIDE the 1000 ms
|
|
125
|
+
* fresh band and refused about half of those joins while their sync was progressing normally;
|
|
126
|
+
* 120 s was sized from fresh joins only and refused the reporter's re-attach.
|
|
127
|
+
*
|
|
128
|
+
* NOTE: accepted tradeoff — what 300 s costs. This wait is what an app's `addStrand` sits in
|
|
129
|
+
* before it is told "not yet", so a strand none of whose members is reachable at all takes five
|
|
130
|
+
* minutes to report instead of two. The slow report was weighed against refusing attaches that
|
|
131
|
+
* were working, and the refusal is the worse failure. The cost is bounded — the rejection is
|
|
132
|
+
* retryable, the strand stays launched and keeps probing, and `strand:writable` fires the
|
|
133
|
+
* moment the sync lands, so an app that listens for the event rather than awaiting the call is
|
|
134
|
+
* unaffected either way. Revisit if the gate ever learns whether ANY other member is connected:
|
|
135
|
+
* "no peer at all" could then be reported at once and this budget would only ever be spent on a
|
|
136
|
+
* sync that is actually in progress — worth more the longer this budget gets.
|
|
65
137
|
*/
|
|
66
|
-
export declare const DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS =
|
|
138
|
+
export declare const DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS = 300000;
|
|
67
139
|
/** Embedder-facing tuning for the gate, threaded from `CadreNodeConfig.strandFirstSync`. */
|
|
68
140
|
export interface StrandFirstSyncConfig {
|
|
69
141
|
/**
|
|
70
|
-
* Default wait for `CadreNode.addStrand` (and `whenStrandWritable`) before a
|
|
71
|
-
*
|
|
142
|
+
* Default wait for `CadreNode.addStrand` (and `whenStrandWritable`) before a machine whose
|
|
143
|
+
* first sync has not completed is reported as not yet writable. Default
|
|
72
144
|
* {@link DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS}.
|
|
73
145
|
*/
|
|
74
146
|
timeoutMs?: number;
|
|
@@ -76,11 +148,12 @@ export interface StrandFirstSyncConfig {
|
|
|
76
148
|
pollIntervalMs?: number;
|
|
77
149
|
}
|
|
78
150
|
/**
|
|
79
|
-
* Thrown by `CadreNode.addStrand` / `whenStrandWritable` when a
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
*
|
|
83
|
-
* `
|
|
151
|
+
* Thrown by `CadreNode.addStrand` / `whenStrandWritable` when a machine has not received the
|
|
152
|
+
* strand's data from another member within the wait budget — because no other member is
|
|
153
|
+
* reachable, or because one is and the sync is still arriving over a slow link; the gate
|
|
154
|
+
* cannot tell the two apart. RETRYABLE: the strand stays launched and keeps probing, so a
|
|
155
|
+
* later `addStrand` (or `whenStrandWritable`) for the same strand completes the attach once
|
|
156
|
+
* the data has arrived; the `strand:writable` event fires at that moment too.
|
|
84
157
|
*/
|
|
85
158
|
export declare class StrandAwaitingFirstSyncError extends Error {
|
|
86
159
|
readonly strandId: string;
|
|
@@ -141,12 +214,22 @@ export declare class StrandFirstSyncGate {
|
|
|
141
214
|
/** Arm the probe loop. The first probe runs after one interval — the caller already probed once. */
|
|
142
215
|
start(): void;
|
|
143
216
|
/**
|
|
144
|
-
*
|
|
145
|
-
*
|
|
217
|
+
* Open without the probe loop: the caller is publishing the database itself — a launch
|
|
218
|
+
* whose first probe passed (or a founder's), which never armed the loop, or a founder
|
|
219
|
+
* bootstrap just run against the gated database. Stops the loop; `onHeaderHeld` is NOT
|
|
220
|
+
* invoked.
|
|
146
221
|
*/
|
|
147
222
|
open(): void;
|
|
148
223
|
/** Disarm the loop; a probe already in flight completes and is then ignored. */
|
|
149
224
|
stop(): void;
|
|
150
225
|
private schedule;
|
|
151
226
|
private probe;
|
|
227
|
+
/**
|
|
228
|
+
* One probe's answer, with a throw from OUTSIDE {@link strandFirstSyncComplete}'s own read
|
|
229
|
+
* guards — the `App` schema lookup, or `getDatabase()` itself — reported as "not yet" like
|
|
230
|
+
* any failing read. The loop is scheduled from `void this.probe()`, so an escaping rejection
|
|
231
|
+
* would be unhandled AND leave nothing scheduled: the strand would stay gated for the whole
|
|
232
|
+
* `timeoutMs` and never recover, which is the one failure this gate must not have.
|
|
233
|
+
*/
|
|
234
|
+
private probeHeld;
|
|
152
235
|
}
|
|
@@ -17,9 +17,17 @@
|
|
|
17
17
|
* freshly initialized `StrandDatabase` back from the app — `StrandInstance.database` stays
|
|
18
18
|
* unset and the instance reports `'syncing'` — and probes on a cadence until the Header is
|
|
19
19
|
* readable AND a read of each app table settles, at which point the database is published
|
|
20
|
-
* and the instance goes `'active'`. A machine
|
|
21
|
-
*
|
|
22
|
-
* offline-first writes on a machine that has synced
|
|
20
|
+
* and the instance goes `'active'`. A machine whose store already holds the Header and every
|
|
21
|
+
* app table (a founder, and normally a restart or a hibernation resume) passes the probe on
|
|
22
|
+
* the first try and is never gated, so offline-first writes on a machine that has synced
|
|
23
|
+
* before keep working.
|
|
24
|
+
*
|
|
25
|
+
* "Has synced before" is decided by what the local store actually holds, not by whether this
|
|
26
|
+
* machine once read the strand: a machine that left soon after its first sync can hold less
|
|
27
|
+
* than it read, and is then gated like a joiner. The re-attach measurement on
|
|
28
|
+
* {@link DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS} saw exactly that — a machine back over the store
|
|
29
|
+
* it kept, without the `Strand.Header` collection it had read before it left. Gating it is
|
|
30
|
+
* right: writing into a collection this machine does not hold would fork it.
|
|
23
31
|
*
|
|
24
32
|
* Why the app tables too: the Header is ONE collection. It reaches a joiner ahead of the
|
|
25
33
|
* founder's app-table collections (peer-join backfill and pull-on-read deliver those
|
|
@@ -36,10 +44,12 @@
|
|
|
36
44
|
* every machine; the first write to it still creates its collection — see `docs/strands.md`
|
|
37
45
|
* ("Joining") for that residual.
|
|
38
46
|
*
|
|
39
|
-
* Owned by `StrandInstanceManager`: created in `buildStrandRuntime` for
|
|
40
|
-
* whose first probe
|
|
41
|
-
* resume rebuild re-probes over the same store), and
|
|
42
|
-
*
|
|
47
|
+
* Owned by `StrandInstanceManager`: created in `buildStrandRuntime` for every launch, armed
|
|
48
|
+
* only for a non-founder launch whose first probe fails, stopped and dropped in
|
|
49
|
+
* `releaseRuntime` (so a quiesce → resume rebuild re-probes over the same store), and opened
|
|
50
|
+
* by `publishDatabase` whenever the database is published without its loop — a launch that
|
|
51
|
+
* needed no gate, or a founder request that ran the bootstrap against the still-gated
|
|
52
|
+
* database (`foundExistingStrand`).
|
|
43
53
|
*
|
|
44
54
|
* NOTE: the gate covers app and membership tables, not the schema catalog. Every launch —
|
|
45
55
|
* joiner or founder — writes the catalog collection (`optimystic/schema` plus two
|
|
@@ -57,26 +67,90 @@ const log = debug('sereus:cadre:strand-first-sync');
|
|
|
57
67
|
/** How often a gated launch re-probes (`Strand.Header`, then every `App` table) while waiting for its first sync. */
|
|
58
68
|
export const DEFAULT_STRAND_FIRST_SYNC_POLL_MS = 500;
|
|
59
69
|
/**
|
|
60
|
-
* How long `CadreNode.addStrand` waits for a
|
|
61
|
-
*
|
|
62
|
-
*
|
|
63
|
-
*
|
|
64
|
-
*
|
|
70
|
+
* How long `CadreNode.addStrand` waits for a machine's first sync before rejecting with
|
|
71
|
+
* {@link StrandAwaitingFirstSyncError}. It has to cover the slowest attach sereus carries,
|
|
72
|
+
* which is not a first join: a machine RE-attaching through a relay on a slow link can take
|
|
73
|
+
* longer than one joining for the first time.
|
|
74
|
+
*
|
|
75
|
+
* Measured over a DIRECT connection (2026-09-16): the joiner reads the founder's rows about
|
|
76
|
+
* 1.3 s after `addStrand` would previously have resolved. That is the fast case, and it is not
|
|
77
|
+
* what this budget has to cover.
|
|
78
|
+
*
|
|
79
|
+
* Every harness figure below was taken on one shape: one Windows developer machine, two
|
|
80
|
+
* relay-only `CadreNode`s (`listenAddrs: []`) on a shared loopback dedicated relay, with a
|
|
81
|
+
* 900 ms one-way per-frame outbound delay applied to every websocket by
|
|
82
|
+
* `integration-tests/src/harness/ws-latency.ts` in `pipelined` mode — frames stay overlapped
|
|
83
|
+
* in flight, so the figure is latency and bandwidth stays unlimited; the harness's other mode
|
|
84
|
+
* (`serial`) is a per-socket frame-rate cap and its delays are NOT comparable
|
|
85
|
+
* (`docs/testing.md` → "Link latency"). A round trip therefore costs about 1.8 s. Times run
|
|
86
|
+
* from the `addStrand` call to writable.
|
|
87
|
+
*
|
|
88
|
+
* FRESH JOIN (2026-09-26), a machine holding nothing of the strand yet: 23, 27, 31 and 41 s
|
|
89
|
+
* over four runs at optimystic's 1000 ms cohort read deadline, and 35, 42 and 46 s over three
|
|
90
|
+
* runs at a 5000 ms one; the re-attach scenario below later took 38.7, 38.7 and 56.9 s for
|
|
91
|
+
* the same fresh join. **The 5000 ms band, 35-57 s, is the one in force**: sereus declares
|
|
92
|
+
* 5000 ms (`COHORT_READ_DEADLINE_MS`, `quereus-plugin-sereus/src/cluster-size.ts`), because
|
|
93
|
+
* 1000 ms is shorter than one round trip on that link and leaves every cohort read with no
|
|
94
|
+
* answer to corroborate against. Widening that deadline makes a consult against a peer that
|
|
95
|
+
* cannot answer cost longer, and this phase runs several of those — which is why both bands
|
|
96
|
+
* are recorded here, and why a change to either number has to be weighed against the other.
|
|
97
|
+
*
|
|
98
|
+
* RE-ATTACH (2026-09-26, the opt-in
|
|
99
|
+
* `integration-tests/src/scenarios/strand-reattach-first-sync-measure.integration.ts` — re-run
|
|
100
|
+
* it before changing this number): B attached once on an undelayed link, left with
|
|
101
|
+
* `stopStrand`, A wrote while it was away, the delay was raised, and B called `addStrand`
|
|
102
|
+
* again.
|
|
103
|
+
* - Over an EMPTY store (one minted per launch, cadre's default with no storage provider):
|
|
104
|
+
* gated in all four runs and writable at 38.7-56.8 s, the same as a fresh join.
|
|
105
|
+
* - Over the store it KEPT (a phone with durable storage), bimodal. In 3 of 8 runs the store
|
|
106
|
+
* held everything and the launch came up writable (5.6 s, the launch itself). In the other
|
|
107
|
+
* 5 it lacked the `Strand.Header` collection B had read on its first attach, so B was gated
|
|
108
|
+
* like a joiner and became writable at 64.1-78.7 s — the slowest attach measured, about
|
|
109
|
+
* twice a fresh join. Why syncing over a partial store is slower sits inside optimystic
|
|
110
|
+
* (`tickets/blocked/report-reattach-over-partial-replica-to-optimystic`).
|
|
111
|
+
* - Neither a redial nor catching up on missed writes explains the time: B's strand node
|
|
112
|
+
* connected to A's at 11 s in every arm, fresh join included, and a run with 20 missed
|
|
113
|
+
* writes fell inside the run-to-run spread of those with 1.
|
|
114
|
+
* This budget covers WRITABLE, not caught up: in the gated kept-store runs the row A wrote
|
|
115
|
+
* while B was away became readable only at 140-162 s.
|
|
116
|
+
*
|
|
117
|
+
* Two samples from outside this harness, neither reproduced here and neither discarded: a
|
|
118
|
+
* reporter's re-attach of the same shape (optimystic #22; their harness, storage and delay
|
|
119
|
+
* injector are unknown to us) opened the gate at about 150 s, after the previous 120 s budget
|
|
120
|
+
* had already rejected; and a real Galaxy S7 joining fresh through relay.sereus.org took 178 s.
|
|
121
|
+
*
|
|
122
|
+
* 300 s clears the worst harness sample (79 s) by about 3.8x, the reporter's re-attach by 2x
|
|
123
|
+
* and the device join by about 1.7x. 240 s was rejected: it clears the device join by only
|
|
124
|
+
* 1.35x, and that sample was not a re-attach. History: the original 30 s sat INSIDE the 1000 ms
|
|
125
|
+
* fresh band and refused about half of those joins while their sync was progressing normally;
|
|
126
|
+
* 120 s was sized from fresh joins only and refused the reporter's re-attach.
|
|
127
|
+
*
|
|
128
|
+
* NOTE: accepted tradeoff — what 300 s costs. This wait is what an app's `addStrand` sits in
|
|
129
|
+
* before it is told "not yet", so a strand none of whose members is reachable at all takes five
|
|
130
|
+
* minutes to report instead of two. The slow report was weighed against refusing attaches that
|
|
131
|
+
* were working, and the refusal is the worse failure. The cost is bounded — the rejection is
|
|
132
|
+
* retryable, the strand stays launched and keeps probing, and `strand:writable` fires the
|
|
133
|
+
* moment the sync lands, so an app that listens for the event rather than awaiting the call is
|
|
134
|
+
* unaffected either way. Revisit if the gate ever learns whether ANY other member is connected:
|
|
135
|
+
* "no peer at all" could then be reported at once and this budget would only ever be spent on a
|
|
136
|
+
* sync that is actually in progress — worth more the longer this budget gets.
|
|
65
137
|
*/
|
|
66
|
-
export const DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS =
|
|
138
|
+
export const DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS = 300000;
|
|
67
139
|
/**
|
|
68
|
-
* Thrown by `CadreNode.addStrand` / `whenStrandWritable` when a
|
|
69
|
-
*
|
|
70
|
-
*
|
|
71
|
-
*
|
|
72
|
-
* `
|
|
140
|
+
* Thrown by `CadreNode.addStrand` / `whenStrandWritable` when a machine has not received the
|
|
141
|
+
* strand's data from another member within the wait budget — because no other member is
|
|
142
|
+
* reachable, or because one is and the sync is still arriving over a slow link; the gate
|
|
143
|
+
* cannot tell the two apart. RETRYABLE: the strand stays launched and keeps probing, so a
|
|
144
|
+
* later `addStrand` (or `whenStrandWritable`) for the same strand completes the attach once
|
|
145
|
+
* the data has arrived; the `strand:writable` event fires at that moment too.
|
|
73
146
|
*/
|
|
74
147
|
export class StrandAwaitingFirstSyncError extends Error {
|
|
75
148
|
constructor(strandId, waitedMs) {
|
|
76
|
-
super(`Strand ${strandId} is not writable yet:
|
|
77
|
-
`
|
|
78
|
-
'
|
|
79
|
-
|
|
149
|
+
super(`Strand ${strandId} is not writable yet: this machine has not yet received the strand's data ` +
|
|
150
|
+
`from another member (waited ${waitedMs} ms). Either no other member is reachable, or the ` +
|
|
151
|
+
'link to one is slow and the sync is still in progress. The strand stays launched and keeps ' +
|
|
152
|
+
'trying — call addStrand or whenStrandWritable(strandId) again to keep waiting, or listen ' +
|
|
153
|
+
"for the 'strand:writable' event.");
|
|
80
154
|
this.name = 'StrandAwaitingFirstSyncError';
|
|
81
155
|
this.strandId = strandId;
|
|
82
156
|
this.waitedMs = waitedMs;
|
|
@@ -162,15 +236,17 @@ export class StrandFirstSyncGate {
|
|
|
162
236
|
log('[%s] first-sync gate armed: no Strand.Header held yet; probing every %dms', this.deps.label, this.pollIntervalMs);
|
|
163
237
|
}
|
|
164
238
|
/**
|
|
165
|
-
*
|
|
166
|
-
*
|
|
239
|
+
* Open without the probe loop: the caller is publishing the database itself — a launch
|
|
240
|
+
* whose first probe passed (or a founder's), which never armed the loop, or a founder
|
|
241
|
+
* bootstrap just run against the gated database. Stops the loop; `onHeaderHeld` is NOT
|
|
242
|
+
* invoked.
|
|
167
243
|
*/
|
|
168
244
|
open() {
|
|
169
245
|
if (this.opened)
|
|
170
246
|
return;
|
|
171
247
|
this.opened = true;
|
|
172
248
|
this.stop();
|
|
173
|
-
log('[%s]
|
|
249
|
+
log('[%s] database published by the caller; first-sync gate not needed or force-opened', this.deps.label);
|
|
174
250
|
}
|
|
175
251
|
/** Disarm the loop; a probe already in flight completes and is then ignored. */
|
|
176
252
|
stop() {
|
|
@@ -190,8 +266,7 @@ export class StrandFirstSyncGate {
|
|
|
190
266
|
if (this.stopped || this.opened || this.probing)
|
|
191
267
|
return;
|
|
192
268
|
this.probing = true;
|
|
193
|
-
const held = await
|
|
194
|
-
.finally(() => { this.probing = false; });
|
|
269
|
+
const held = await this.probeHeld();
|
|
195
270
|
// Re-check after the await: a release or a force-open may have landed mid-read.
|
|
196
271
|
if (this.stopped || this.opened)
|
|
197
272
|
return;
|
|
@@ -204,5 +279,24 @@ export class StrandFirstSyncGate {
|
|
|
204
279
|
log('[%s] first-sync gate opened: Strand.Header and every App table read from a peer', this.deps.label);
|
|
205
280
|
this.deps.onHeaderHeld();
|
|
206
281
|
}
|
|
282
|
+
/**
|
|
283
|
+
* One probe's answer, with a throw from OUTSIDE {@link strandFirstSyncComplete}'s own read
|
|
284
|
+
* guards — the `App` schema lookup, or `getDatabase()` itself — reported as "not yet" like
|
|
285
|
+
* any failing read. The loop is scheduled from `void this.probe()`, so an escaping rejection
|
|
286
|
+
* would be unhandled AND leave nothing scheduled: the strand would stay gated for the whole
|
|
287
|
+
* `timeoutMs` and never recover, which is the one failure this gate must not have.
|
|
288
|
+
*/
|
|
289
|
+
async probeHeld() {
|
|
290
|
+
try {
|
|
291
|
+
return await strandFirstSyncComplete(this.deps.database.getDatabase(), this.deps.label);
|
|
292
|
+
}
|
|
293
|
+
catch (error) {
|
|
294
|
+
log('[%s] first-sync probe threw (treated as not yet synced): %s', this.deps.label, error instanceof Error ? error.message : String(error));
|
|
295
|
+
return false;
|
|
296
|
+
}
|
|
297
|
+
finally {
|
|
298
|
+
this.probing = false;
|
|
299
|
+
}
|
|
300
|
+
}
|
|
207
301
|
}
|
|
208
302
|
//# sourceMappingURL=strand-first-sync-gate.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"strand-first-sync-gate.js","sourceRoot":"","sources":["../src/strand-first-sync-gate.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"strand-first-sync-gate.js","sourceRoot":"","sources":["../src/strand-first-sync-gate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8DG;AAEH,OAAO,KAAK,MAAM,OAAO,CAAC;AAG1B,OAAO,EAAE,uBAAuB,EAAyB,MAAM,wBAAwB,CAAC;AAExF,MAAM,GAAG,GAAG,KAAK,CAAC,gCAAgC,CAAC,CAAC;AAEpD,qHAAqH;AACrH,MAAM,CAAC,MAAM,iCAAiC,GAAG,GAAG,CAAC;AAErD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoEG;AACH,MAAM,CAAC,MAAM,oCAAoC,GAAG,MAAO,CAAC;AAc5D;;;;;;;GAOG;AACH,MAAM,OAAO,4BAA6B,SAAQ,KAAK;IAIrD,YAAY,QAAgB,EAAE,QAAgB;QAC5C,KAAK,CACH,UAAU,QAAQ,4EAA4E;YAC9F,+BAA+B,QAAQ,oDAAoD;YAC3F,6FAA6F;YAC7F,2FAA2F;YAC3F,kCAAkC,CACnC,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,8BAA8B,CAAC;QAC3C,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC3B,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,EAAY,EAAE,KAAa;IAChE,IAAI,CAAC;QACH,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,EAAE,CAAC,IAAI,CAAC,6CAA6C,CAAC,EAAE,CAAC;YAC/E,OAAO,CAAE,GAAG,CAAC,KAAgB,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QAC1C,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,GAAG,CAAC,wDAAwD,EAAE,KAAK,EACjE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QAC1D,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,iFAAiF;AACjF,SAAS,aAAa,CAAC,EAAY;IACjC,MAAM,GAAG,GAAG,EAAE,CAAC,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC9C,OAAO,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AAC/G,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,EAAY,EAAE,KAAa;IACjE,KAAK,MAAM,KAAK,IAAI,aAAa,CAAC,EAAE,CAAC,EAAE,CAAC;QACtC,IAAI,CAAC;YACH,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,EAAE,CAAC,IAAI,CAAC,sCAAsC,KAAK,GAAG,CAAC,EAAE,CAAC;gBACjF,MAAM;YACR,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,GAAG,CAAC,0DAA0D,EAAE,KAAK,EAAE,KAAK,EAC1E,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAC1D,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,EAAY,EAAE,KAAa;IACvE,OAAO,MAAM,gBAAgB,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,MAAM,iBAAiB,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;AACjF,CAAC;AAiBD;;;;GAIG;AACH,MAAM,OAAO,mBAAmB;IAQ9B,YAA6B,IAA6B,EAAE,MAA8B;QAA7D,SAAI,GAAJ,IAAI,CAAyB;QAJlD,YAAO,GAAG,KAAK,CAAC;QAChB,WAAM,GAAG,KAAK,CAAC;QACf,YAAO,GAAG,KAAK,CAAC;QAGtB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,uBAAuB,CAAC;QAC3D,IAAI,CAAC,cAAc,GAAG,MAAM,EAAE,cAAc,IAAI,iCAAiC,CAAC;IACpF,CAAC;IAED,8CAA8C;IAC9C,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC;IAC5B,CAAC;IAED,2FAA2F;IAC3F,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,oGAAoG;IACpG,KAAK;QACH,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS;YAAE,OAAO;QACpE,IAAI,CAAC,QAAQ,EAAE,CAAC;QAChB,GAAG,CAAC,2EAA2E,EAC7E,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;IAC1C,CAAC;IAED;;;;;OAKG;IACH,IAAI;QACF,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QACxB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,IAAI,EAAE,CAAC;QACZ,GAAG,CAAC,mFAAmF,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC5G,CAAC;IAED,gFAAgF;IAChF,IAAI;QACF,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC7B,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACxC,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC;QACzB,CAAC;IACH,CAAC;IAEO,QAAQ;QACd,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,GAAG,EAAE;YAC1C,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC;YACvB,KAAK,IAAI,CAAC,KAAK,EAAE,CAAC;QACpB,CAAC,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;IAC1B,CAAC;IAEO,KAAK,CAAC,KAAK;QACjB,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QACxD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QACpC,gFAAgF;QAChF,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QACxC,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,IAAI,EAAE,CAAC;QACZ,GAAG,CAAC,iFAAiF,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACxG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;IAC3B,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,SAAS;QACrB,IAAI,CAAC;YACH,OAAO,MAAM,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC1F,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,GAAG,CAAC,6DAA6D,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,EAChF,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAC1D,OAAO,KAAK,CAAC;QACf,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QACvB,CAAC;IACH,CAAC;CACF"}
|
|
@@ -157,6 +157,18 @@ export interface StartStrandConfig {
|
|
|
157
157
|
* this node's behalf — what to do about it is the app's call.
|
|
158
158
|
*/
|
|
159
159
|
onSelfRevoked?: (strandId: string) => void;
|
|
160
|
+
/**
|
|
161
|
+
* Called when the membership reconciler reports a blocked re-join — this node holds a
|
|
162
|
+
* staged membership invitation for the strand that its writes cannot redeem, which is
|
|
163
|
+
* what a REMOVED party handed a fresh invitation hits (see "Reporting a blocked
|
|
164
|
+
* re-join" in `strand-membership-reconciler.ts` for the two triggers, one of them a
|
|
165
|
+
* suspicion rather than a verdict). At most once per re-arm of the loop — a further
|
|
166
|
+
* invitation staged while the loop is still running does not reset the report — and
|
|
167
|
+
* only for closed strands with a party key. `CadreNode` wires it to its `strand:rejoin-blocked`
|
|
168
|
+
* event; nothing is stopped or torn down — a remaining manager admitting this party's
|
|
169
|
+
* key directly is the remedy, and the loop finishes the join by itself once that lands.
|
|
170
|
+
*/
|
|
171
|
+
onRejoinBlocked?: (strandId: string) => void;
|
|
160
172
|
/**
|
|
161
173
|
* Tuning for the JOINING machine's first-sync write gate (`strand-first-sync-gate.ts`),
|
|
162
174
|
* forwarded from {@link CadreNodeConfig.strandFirstSync}: the Header probe cadence while
|
|
@@ -370,8 +382,11 @@ export declare class StrandInstanceManager {
|
|
|
370
382
|
/**
|
|
371
383
|
* One {@link superviseRelayReservation} per relay dial addr, each over exactly
|
|
372
384
|
* that relay so "held" is judged per relay (`circuitMultiaddrsVia`) and losing
|
|
373
|
-
* one relay re-drives only that one. Default timings — the control node's (2 s
|
|
374
|
-
* doubling to 60 s between failed attempts, a 5 s liveness check while held).
|
|
385
|
+
* one relay re-drives only that one. Default retry timings — the control node's (2 s
|
|
386
|
+
* doubling to 60 s between failed attempts, a 5 s liveness check while held). Each DRIVE's
|
|
387
|
+
* own deadline is counted in link round trips at this host's declared
|
|
388
|
+
* `network.linkRoundTripMs` (`link-budget.ts`), so the same declaration that lengthens the
|
|
389
|
+
* control node's drive lengthens these.
|
|
375
390
|
*
|
|
376
391
|
* Each supervisor's `beforeRedrive` is the caller's
|
|
377
392
|
* {@link StartStrandConfig.announceDelegateToRelay} for THIS relay and THIS
|
|
@@ -416,6 +431,16 @@ export declare class StrandInstanceManager {
|
|
|
416
431
|
* finished.
|
|
417
432
|
*/
|
|
418
433
|
refreshRevocationEnforcement(strandId: string): Promise<void>;
|
|
434
|
+
/**
|
|
435
|
+
* A fresh membership invitation was staged for `strandId` (`CadreNode`'s
|
|
436
|
+
* `adoptFormationMembershipInvite`): re-arm the strand's membership reconciler so the
|
|
437
|
+
* invitation is attempted now rather than never — the loop finished during the first
|
|
438
|
+
* join and would otherwise stay stopped until a relaunch. Quiet no-op when no reconciler
|
|
439
|
+
* is armed: a first formation stages before the strand is added, and bring-up then arms
|
|
440
|
+
* a loop that finds the invitation by itself. Returns at once; the pass runs on the
|
|
441
|
+
* loop's own serialized chain and never rejects.
|
|
442
|
+
*/
|
|
443
|
+
notifyMembershipInviteStaged(strandId: string): void;
|
|
419
444
|
/**
|
|
420
445
|
* Best-effort removal of THIS machine's own `Strand.MemberPeer` binding — the
|
|
421
446
|
* self arm of `removeMemberPeer`, signed with the retained launch config's party
|
|
@@ -13,6 +13,7 @@ import { StrandFirstSyncGate, StrandAwaitingFirstSyncError, strandFirstSyncCompl
|
|
|
13
13
|
import { DEFAULT_CONNECTION_MONITOR, resolveStrandClusterSize, strandClusterPolicy } from './types.js';
|
|
14
14
|
import { strandNodeAddrs } from './strand-network-config.js';
|
|
15
15
|
import { superviseRelayReservation } from './relay-reservation.js';
|
|
16
|
+
import { connectionManagerTimeouts, peerJoinPushBudget, relayReservationBudgetMs, resolveLinkRoundTripMs } from './link-budget.js';
|
|
16
17
|
const log = debug('sereus:cadre:strand-manager');
|
|
17
18
|
const timing = debug('sereus:cadre:timing');
|
|
18
19
|
/**
|
|
@@ -319,6 +320,11 @@ export class StrandInstanceManager {
|
|
|
319
320
|
async buildStrandRuntime(instance, config) {
|
|
320
321
|
const strandId = instance.strandId;
|
|
321
322
|
const { sAppConfig } = config;
|
|
323
|
+
// Refuse a bad `linkRoundTripMs` before any strand bring-up, for the reason
|
|
324
|
+
// `CadreNode.start()` does: every budget derived from it here is behind a condition
|
|
325
|
+
// (backfill disabled, no relay addrs), so a zero or NaN declaration would otherwise
|
|
326
|
+
// surface later inside a path that logs and carries on. `link-budget.ts`.
|
|
327
|
+
resolveLinkRoundTripMs(config.network?.linkRoundTripMs);
|
|
322
328
|
// The store the instance OWNS (resolved once in `startStrand`), not a fresh
|
|
323
329
|
// resolution: a rebuild must reach the same backend through the same warm cache.
|
|
324
330
|
const strandStorage = this.strandStorages.get(strandId);
|
|
@@ -410,7 +416,12 @@ export class StrandInstanceManager {
|
|
|
410
416
|
// returns the frozen STRAND_CLUSTER_POLICY itself, declaring nothing. Resolved
|
|
411
417
|
// HERE rather than at `startStrand`, so a wake from hibernation would pick up a
|
|
412
418
|
// serving set that changed while the strand slept.
|
|
413
|
-
|
|
419
|
+
// The read deadline is the same field the control node reads, because the two
|
|
420
|
+
// networks ride one link; absent, the base policy's COHORT_READ_DEADLINE_MS stands.
|
|
421
|
+
clusterPolicy: strandClusterPolicy(strandClusterSize, {
|
|
422
|
+
servingMachines: config.servingMachines,
|
|
423
|
+
cohortQueryTimeoutMs: config.network?.cohortQueryTimeoutMs
|
|
424
|
+
}),
|
|
414
425
|
arachnode: {
|
|
415
426
|
enableRingZulu: config.profile === 'storage'
|
|
416
427
|
},
|
|
@@ -422,6 +433,10 @@ export class StrandInstanceManager {
|
|
|
422
433
|
// widened ping deadline has to reach the strand nodes too (see
|
|
423
434
|
// DEFAULT_CONNECTION_MONITOR).
|
|
424
435
|
connectionMonitor: config.network?.connectionMonitor ?? DEFAULT_CONNECTION_MONITOR,
|
|
436
|
+
// The same declared-link limits the control node takes, and for the same reason: a
|
|
437
|
+
// strand node is the listener for every other member's strand node (`link-budget.ts`,
|
|
438
|
+
// `connectionManagerTimeouts`).
|
|
439
|
+
connectionManager: connectionManagerTimeouts(config.network?.linkRoundTripMs),
|
|
425
440
|
// Listen entries plus the WebSocket transport switch they imply — a strand node
|
|
426
441
|
// announces nothing the operator configured (`strand-network-config.ts`), and
|
|
427
442
|
// spreads AFTER `transports` above because the switch is a no-op whenever the
|
|
@@ -525,7 +540,8 @@ export class StrandInstanceManager {
|
|
|
525
540
|
const ownPeerId = instance.libp2pNode?.peerId.toString();
|
|
526
541
|
return ownPeerId !== undefined && revocationEnforcer.isRevoked(ownPeerId);
|
|
527
542
|
}
|
|
528
|
-
: undefined
|
|
543
|
+
: undefined,
|
|
544
|
+
onRejoinBlocked: () => config.onRejoinBlocked?.(strandId)
|
|
529
545
|
}, {
|
|
530
546
|
pollIntervalMs: config.membershipReconciliation?.pollIntervalMs
|
|
531
547
|
?? config.revocationEnforcement?.pollIntervalMs
|
|
@@ -564,7 +580,15 @@ export class StrandInstanceManager {
|
|
|
564
580
|
// The same prefix the receiver registered its block-transfer handler
|
|
565
581
|
// under — derived from networkName above, never re-spelled here.
|
|
566
582
|
protocolPrefix
|
|
567
|
-
},
|
|
583
|
+
}, {
|
|
584
|
+
// Dial and response deadlines counted in link round trips, not fixed milliseconds:
|
|
585
|
+
// a relayed dial costs a fixed number of exchanges, so a fixed budget has a link
|
|
586
|
+
// speed above which this catch-up can never reach the peer at all. See
|
|
587
|
+
// `link-budget.ts`. Spread BEFORE the host's own config so an explicit
|
|
588
|
+
// `strandBackfill.dialTimeoutMs` still wins.
|
|
589
|
+
...peerJoinPushBudget(config.network?.linkRoundTripMs),
|
|
590
|
+
...config.backfill
|
|
591
|
+
});
|
|
568
592
|
backfill.start();
|
|
569
593
|
this.backfills.set(strandId, backfill);
|
|
570
594
|
}
|
|
@@ -579,10 +603,12 @@ export class StrandInstanceManager {
|
|
|
579
603
|
// The strand's database is up, and the supervisor keeps trying on its backoff;
|
|
580
604
|
// failing here would only trade that for `StrandWatcher`'s full-rebuild retry.
|
|
581
605
|
//
|
|
582
|
-
// NOTE: a relay that is down costs this launch one full drive
|
|
583
|
-
//
|
|
584
|
-
//
|
|
585
|
-
//
|
|
606
|
+
// NOTE: a relay that is down costs this launch one full drive — the reservation budget
|
|
607
|
+
// counted from the declared link round trip, 14 s at its default (`link-budget.ts`) — and
|
|
608
|
+
// `StrandWatcher` launches strands one at a time, so N strands cost N of those in
|
|
609
|
+
// bring-up during a relay outage, and MORE on a host that declared a slower link. If that
|
|
610
|
+
// ever matters, stop awaiting here (the circuit addr then lands after `active`) rather
|
|
611
|
+
// than shortening the drive.
|
|
586
612
|
t0 = performance.now();
|
|
587
613
|
await this.awaitFirstRelayAttempts(strandId);
|
|
588
614
|
timing('[buildStrandRuntime:%s] relay first attempts: %dms', strandId, Math.round(performance.now() - t0));
|
|
@@ -619,10 +645,9 @@ export class StrandInstanceManager {
|
|
|
619
645
|
// Relay supervisors FIRST of all — before anything below is torn down — so no
|
|
620
646
|
// re-drive is scheduled against a node being stopped. `stop()` is synchronous
|
|
621
647
|
// and never awaits a drive, so a relay that is down cannot delay this teardown.
|
|
622
|
-
//
|
|
623
|
-
//
|
|
624
|
-
//
|
|
625
|
-
// discarded.
|
|
648
|
+
// `stop()` also cancels the drive already in flight, so an attempt started
|
|
649
|
+
// moments before this does not keep dialing and polling the node the lines
|
|
650
|
+
// below are stopping; its result is discarded either way.
|
|
626
651
|
const relaySupervisors = this.relaySupervisors.get(instance.strandId);
|
|
627
652
|
if (relaySupervisors) {
|
|
628
653
|
relaySupervisors.forEach((supervisor) => supervisor.stop());
|
|
@@ -672,8 +697,11 @@ export class StrandInstanceManager {
|
|
|
672
697
|
/**
|
|
673
698
|
* One {@link superviseRelayReservation} per relay dial addr, each over exactly
|
|
674
699
|
* that relay so "held" is judged per relay (`circuitMultiaddrsVia`) and losing
|
|
675
|
-
* one relay re-drives only that one. Default timings — the control node's (2 s
|
|
676
|
-
* doubling to 60 s between failed attempts, a 5 s liveness check while held).
|
|
700
|
+
* one relay re-drives only that one. Default retry timings — the control node's (2 s
|
|
701
|
+
* doubling to 60 s between failed attempts, a 5 s liveness check while held). Each DRIVE's
|
|
702
|
+
* own deadline is counted in link round trips at this host's declared
|
|
703
|
+
* `network.linkRoundTripMs` (`link-budget.ts`), so the same declaration that lengthens the
|
|
704
|
+
* control node's drive lengthens these.
|
|
677
705
|
*
|
|
678
706
|
* Each supervisor's `beforeRedrive` is the caller's
|
|
679
707
|
* {@link StartStrandConfig.announceDelegateToRelay} for THIS relay and THIS
|
|
@@ -687,7 +715,9 @@ export class StrandInstanceManager {
|
|
|
687
715
|
}
|
|
688
716
|
const announce = config.announceDelegateToRelay;
|
|
689
717
|
const delegatePeerId = node.peerId.toString();
|
|
718
|
+
const timeoutMs = relayReservationBudgetMs(config.network?.linkRoundTripMs);
|
|
690
719
|
return relayDialAddrs.map((relayAddr) => superviseRelayReservation(node, [relayAddr], {
|
|
720
|
+
timeoutMs,
|
|
691
721
|
...(announce && { beforeRedrive: () => announce(strandId, relayAddr, delegatePeerId) })
|
|
692
722
|
}));
|
|
693
723
|
}
|
|
@@ -757,6 +787,23 @@ export class StrandInstanceManager {
|
|
|
757
787
|
}
|
|
758
788
|
await enforcer.refresh();
|
|
759
789
|
}
|
|
790
|
+
/**
|
|
791
|
+
* A fresh membership invitation was staged for `strandId` (`CadreNode`'s
|
|
792
|
+
* `adoptFormationMembershipInvite`): re-arm the strand's membership reconciler so the
|
|
793
|
+
* invitation is attempted now rather than never — the loop finished during the first
|
|
794
|
+
* join and would otherwise stay stopped until a relaunch. Quiet no-op when no reconciler
|
|
795
|
+
* is armed: a first formation stages before the strand is added, and bring-up then arms
|
|
796
|
+
* a loop that finds the invitation by itself. Returns at once; the pass runs on the
|
|
797
|
+
* loop's own serialized chain and never rejects.
|
|
798
|
+
*/
|
|
799
|
+
notifyMembershipInviteStaged(strandId) {
|
|
800
|
+
const reconciler = this.membershipReconcilers.get(strandId);
|
|
801
|
+
if (!reconciler) {
|
|
802
|
+
log('notifyMembershipInviteStaged: strand %s has no armed membership reconciler — bring-up will find the invitation', strandId);
|
|
803
|
+
return;
|
|
804
|
+
}
|
|
805
|
+
void reconciler.rearm();
|
|
806
|
+
}
|
|
760
807
|
/**
|
|
761
808
|
* Best-effort removal of THIS machine's own `Strand.MemberPeer` binding — the
|
|
762
809
|
* self arm of `removeMemberPeer`, signed with the retained launch config's party
|