@effectstream/runtime 0.100.22 → 0.100.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -0
- package/conciseinputhash-determinism.md +114 -0
- package/package.json +12 -12
- package/src/api/apply-status.ts +30 -0
- package/src/api/http-server.ts +68 -7
- package/src/api/stream-status.ts +35 -0
- package/src/coalesce.ts +291 -0
- package/src/finalized-stream.ts +73 -0
- package/src/main.ts +111 -89
- package/src/process-blocks.ts +126 -27
- package/src/types.ts +7 -0
- package/test/coalesce.unit.test.ts +69 -0
- package/test/reproduction/buffering.test.ts +650 -0
- package/test/reproduction/coalesce-memory.test.ts +181 -0
- package/test/reproduction/coalesce.test.ts +133 -0
- package/test/reproduction/consistency-snapshot.ts +168 -0
- package/test/reproduction/consistency.test.ts +197 -0
- package/test/reproduction/harness.ts +574 -270
- package/test/reproduction/node-runner.ts +154 -0
- package/test/reproduction/poll.ts +160 -0
- package/test/reproduction/resume.test.ts +77 -0
- package/test/reproduction/sanity.test.ts +40 -91
- package/test/reproduction/scenario.ts +126 -0
|
@@ -0,0 +1,650 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Verifies the fetch-backpressure feature (Option B′) over the synthetic `test`
|
|
3
|
+
* chain through the real runtime (`start()` + in-process PGLite).
|
|
4
|
+
*
|
|
5
|
+
* 1a — unbounded buffering: with a cap, the fetch loop pauses when the buffer is
|
|
6
|
+
* full, so it stays bounded (~cap + stepSize) instead of racing to the whole
|
|
7
|
+
* backlog — while the node keeps applying blocks (liveness).
|
|
8
|
+
* 1b — head-of-line blocking: a stalled slow chain STILL halts block production
|
|
9
|
+
* (the merge gate is correct), but the fast chain's buffer is now bounded by
|
|
10
|
+
* the cap instead of ballooning behind the head-of-line block.
|
|
11
|
+
* 1c — skip-ahead: the merge-demand exemption lifts the cap while the merge is gated
|
|
12
|
+
* on a chain whose page is far behind the (skipped-ahead) root, so it advances
|
|
13
|
+
* instead of deadlocking.
|
|
14
|
+
* 1d — density: the same exemption when a parallel chain is finer-grained than the cap.
|
|
15
|
+
*
|
|
16
|
+
* Before the fix these buffers reached ~49k–50k (see
|
|
17
|
+
* e2e/perf/results/BACKPRESSURE-BASELINE.md); these tests assert they now
|
|
18
|
+
* stay at the cap. Each test also writes a JSON time-series artifact to
|
|
19
|
+
* e2e/perf/results/ whose field names match e2e/perf/metrics.ts `Sample`, so the
|
|
20
|
+
* perf HTML report's Backpressure section can render the same signals.
|
|
21
|
+
*/
|
|
22
|
+
import { afterEach, beforeEach, expect, test } from "bun:test";
|
|
23
|
+
import {
|
|
24
|
+
ConfigBuilder,
|
|
25
|
+
ConfigNetworkType,
|
|
26
|
+
ConfigSyncProtocolType,
|
|
27
|
+
} from "@effectstream/config";
|
|
28
|
+
import { TestChainControl } from "@effectstream/sync";
|
|
29
|
+
import fs from "node:fs";
|
|
30
|
+
import path from "node:path";
|
|
31
|
+
import {
|
|
32
|
+
bufferSize,
|
|
33
|
+
type Harness,
|
|
34
|
+
latestFinalizedHeight,
|
|
35
|
+
type RunningNode,
|
|
36
|
+
setupHarness,
|
|
37
|
+
sleep,
|
|
38
|
+
TEST_PRIMITIVE_TYPE,
|
|
39
|
+
} from "./harness.ts";
|
|
40
|
+
|
|
41
|
+
const START_TIME = 1_700_000_000_000;
|
|
42
|
+
const BLOCK_TIME_MS = 1000;
|
|
43
|
+
const STEP_SIZE = 1000;
|
|
44
|
+
// A deep backlog the arithmetic fetcher would race through if uncapped (baseline
|
|
45
|
+
// hit ~49k–50k). The slow chain (1b) is pinned low so the merge stalls at its tip.
|
|
46
|
+
const BIG_TIP = 50_000;
|
|
47
|
+
const SLOW_TIP = 50;
|
|
48
|
+
// Explicit small cap so the bound is deterministic and far below the backlog.
|
|
49
|
+
// The fetch pauses at >= CAP; one in-flight chunk can overshoot by ~stepSize, so
|
|
50
|
+
// the buffer settles in [~CAP, CAP + stepSize]. We assert it lands in that band.
|
|
51
|
+
const CAP = 3000;
|
|
52
|
+
const CAP_FLOOR = CAP - STEP_SIZE; // proves backpressure actually engaged
|
|
53
|
+
const BUFFER_BOUND = CAP + STEP_SIZE + 200; // upper bound incl. one overshoot chunk
|
|
54
|
+
|
|
55
|
+
// e2e/perf/results lives at the repo root: reproduction → test → runtime →
|
|
56
|
+
// node-sdk → packages → <root>/e2e/perf/results.
|
|
57
|
+
const RESULTS_DIR = path.resolve(
|
|
58
|
+
import.meta.dirname!,
|
|
59
|
+
"../../../../../e2e/perf/results",
|
|
60
|
+
);
|
|
61
|
+
|
|
62
|
+
/** One time-series row; field names mirror e2e/perf/metrics.ts `Sample`. */
|
|
63
|
+
type BufSample = {
|
|
64
|
+
t: number;
|
|
65
|
+
wall: number;
|
|
66
|
+
mainBuf: number;
|
|
67
|
+
evmBuf: number; // the parallel chain's buffer (report labels it "parallel buf")
|
|
68
|
+
mainOwnBlock: number | null;
|
|
69
|
+
evmOwnBlock: number | null;
|
|
70
|
+
rss: number;
|
|
71
|
+
appliedBlock: number | null;
|
|
72
|
+
};
|
|
73
|
+
|
|
74
|
+
function ownBlock(node: RunningNode, name: string): number | null {
|
|
75
|
+
return node.syncProtocols().find((p) => p.name === name)?.lastPage
|
|
76
|
+
?.ownBlockNumber ?? null;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** Backpressure "fired" counter (rising edges) exposed on SyncState / /debug/metrics. */
|
|
80
|
+
function pauses(node: RunningNode, name: string): number {
|
|
81
|
+
return node.syncProtocols().find((p) => p.name === name)
|
|
82
|
+
?.backpressurePauses ?? 0;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** Side-effect-only artifact write; never throws into the test. */
|
|
86
|
+
function writeArtifact(
|
|
87
|
+
name: string,
|
|
88
|
+
samples: BufSample[],
|
|
89
|
+
meta: Record<string, unknown>,
|
|
90
|
+
): void {
|
|
91
|
+
try {
|
|
92
|
+
fs.mkdirSync(RESULTS_DIR, { recursive: true });
|
|
93
|
+
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
|
|
94
|
+
const file = path.join(RESULTS_DIR, `${name}-${stamp}.json`);
|
|
95
|
+
fs.writeFileSync(file, JSON.stringify({ meta, samples }, null, 2));
|
|
96
|
+
console.log(`buffering artifact: ${file}`);
|
|
97
|
+
} catch (e) {
|
|
98
|
+
console.warn(`failed to write buffering artifact: ${String(e)}`);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
let h: Harness;
|
|
103
|
+
beforeEach(async () => {
|
|
104
|
+
h = await setupHarness();
|
|
105
|
+
});
|
|
106
|
+
afterEach(async () => {
|
|
107
|
+
await h?.teardown();
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
// ── 1a — unbounded buffering, now bounded by the cap ────────────────────────────
|
|
111
|
+
|
|
112
|
+
const MAIN_1A = "mainClock1a";
|
|
113
|
+
const PAR_1A = "parallelP1a";
|
|
114
|
+
|
|
115
|
+
function buildConfig1a() {
|
|
116
|
+
return new ConfigBuilder()
|
|
117
|
+
.setNamespace((b) => b.setSecurityNamespace("sync-buffering-1a"))
|
|
118
|
+
.buildNetworks((b) =>
|
|
119
|
+
b
|
|
120
|
+
.addNetwork({
|
|
121
|
+
name: "clock",
|
|
122
|
+
type: ConfigNetworkType.TEST,
|
|
123
|
+
startTime: START_TIME,
|
|
124
|
+
blockTimeMS: BLOCK_TIME_MS,
|
|
125
|
+
})
|
|
126
|
+
.addNetwork({
|
|
127
|
+
name: "chainP",
|
|
128
|
+
type: ConfigNetworkType.TEST,
|
|
129
|
+
startTime: START_TIME,
|
|
130
|
+
blockTimeMS: BLOCK_TIME_MS,
|
|
131
|
+
})
|
|
132
|
+
)
|
|
133
|
+
.buildDeployments((b) => b)
|
|
134
|
+
.buildSyncProtocols((b) =>
|
|
135
|
+
b
|
|
136
|
+
.addMain(
|
|
137
|
+
(n) => n.clock,
|
|
138
|
+
() => ({
|
|
139
|
+
name: MAIN_1A,
|
|
140
|
+
type: ConfigSyncProtocolType.TEST_MAIN,
|
|
141
|
+
startBlockHeight: 1,
|
|
142
|
+
pollingInterval: 30,
|
|
143
|
+
stepSize: STEP_SIZE,
|
|
144
|
+
maxBufferedPages: CAP,
|
|
145
|
+
}),
|
|
146
|
+
)
|
|
147
|
+
.addParallel(
|
|
148
|
+
(n) => (n as any).chainP,
|
|
149
|
+
() => ({
|
|
150
|
+
name: PAR_1A,
|
|
151
|
+
type: ConfigSyncProtocolType.TEST_PARALLEL,
|
|
152
|
+
startBlockHeight: 1,
|
|
153
|
+
pollingInterval: 30,
|
|
154
|
+
stepSize: STEP_SIZE,
|
|
155
|
+
delayMs: 0,
|
|
156
|
+
events: [],
|
|
157
|
+
maxBufferedPages: CAP,
|
|
158
|
+
}),
|
|
159
|
+
)
|
|
160
|
+
)
|
|
161
|
+
.buildPrimitives((b) =>
|
|
162
|
+
b.addPrimitive(
|
|
163
|
+
(sp) => (sp as any)[PAR_1A],
|
|
164
|
+
() => ({ name: "noEvt1a", type: TEST_PRIMITIVE_TYPE, startBlockHeight: 1 }),
|
|
165
|
+
)
|
|
166
|
+
)
|
|
167
|
+
.build();
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
test("1a: backpressure caps the buffer while the node keeps applying blocks", async () => {
|
|
171
|
+
// Tips far ahead of where the one-block-per-txn merge can drain to: without a
|
|
172
|
+
// cap the fetcher would race to ~50k; with the cap it must plateau near CAP.
|
|
173
|
+
TestChainControl.setTip(MAIN_1A, BIG_TIP);
|
|
174
|
+
TestChainControl.setTip(PAR_1A, BIG_TIP);
|
|
175
|
+
|
|
176
|
+
const node = await h.runNode({ config: buildConfig1a(), apiPort: 19141 });
|
|
177
|
+
const samples: BufSample[] = [];
|
|
178
|
+
let peakMain = 0;
|
|
179
|
+
let peakEvm = 0;
|
|
180
|
+
let lastApplied = 0;
|
|
181
|
+
let mainPauses = 0;
|
|
182
|
+
try {
|
|
183
|
+
const startMs = Date.now();
|
|
184
|
+
const deadline = startMs + 30_000;
|
|
185
|
+
while (Date.now() < deadline) {
|
|
186
|
+
const wall = Date.now();
|
|
187
|
+
const mainBuf = bufferSize(node, MAIN_1A);
|
|
188
|
+
const evmBuf = bufferSize(node, PAR_1A);
|
|
189
|
+
const appliedBlock = await latestFinalizedHeight(h.pool);
|
|
190
|
+
samples.push({
|
|
191
|
+
t: wall - startMs,
|
|
192
|
+
wall,
|
|
193
|
+
mainBuf,
|
|
194
|
+
evmBuf,
|
|
195
|
+
mainOwnBlock: ownBlock(node, MAIN_1A),
|
|
196
|
+
evmOwnBlock: ownBlock(node, PAR_1A),
|
|
197
|
+
rss: process.memoryUsage().rss,
|
|
198
|
+
appliedBlock,
|
|
199
|
+
});
|
|
200
|
+
peakMain = Math.max(peakMain, mainBuf);
|
|
201
|
+
peakEvm = Math.max(peakEvm, evmBuf);
|
|
202
|
+
lastApplied = appliedBlock ?? lastApplied;
|
|
203
|
+
// Backpressure engaged (buffer hit the cap) AND the node is still applying
|
|
204
|
+
// blocks → we've captured the bounded steady state; stop early.
|
|
205
|
+
if (peakMain >= CAP && lastApplied >= 20) break;
|
|
206
|
+
await sleep(50);
|
|
207
|
+
}
|
|
208
|
+
} finally {
|
|
209
|
+
mainPauses = pauses(node, MAIN_1A);
|
|
210
|
+
writeArtifact("buffering-1a", samples, {
|
|
211
|
+
mode: "1a-bounded-by-cap",
|
|
212
|
+
stepSize: STEP_SIZE,
|
|
213
|
+
bigTip: BIG_TIP,
|
|
214
|
+
cap: CAP,
|
|
215
|
+
bufferBound: BUFFER_BOUND,
|
|
216
|
+
peakMainBuf: peakMain,
|
|
217
|
+
peakEvmBuf: peakEvm,
|
|
218
|
+
lastApplied,
|
|
219
|
+
pauses: mainPauses,
|
|
220
|
+
});
|
|
221
|
+
await node.stop();
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
// Backpressure engaged: the buffer reached the cap (not starved)...
|
|
225
|
+
expect(peakMain).toBeGreaterThanOrEqual(CAP_FLOOR);
|
|
226
|
+
// ...but stayed bounded — nowhere near the uncapped ~49k backlog.
|
|
227
|
+
expect(Math.max(peakMain, peakEvm)).toBeLessThanOrEqual(BUFFER_BOUND);
|
|
228
|
+
// ...the node kept producing/applying blocks (no deadlock)...
|
|
229
|
+
expect(lastApplied).toBeGreaterThanOrEqual(20);
|
|
230
|
+
// ...and the cap actually fired (the observability metric populated).
|
|
231
|
+
expect(mainPauses).toBeGreaterThan(0);
|
|
232
|
+
}, 60_000);
|
|
233
|
+
|
|
234
|
+
// ── 1b — head-of-line blocking, sibling buffer now bounded ──────────────────────
|
|
235
|
+
|
|
236
|
+
const MAIN_1B = "mainClock1b";
|
|
237
|
+
const FAST_1B = "fastP1b";
|
|
238
|
+
const SLOW_1B = "slowP1b";
|
|
239
|
+
|
|
240
|
+
function buildConfig1b() {
|
|
241
|
+
return new ConfigBuilder()
|
|
242
|
+
.setNamespace((b) => b.setSecurityNamespace("sync-buffering-1b"))
|
|
243
|
+
.buildNetworks((b) =>
|
|
244
|
+
b
|
|
245
|
+
.addNetwork({
|
|
246
|
+
name: "clock",
|
|
247
|
+
type: ConfigNetworkType.TEST,
|
|
248
|
+
startTime: START_TIME,
|
|
249
|
+
blockTimeMS: BLOCK_TIME_MS,
|
|
250
|
+
})
|
|
251
|
+
.addNetwork({
|
|
252
|
+
name: "chainFast",
|
|
253
|
+
type: ConfigNetworkType.TEST,
|
|
254
|
+
startTime: START_TIME,
|
|
255
|
+
blockTimeMS: BLOCK_TIME_MS,
|
|
256
|
+
})
|
|
257
|
+
.addNetwork({
|
|
258
|
+
name: "chainSlow",
|
|
259
|
+
type: ConfigNetworkType.TEST,
|
|
260
|
+
startTime: START_TIME,
|
|
261
|
+
blockTimeMS: BLOCK_TIME_MS,
|
|
262
|
+
})
|
|
263
|
+
)
|
|
264
|
+
.buildDeployments((b) => b)
|
|
265
|
+
.buildSyncProtocols((b) =>
|
|
266
|
+
b
|
|
267
|
+
.addMain(
|
|
268
|
+
(n) => n.clock,
|
|
269
|
+
() => ({
|
|
270
|
+
name: MAIN_1B,
|
|
271
|
+
type: ConfigSyncProtocolType.TEST_MAIN,
|
|
272
|
+
startBlockHeight: 1,
|
|
273
|
+
pollingInterval: 30,
|
|
274
|
+
stepSize: STEP_SIZE,
|
|
275
|
+
maxBufferedPages: CAP,
|
|
276
|
+
}),
|
|
277
|
+
)
|
|
278
|
+
.addParallel(
|
|
279
|
+
(n) => (n as any).chainFast,
|
|
280
|
+
() => ({
|
|
281
|
+
name: FAST_1B,
|
|
282
|
+
type: ConfigSyncProtocolType.TEST_PARALLEL,
|
|
283
|
+
startBlockHeight: 1,
|
|
284
|
+
pollingInterval: 30,
|
|
285
|
+
stepSize: STEP_SIZE,
|
|
286
|
+
delayMs: 0,
|
|
287
|
+
events: [],
|
|
288
|
+
maxBufferedPages: CAP,
|
|
289
|
+
}),
|
|
290
|
+
)
|
|
291
|
+
.addParallel(
|
|
292
|
+
(n) => (n as any).chainSlow,
|
|
293
|
+
() => ({
|
|
294
|
+
name: SLOW_1B,
|
|
295
|
+
type: ConfigSyncProtocolType.TEST_PARALLEL,
|
|
296
|
+
startBlockHeight: 1,
|
|
297
|
+
pollingInterval: 30,
|
|
298
|
+
stepSize: STEP_SIZE,
|
|
299
|
+
delayMs: 0,
|
|
300
|
+
events: [],
|
|
301
|
+
maxBufferedPages: CAP,
|
|
302
|
+
}),
|
|
303
|
+
)
|
|
304
|
+
)
|
|
305
|
+
.buildPrimitives((b) =>
|
|
306
|
+
b
|
|
307
|
+
.addPrimitive(
|
|
308
|
+
(sp) => (sp as any)[FAST_1B],
|
|
309
|
+
() => ({ name: "noEvtFast", type: TEST_PRIMITIVE_TYPE, startBlockHeight: 1 }),
|
|
310
|
+
)
|
|
311
|
+
.addPrimitive(
|
|
312
|
+
(sp) => (sp as any)[SLOW_1B],
|
|
313
|
+
() => ({ name: "noEvtSlow", type: TEST_PRIMITIVE_TYPE, startBlockHeight: 1 }),
|
|
314
|
+
)
|
|
315
|
+
)
|
|
316
|
+
.build();
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
test("1b: a stalled chain still halts production, but the fast chain's buffer stays capped", async () => {
|
|
320
|
+
// slowP pinned at a low tip → the merge stalls there forever (correct). fastP and
|
|
321
|
+
// the main clock keep fetching toward their high tips, but the cap now bounds them.
|
|
322
|
+
TestChainControl.setTip(MAIN_1B, BIG_TIP);
|
|
323
|
+
TestChainControl.setTip(FAST_1B, BIG_TIP);
|
|
324
|
+
TestChainControl.setTip(SLOW_1B, SLOW_TIP);
|
|
325
|
+
|
|
326
|
+
const node = await h.runNode({ config: buildConfig1b(), apiPort: 19142 });
|
|
327
|
+
const samples: BufSample[] = [];
|
|
328
|
+
let peakMain = 0;
|
|
329
|
+
let peakFast = 0;
|
|
330
|
+
let stalledHeight = -1;
|
|
331
|
+
let fastPauses = 0;
|
|
332
|
+
try {
|
|
333
|
+
const startMs = Date.now();
|
|
334
|
+
// Generous for slow CI; exits early once the stall + capped sibling are captured.
|
|
335
|
+
const deadline = startMs + 30_000;
|
|
336
|
+
let lastHeight = -1;
|
|
337
|
+
let lastChange = Date.now();
|
|
338
|
+
while (Date.now() < deadline) {
|
|
339
|
+
const wall = Date.now();
|
|
340
|
+
const mainBuf = bufferSize(node, MAIN_1B);
|
|
341
|
+
const fastBuf = bufferSize(node, FAST_1B);
|
|
342
|
+
const appliedBlock = await latestFinalizedHeight(h.pool);
|
|
343
|
+
samples.push({
|
|
344
|
+
t: wall - startMs,
|
|
345
|
+
wall,
|
|
346
|
+
mainBuf,
|
|
347
|
+
evmBuf: fastBuf,
|
|
348
|
+
mainOwnBlock: ownBlock(node, MAIN_1B),
|
|
349
|
+
evmOwnBlock: ownBlock(node, FAST_1B),
|
|
350
|
+
rss: process.memoryUsage().rss,
|
|
351
|
+
appliedBlock,
|
|
352
|
+
});
|
|
353
|
+
peakMain = Math.max(peakMain, mainBuf);
|
|
354
|
+
peakFast = Math.max(peakFast, fastBuf);
|
|
355
|
+
if (appliedBlock !== lastHeight) {
|
|
356
|
+
lastHeight = appliedBlock;
|
|
357
|
+
lastChange = wall;
|
|
358
|
+
}
|
|
359
|
+
// Production has stalled (height flat ≥1.5s) AND the fast chain has hit its
|
|
360
|
+
// cap → the head-of-line block holds while the sibling buffer is bounded.
|
|
361
|
+
if (wall - lastChange >= 1500 && peakFast >= CAP_FLOOR) {
|
|
362
|
+
stalledHeight = appliedBlock ?? -1;
|
|
363
|
+
break;
|
|
364
|
+
}
|
|
365
|
+
await sleep(50);
|
|
366
|
+
}
|
|
367
|
+
} finally {
|
|
368
|
+
fastPauses = pauses(node, FAST_1B);
|
|
369
|
+
writeArtifact("buffering-1b", samples, {
|
|
370
|
+
mode: "1b-hol-sibling-bounded",
|
|
371
|
+
stepSize: STEP_SIZE,
|
|
372
|
+
bigTip: BIG_TIP,
|
|
373
|
+
slowTip: SLOW_TIP,
|
|
374
|
+
cap: CAP,
|
|
375
|
+
bufferBound: BUFFER_BOUND,
|
|
376
|
+
stalledHeight,
|
|
377
|
+
peakMainBuf: peakMain,
|
|
378
|
+
peakFastBuf: peakFast,
|
|
379
|
+
fastPauses,
|
|
380
|
+
});
|
|
381
|
+
await node.stop();
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
// Block production still stalls at ~the slow chain's tip (the merge gate is correct)...
|
|
385
|
+
expect(stalledHeight).toBeGreaterThanOrEqual(0);
|
|
386
|
+
expect(stalledHeight).toBeLessThanOrEqual(SLOW_TIP + 10);
|
|
387
|
+
// ...the fast chain's buffer engaged the cap...
|
|
388
|
+
expect(peakFast).toBeGreaterThanOrEqual(CAP_FLOOR);
|
|
389
|
+
// ...but stayed bounded behind the head-of-line block (was ~50k uncapped)...
|
|
390
|
+
expect(peakFast).toBeLessThanOrEqual(BUFFER_BOUND);
|
|
391
|
+
expect(peakMain).toBeLessThanOrEqual(BUFFER_BOUND);
|
|
392
|
+
// ...and the cap actually fired on the fast chain (observability metric populated).
|
|
393
|
+
expect(fastPauses).toBeGreaterThan(0);
|
|
394
|
+
}, 60_000);
|
|
395
|
+
|
|
396
|
+
// ── 1c — skip-ahead: merge-demand exemption keeps the cap from deadlocking ────────
|
|
397
|
+
//
|
|
398
|
+
// The root (main clock) starts far ahead of a parallel chain that starts at block 1.
|
|
399
|
+
// To produce the first root block at τ (main page SKIP_START) the merge needs the
|
|
400
|
+
// parallel page PAST τ — so the fetcher must scan ~SKIP_START blocks, more than the
|
|
401
|
+
// cap. The exemption lifts the cap while the merge is gated on this chain, so it
|
|
402
|
+
// reaches τ and production proceeds (without it the node wedges on the first block).
|
|
403
|
+
|
|
404
|
+
const MAIN_1C = "mainClock1c";
|
|
405
|
+
const PAR_1C = "parallelP1c";
|
|
406
|
+
// Main genesis page (skip-ahead); cap far below it so the parallel chain must buffer
|
|
407
|
+
// past the cap to reach τ.
|
|
408
|
+
const SKIP_START = 300;
|
|
409
|
+
const SMALL_STEP = 50;
|
|
410
|
+
const SMALL_CAP = 100;
|
|
411
|
+
// Liveness: past the first non-empty block at SKIP_START.
|
|
412
|
+
const LIVE_TARGET_1C = SKIP_START + 10;
|
|
413
|
+
// Bounded by necessity: at most the skip window + one in-flight chunk.
|
|
414
|
+
const SKIP_BUFFER_BOUND = SKIP_START + SMALL_STEP + 200;
|
|
415
|
+
|
|
416
|
+
function buildConfig1c() {
|
|
417
|
+
return new ConfigBuilder()
|
|
418
|
+
.setNamespace((b) => b.setSecurityNamespace("sync-buffering-1c"))
|
|
419
|
+
.buildNetworks((b) =>
|
|
420
|
+
b
|
|
421
|
+
.addNetwork({
|
|
422
|
+
name: "clock",
|
|
423
|
+
type: ConfigNetworkType.TEST,
|
|
424
|
+
startTime: START_TIME,
|
|
425
|
+
blockTimeMS: BLOCK_TIME_MS,
|
|
426
|
+
})
|
|
427
|
+
.addNetwork({
|
|
428
|
+
name: "chainP",
|
|
429
|
+
type: ConfigNetworkType.TEST,
|
|
430
|
+
startTime: START_TIME,
|
|
431
|
+
blockTimeMS: BLOCK_TIME_MS,
|
|
432
|
+
})
|
|
433
|
+
)
|
|
434
|
+
.buildDeployments((b) => b)
|
|
435
|
+
.buildSyncProtocols((b) =>
|
|
436
|
+
b
|
|
437
|
+
.addMain(
|
|
438
|
+
(n) => n.clock,
|
|
439
|
+
() => ({
|
|
440
|
+
name: MAIN_1C,
|
|
441
|
+
type: ConfigSyncProtocolType.TEST_MAIN,
|
|
442
|
+
// Skipped far ahead of the parallel chain (the recovery skip-ahead).
|
|
443
|
+
startBlockHeight: SKIP_START,
|
|
444
|
+
pollingInterval: 30,
|
|
445
|
+
stepSize: SMALL_STEP,
|
|
446
|
+
maxBufferedPages: SMALL_CAP,
|
|
447
|
+
}),
|
|
448
|
+
)
|
|
449
|
+
.addParallel(
|
|
450
|
+
(n) => (n as any).chainP,
|
|
451
|
+
() => ({
|
|
452
|
+
name: PAR_1C,
|
|
453
|
+
type: ConfigSyncProtocolType.TEST_PARALLEL,
|
|
454
|
+
startBlockHeight: 1,
|
|
455
|
+
pollingInterval: 30,
|
|
456
|
+
stepSize: SMALL_STEP,
|
|
457
|
+
delayMs: 0,
|
|
458
|
+
events: [],
|
|
459
|
+
maxBufferedPages: SMALL_CAP,
|
|
460
|
+
}),
|
|
461
|
+
)
|
|
462
|
+
)
|
|
463
|
+
.buildPrimitives((b) =>
|
|
464
|
+
b.addPrimitive(
|
|
465
|
+
(sp) => (sp as any)[PAR_1C],
|
|
466
|
+
() => ({ name: "noEvt1c", type: TEST_PRIMITIVE_TYPE, startBlockHeight: 1 }),
|
|
467
|
+
)
|
|
468
|
+
)
|
|
469
|
+
.build();
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
test("1c: a skipped-ahead root does NOT deadlock the merge on the first non-empty block", async () => {
|
|
473
|
+
// Main skipped to SKIP_START; parallel starts at 1 with a high tip.
|
|
474
|
+
TestChainControl.setTip(MAIN_1C, SKIP_START + 100);
|
|
475
|
+
TestChainControl.setTip(PAR_1C, 10_000);
|
|
476
|
+
|
|
477
|
+
const node = await h.runNode({ config: buildConfig1c(), apiPort: 19143 });
|
|
478
|
+
const samples: BufSample[] = [];
|
|
479
|
+
let peakPar = 0;
|
|
480
|
+
let lastApplied = 0;
|
|
481
|
+
try {
|
|
482
|
+
const startMs = Date.now();
|
|
483
|
+
// Generous for slow CI; exits early once the liveness target is reached.
|
|
484
|
+
const deadline = startMs + 30_000;
|
|
485
|
+
while (Date.now() < deadline) {
|
|
486
|
+
const wall = Date.now();
|
|
487
|
+
const mainBuf = bufferSize(node, MAIN_1C);
|
|
488
|
+
const parBuf = bufferSize(node, PAR_1C);
|
|
489
|
+
const appliedBlock = await latestFinalizedHeight(h.pool);
|
|
490
|
+
samples.push({
|
|
491
|
+
t: wall - startMs,
|
|
492
|
+
wall,
|
|
493
|
+
mainBuf,
|
|
494
|
+
evmBuf: parBuf,
|
|
495
|
+
mainOwnBlock: ownBlock(node, MAIN_1C),
|
|
496
|
+
evmOwnBlock: ownBlock(node, PAR_1C),
|
|
497
|
+
rss: process.memoryUsage().rss,
|
|
498
|
+
appliedBlock,
|
|
499
|
+
});
|
|
500
|
+
peakPar = Math.max(peakPar, parBuf);
|
|
501
|
+
lastApplied = appliedBlock ?? lastApplied;
|
|
502
|
+
// Got past the first non-empty block → the deadlock is broken; stop early.
|
|
503
|
+
if (lastApplied >= LIVE_TARGET_1C) break;
|
|
504
|
+
await sleep(50);
|
|
505
|
+
}
|
|
506
|
+
} finally {
|
|
507
|
+
writeArtifact("buffering-1c", samples, {
|
|
508
|
+
mode: "1c-skip-ahead-deadlock",
|
|
509
|
+
stepSize: SMALL_STEP,
|
|
510
|
+
skipStart: SKIP_START,
|
|
511
|
+
cap: SMALL_CAP,
|
|
512
|
+
liveTarget: LIVE_TARGET_1C,
|
|
513
|
+
peakParBuf: peakPar,
|
|
514
|
+
lastApplied,
|
|
515
|
+
});
|
|
516
|
+
await node.stop();
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
// Liveness: production advanced past the first non-empty block (no wedge).
|
|
520
|
+
expect(lastApplied).toBeGreaterThanOrEqual(LIVE_TARGET_1C);
|
|
521
|
+
// Bounded by necessity: only the skip window + one chunk, not the whole tip.
|
|
522
|
+
expect(peakPar).toBeLessThanOrEqual(SKIP_BUFFER_BOUND);
|
|
523
|
+
}, 60_000);
|
|
524
|
+
|
|
525
|
+
// ── 1d — density: same exemption when a chain is finer-grained than the cap ───────
|
|
526
|
+
//
|
|
527
|
+
// Same circular wait without a skip-ahead: a parallel chain far finer than the root
|
|
528
|
+
// (100 parallel blocks per root slot). Each root block needs the parallel page past
|
|
529
|
+
// the slot boundary (~PER_SLOT buffered items), but the cap is below that. The
|
|
530
|
+
// exemption lets the per-slot window buffer and drain each slot (without it the node
|
|
531
|
+
// wedges on the first block).
|
|
532
|
+
|
|
533
|
+
const MAIN_1D = "mainClock1d";
|
|
534
|
+
const PAR_1D = "parallelP1d";
|
|
535
|
+
const DENSE_BLOCK_MS = 10; // 100 parallel blocks per 1000ms root slot
|
|
536
|
+
const PER_SLOT = BLOCK_TIME_MS / DENSE_BLOCK_MS; // 100 parallel items per root slot
|
|
537
|
+
// Step + cap small enough that cap + one overshoot chunk (~40) stays below PER_SLOT
|
|
538
|
+
// (100), so the fetcher pauses before its page can cross the slot boundary.
|
|
539
|
+
const DENSE_STEP = 10;
|
|
540
|
+
const CAP_1D = 30;
|
|
541
|
+
const LIVE_TARGET_1D = 20;
|
|
542
|
+
// Bounded by the per-slot need plus one chunk — not the whole tip.
|
|
543
|
+
const DENSE_BUFFER_BOUND = PER_SLOT + DENSE_STEP + 150;
|
|
544
|
+
|
|
545
|
+
function buildConfig1d() {
|
|
546
|
+
return new ConfigBuilder()
|
|
547
|
+
.setNamespace((b) => b.setSecurityNamespace("sync-buffering-1d"))
|
|
548
|
+
.buildNetworks((b) =>
|
|
549
|
+
b
|
|
550
|
+
.addNetwork({
|
|
551
|
+
name: "clock",
|
|
552
|
+
type: ConfigNetworkType.TEST,
|
|
553
|
+
startTime: START_TIME,
|
|
554
|
+
blockTimeMS: BLOCK_TIME_MS,
|
|
555
|
+
})
|
|
556
|
+
.addNetwork({
|
|
557
|
+
name: "chainDense",
|
|
558
|
+
type: ConfigNetworkType.TEST,
|
|
559
|
+
startTime: START_TIME,
|
|
560
|
+
blockTimeMS: DENSE_BLOCK_MS,
|
|
561
|
+
})
|
|
562
|
+
)
|
|
563
|
+
.buildDeployments((b) => b)
|
|
564
|
+
.buildSyncProtocols((b) =>
|
|
565
|
+
b
|
|
566
|
+
.addMain(
|
|
567
|
+
(n) => n.clock,
|
|
568
|
+
() => ({
|
|
569
|
+
name: MAIN_1D,
|
|
570
|
+
type: ConfigSyncProtocolType.TEST_MAIN,
|
|
571
|
+
startBlockHeight: 1,
|
|
572
|
+
pollingInterval: 30,
|
|
573
|
+
stepSize: DENSE_STEP,
|
|
574
|
+
maxBufferedPages: 200,
|
|
575
|
+
}),
|
|
576
|
+
)
|
|
577
|
+
.addParallel(
|
|
578
|
+
(n) => (n as any).chainDense,
|
|
579
|
+
() => ({
|
|
580
|
+
name: PAR_1D,
|
|
581
|
+
type: ConfigSyncProtocolType.TEST_PARALLEL,
|
|
582
|
+
startBlockHeight: 1,
|
|
583
|
+
pollingInterval: 30,
|
|
584
|
+
stepSize: DENSE_STEP,
|
|
585
|
+
delayMs: 0,
|
|
586
|
+
events: [],
|
|
587
|
+
maxBufferedPages: CAP_1D,
|
|
588
|
+
}),
|
|
589
|
+
)
|
|
590
|
+
)
|
|
591
|
+
.buildPrimitives((b) =>
|
|
592
|
+
b.addPrimitive(
|
|
593
|
+
(sp) => (sp as any)[PAR_1D],
|
|
594
|
+
() => ({ name: "noEvt1d", type: TEST_PRIMITIVE_TYPE, startBlockHeight: 1 }),
|
|
595
|
+
)
|
|
596
|
+
)
|
|
597
|
+
.build();
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
test("1d: a parallel chain denser than the cap does NOT deadlock the merge", async () => {
|
|
601
|
+
TestChainControl.setTip(MAIN_1D, 60);
|
|
602
|
+
TestChainControl.setTip(PAR_1D, 10_000);
|
|
603
|
+
|
|
604
|
+
const node = await h.runNode({ config: buildConfig1d(), apiPort: 19144 });
|
|
605
|
+
const samples: BufSample[] = [];
|
|
606
|
+
let peakPar = 0;
|
|
607
|
+
let lastApplied = 0;
|
|
608
|
+
try {
|
|
609
|
+
const startMs = Date.now();
|
|
610
|
+
// Generous for slow CI; exits early once the liveness target is reached.
|
|
611
|
+
const deadline = startMs + 30_000;
|
|
612
|
+
while (Date.now() < deadline) {
|
|
613
|
+
const wall = Date.now();
|
|
614
|
+
const mainBuf = bufferSize(node, MAIN_1D);
|
|
615
|
+
const parBuf = bufferSize(node, PAR_1D);
|
|
616
|
+
const appliedBlock = await latestFinalizedHeight(h.pool);
|
|
617
|
+
samples.push({
|
|
618
|
+
t: wall - startMs,
|
|
619
|
+
wall,
|
|
620
|
+
mainBuf,
|
|
621
|
+
evmBuf: parBuf,
|
|
622
|
+
mainOwnBlock: ownBlock(node, MAIN_1D),
|
|
623
|
+
evmOwnBlock: ownBlock(node, PAR_1D),
|
|
624
|
+
rss: process.memoryUsage().rss,
|
|
625
|
+
appliedBlock,
|
|
626
|
+
});
|
|
627
|
+
peakPar = Math.max(peakPar, parBuf);
|
|
628
|
+
lastApplied = appliedBlock ?? lastApplied;
|
|
629
|
+
if (lastApplied >= LIVE_TARGET_1D) break;
|
|
630
|
+
await sleep(50);
|
|
631
|
+
}
|
|
632
|
+
} finally {
|
|
633
|
+
writeArtifact("buffering-1d", samples, {
|
|
634
|
+
mode: "1d-density-deadlock",
|
|
635
|
+
stepSize: SMALL_STEP,
|
|
636
|
+
denseBlockMs: DENSE_BLOCK_MS,
|
|
637
|
+
perSlot: PER_SLOT,
|
|
638
|
+
cap: CAP_1D,
|
|
639
|
+
liveTarget: LIVE_TARGET_1D,
|
|
640
|
+
peakParBuf: peakPar,
|
|
641
|
+
lastApplied,
|
|
642
|
+
});
|
|
643
|
+
await node.stop();
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
// Liveness: the node produces blocks (no wedge on block 1).
|
|
647
|
+
expect(lastApplied).toBeGreaterThanOrEqual(LIVE_TARGET_1D);
|
|
648
|
+
// Bounded by the per-slot need: ~one slot of parallel data plus one chunk.
|
|
649
|
+
expect(peakPar).toBeLessThanOrEqual(DENSE_BUFFER_BOUND);
|
|
650
|
+
}, 60_000);
|