@effectstream/runtime 0.100.19 → 0.100.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,26 +2,90 @@
2
2
  * Sanity end-to-end test for the synthetic `test` chain (no fixes required).
3
3
  *
4
4
  * Proves the foundation this PR adds works: a TEST_MAIN clock + a TEST_PARALLEL
5
- * chain sync through the real runtime (`start()`), a config-declared event is
6
- * processed into `effectstream.primitive_accounting`, and committed state
7
- * survives a restart.
8
- *
9
- * Runs on whichever backend `setupHarness` picks: the default in-memory PGLite
10
- * (no external dependency) or real Postgres when `PGLITE=false`. Either way
11
- * each node runs in its own subprocess (see harness.ts / node-runner.ts), so a
12
- * restart is a genuine new OS process booting against the same database.
5
+ * chain sync through the real runtime (`start()` + PGLite), a config-declared
6
+ * event is processed into `effectstream.primitive_accounting`, and committed
7
+ * state survives a restart.
13
8
  *
14
9
  * (The buffering / resume / fast-catchup reproduction tests land alongside their
15
10
  * fixes in follow-up PRs — they assert behaviour that only those fixes provide.)
16
11
  */
17
12
  import { afterAll, beforeAll, expect, test } from "bun:test";
13
+ import {
14
+ ConfigBuilder,
15
+ ConfigNetworkType,
16
+ ConfigSyncProtocolType,
17
+ } from "@effectstream/config";
18
+ import { TestChainControl } from "@effectstream/sync";
18
19
  import {
19
20
  countEventRows,
20
21
  type Harness,
21
22
  latestFinalizedHeight,
22
23
  setupHarness,
24
+ TEST_PRIMITIVE_TYPE,
25
+ waitForHeight,
26
+ waitUntilStable,
23
27
  } from "./harness.ts";
24
28
 
29
+ const START_TIME = 1_700_000_000_000;
30
+ const BLOCK_TIME_MS = 1000;
31
+
32
+ function buildConfig() {
33
+ return new ConfigBuilder()
34
+ .setNamespace((b) => b.setSecurityNamespace("sync-sanity"))
35
+ .buildNetworks((b) =>
36
+ b
37
+ .addNetwork({
38
+ name: "clock",
39
+ type: ConfigNetworkType.TEST,
40
+ startTime: START_TIME,
41
+ blockTimeMS: BLOCK_TIME_MS,
42
+ })
43
+ .addNetwork({
44
+ name: "chainP",
45
+ type: ConfigNetworkType.TEST,
46
+ startTime: START_TIME,
47
+ blockTimeMS: BLOCK_TIME_MS,
48
+ })
49
+ )
50
+ .buildDeployments((b) => b)
51
+ .buildSyncProtocols((b) =>
52
+ b
53
+ .addMain(
54
+ (n) => n.clock,
55
+ () => ({
56
+ name: "mainClock",
57
+ type: ConfigSyncProtocolType.TEST_MAIN,
58
+ startBlockHeight: 1,
59
+ pollingInterval: 30,
60
+ stepSize: 1000,
61
+ }),
62
+ )
63
+ .addParallel(
64
+ (n) => (n as any).chainP,
65
+ () => ({
66
+ name: "parallelP",
67
+ type: ConfigSyncProtocolType.TEST_PARALLEL,
68
+ startBlockHeight: 1,
69
+ pollingInterval: 30,
70
+ stepSize: 1000,
71
+ delayMs: 0,
72
+ events: [{ atBlock: 50, payload: { marker: "evt50" } }],
73
+ }),
74
+ )
75
+ )
76
+ .buildPrimitives((b) =>
77
+ b.addPrimitive(
78
+ (sp) => (sp as any).parallelP,
79
+ () => ({
80
+ name: "testEvt",
81
+ type: TEST_PRIMITIVE_TYPE,
82
+ startBlockHeight: 1,
83
+ }),
84
+ )
85
+ )
86
+ .build();
87
+ }
88
+
25
89
  let h: Harness;
26
90
 
27
91
  beforeAll(async () => {
@@ -33,42 +97,29 @@ afterAll(async () => {
33
97
  });
34
98
 
35
99
  test("test chain syncs, processes a configured event, and survives restart", async () => {
36
- const events = [{ atBlock: 50, marker: "evt50" }];
37
-
38
100
  // Parallel tip (200) stays ahead of the main clock (100) so the merge is not
39
101
  // gated at the chunk boundary.
40
- await h.runToHeight({
41
- events,
42
- tips: { mainClock: 100, parallelP: 200 },
43
- target: 100,
44
- apiPort: 19131,
45
- });
102
+ TestChainControl.setTip("mainClock", 100);
103
+ TestChainControl.setTip("parallelP", 200);
104
+
105
+ const n1 = await h.runNode({ config: buildConfig(), apiPort: 19131 });
106
+ await waitForHeight(n1, h.pool, 100);
107
+ await waitUntilStable(n1, h.pool);
46
108
 
47
- const height1 = await latestFinalizedHeight(h.pool);
48
- const events1 = await countEventRows(h.pool, "evt50");
49
- console.log(
50
- `[sanity] after sync: height ${height1} (expect 100), ` +
51
- `evt50 rows ${events1} (expect 1)`,
52
- );
53
- expect(height1).toBe(100);
109
+ expect(await latestFinalizedHeight(h.pool)).toBe(100);
54
110
  // The event declared at block 50 was processed into primitive_accounting.
55
- expect(events1).toBe(1);
111
+ expect(await countEventRows(h.pool, "evt50")).toBe(1);
112
+ await n1.stop();
113
+
114
+ // Restart: committed state must persist (chain already fully caught up).
115
+ await h.simulateRestart();
116
+ TestChainControl.setTip("mainClock", 100);
117
+ TestChainControl.setTip("parallelP", 200);
56
118
 
57
- // Restart: a new process boots against the same database. Committed state must
58
- // persist; the chain is already fully caught up, so nothing new is produced.
59
- await h.runToHeight({
60
- events,
61
- tips: { mainClock: 100, parallelP: 200 },
62
- target: 100,
63
- apiPort: 19132,
64
- });
119
+ const n2 = await h.runNode({ config: buildConfig(), apiPort: 19132 });
120
+ await waitUntilStable(n2, h.pool);
65
121
 
66
- const height2 = await latestFinalizedHeight(h.pool);
67
- const events2 = await countEventRows(h.pool, "evt50");
68
- console.log(
69
- `[sanity] after restart: height ${height2} (expect 100, was ${height1}), ` +
70
- `evt50 rows ${events2} (expect 1, was ${events1})`,
71
- );
72
- expect(height2).toBe(100);
73
- expect(events2).toBe(1);
122
+ expect(await latestFinalizedHeight(h.pool)).toBe(100);
123
+ expect(await countEventRows(h.pool, "evt50")).toBe(1);
124
+ await n2.stop();
74
125
  }, 60_000);
@@ -1,114 +0,0 @@
1
- # Bug Report: non-deterministic `effectstream_tx_hash` (`conciseInputHash`)
2
-
3
- **Package:** `@effectstream/node-sdk` — runtime
4
- **Location:** `runtime/src/process-blocks.ts` (scheduled-input loop, STEP 5)
5
- **Severity:** Medium — silent cross-node state divergence in a derived column; not (currently) consensus-relevant
6
- **Status:** Open
7
-
8
- ---
9
-
10
- ## Summary
11
-
12
- When the runtime records the result of a **scheduled** STF input, it fabricates the
13
- `effectstream_tx_hash` with `Math.random()`. The value is therefore different on every run
14
- and on every node, so two nodes that replay the exact same source blocks produce **different
15
- `rollup_input_result.effectstream_tx_hash` values** for the same input. For a framework whose
16
- core promise is "the database is a pure function of the source blocks," this is a determinism
17
- hole.
18
-
19
- ## The code
20
-
21
- `runtime/src/process-blocks.ts`, inside the loop over `scheduledData` (STEP 5):
22
-
23
- ```ts
24
- const conciseInputHash = `0x${
25
- Array(64).fill(0).map(() => Math.floor(Math.random() * 16).toString(16)).join("")
26
- }`;
27
- yield* until(
28
- insertGameInputResult.run({
29
- id: data.id,
30
- success,
31
- effectstream_tx_hash: Buffer.from(conciseInputHash),
32
- index_in_block,
33
- block_height: value.blockNumber,
34
- }, dbConn),
35
- );
36
- ```
37
-
38
- This runs only for **scheduled** inputs (engine-queued timers / derived inputs that have no
39
- real on-chain transaction), so the engine has to synthesize a hash. It does so randomly.
40
-
41
- ## Why it matters / why it's currently masked
42
-
43
- - **Block hash is unaffected.** `generateEffectstreamBlockHash` (`@effectstream/crypto`) hashes
44
- `blockInfo[].blockHash` (source-chain hashes) + the previous block hash — never
45
- `effectstream_tx_hash`. So the block-hash chain stays deterministic; the divergence is
46
- confined to the `rollup_input_result.effectstream_tx_hash` column and anything that reads it.
47
- - **Masked in tests.** The reproduction tests use a no-op STF (`stateMachinePayload: null`),
48
- which schedules no STF inputs, so this branch never executes. That is exactly why
49
- `consistency-snapshot.ts` lists `effectstream_tx_hash` in `VOLATILE_COLUMNS` and excludes it
50
- — the snapshot would otherwise diff on every run once real STFs are involved.
51
-
52
- ## Two existing quirks to fix alongside
53
-
54
- 1. **Encoding bug.** `Buffer.from(conciseInputHash)` is given the *string* `"0x" + 64 hex`
55
- with no encoding, so it stores 66 **UTF-8** bytes, not the 32 decoded bytes. Whatever
56
- derivation replaces `Math.random()`, encode it deliberately (`Buffer.from(hex, "hex")`, or
57
- commit to the `0x`-string form consistently).
58
-
59
- 2. **Ordering prerequisite (the real root).** The hash inputs that give uniqueness
60
- (`index_in_block`, and the STF execution order itself) are only deterministic if the
61
- `scheduledData` order is deterministic. Today it is not fully:
62
- ```ts
63
- // TODO What should be the order of the scheduled data - per id?
64
- const scheduledData = [...scheduledData1, ...scheduledData2];
65
- ```
66
- - `scheduledData1` (block-scheduled) is `ORDER BY id ASC` → deterministic given a
67
- deterministic scheduling history.
68
- - `scheduledData2` (timestamp-scheduled) is `ORDER BY future_ms_timestamp ASC` **only** —
69
- **same-timestamp rows have no tiebreak**, so their order is arbitrary.
70
-
71
- This is bigger than the hash: the same order drives **STF execution** and the
72
- **`randomGenerator` (Prando) draw sequence**. If it isn't pinned, game state itself can
73
- diverge; the random tx-hash is just the most visible symptom.
74
-
75
- ## Suggested fixes (in order)
76
-
77
- ### 1. Pin the scheduled-input order (prerequisite, do first)
78
- Give the queries a total deterministic sort and a deterministic merge:
79
- - timestamp list → `ORDER BY future_ms_timestamp ASC, id ASC` (break ties by `id`);
80
- - keep block-scheduled before timestamp-scheduled (or merge both by a single deterministic
81
- key). This is required for deterministic STFs regardless of the hash, and it makes
82
- `index_in_block` deterministic.
83
-
84
- ### 2. Derive the hash deterministically (recommended)
85
- Replace `Math.random()` with a content-addressed hash over the input's identity + position,
86
- reusing the `sha512` already used for the block hash:
87
-
88
- ```
89
- effectstream_tx_hash =
90
- sha512(block_height ‖ index_in_block ‖ input_data ‖ from_address ‖ from_address_type)
91
- ```
92
-
93
- - `block_height` + `index_in_block` give **uniqueness** (one result per position; needs fix #1).
94
- - `input_data` + `from_address` + `from_address_type` make it **content-addressed**
95
- (tx-hash-like, collision-resistant). All are non-null columns already on the scheduled row.
96
-
97
- ### Alternatives considered
98
- - **Reuse the seeded `Prando` (`new Prando(blockHash)`):** smallest change and deterministic,
99
- but the shared `randomGenerator` is also consumed by the STFs — drawing hash bytes from it
100
- would shift the RNG stream every later STF sees. Would require a *separate*
101
- `new Prando(blockHash + ":txhash")`. Produces opaque (non-content) hashes.
102
- - **`H(data.id, block_height)`:** `id` is a global SERIAL counter — a weaker determinism
103
- guarantee than the block-scoped `index_in_block`, and `block_height` adds nothing once `id`
104
- is unique. Prefer `index_in_block`.
105
- - **Provenance: `H(origin_tx_hash, …)`:** ties to the originating real tx, but
106
- `origin_tx_hash` is `null` for timestamp-scheduled inputs, so it needs a fallback.
107
-
108
- ## Verification
109
- - Add a determinism test with a **real** STF (one that schedules a derived input) and assert
110
- two from-scratch syncs produce identical `rollup_input_result.effectstream_tx_hash` —
111
- i.e. drop `effectstream_tx_hash` from `VOLATILE_COLUMNS` once fixed and confirm
112
- `consistency.test.ts` Test A still passes.
113
- - Cover the timestamp-tie case explicitly: schedule ≥2 inputs at the same `future_ms_timestamp`
114
- and assert identical ordering / hashes across runs.
@@ -1,30 +0,0 @@
1
- /**
2
- * Apply-stage liveness for /debug/metrics.
3
- *
4
- * The serial finalized-block loop in main.ts calls recordAppliedBlock() right
5
- * after each block COMMITs; the /debug/metrics handler reads `appliedBlockStatus`
6
- * to report true lag (now − applied block timestamp). Unlike a protocol's `buf`
7
- * (fetch backlog), this stays high when the node is write/apply-bound.
8
- *
9
- * Process-global: one node runs per process, so a module-level singleton keeps
10
- * main.ts to a single call instead of threading a ref through startHttpServer.
11
- */
12
- export type AppliedBlockStatus = {
13
- blockNumber: number | null;
14
- timestamp: number | null;
15
- appliedAtMs: number | null;
16
- };
17
-
18
- export const appliedBlockStatus: AppliedBlockStatus = {
19
- blockNumber: null,
20
- timestamp: null,
21
- appliedAtMs: null,
22
- };
23
-
24
- export function recordAppliedBlock(
25
- block: { blockNumber: number | bigint; timestamp: number | bigint },
26
- ): void {
27
- appliedBlockStatus.blockNumber = Number(block.blockNumber);
28
- appliedBlockStatus.timestamp = Number(block.timestamp);
29
- appliedBlockStatus.appliedAtMs = Date.now();
30
- }
@@ -1,389 +0,0 @@
1
- /**
2
- * Verifies the fetch-backpressure feature (Option B′) over the synthetic `test`
3
- * chain through the real runtime (`start()` + in-process PGLite).
4
- *
5
- * 1a — unbounded buffering: with a cap, the fetch loop pauses when the buffer is
6
- * full, so it stays bounded (~cap + stepSize) instead of racing to the whole
7
- * backlog — while the node keeps applying blocks (liveness).
8
- * 1b — head-of-line blocking: a stalled slow chain STILL halts block production
9
- * (the merge gate is correct), but the fast chain's buffer is now bounded by
10
- * the cap instead of ballooning behind the head-of-line block.
11
- *
12
- * Before the fix these buffers reached ~49k–50k (see
13
- * e2e/perf/results/BACKPRESSURE-BASELINE.md); these tests assert they now
14
- * stay at the cap. Each test also writes a JSON time-series artifact to
15
- * e2e/perf/results/ whose field names match e2e/perf/metrics.ts `Sample`, so the
16
- * perf HTML report's Backpressure section can render the same signals.
17
- */
18
- import { afterEach, beforeEach, expect, test } from "bun:test";
19
- import {
20
- ConfigBuilder,
21
- ConfigNetworkType,
22
- ConfigSyncProtocolType,
23
- } from "@effectstream/config";
24
- import { TestChainControl } from "@effectstream/sync";
25
- import fs from "node:fs";
26
- import path from "node:path";
27
- import {
28
- bufferSize,
29
- type Harness,
30
- latestFinalizedHeight,
31
- type RunningNode,
32
- setupHarness,
33
- sleep,
34
- TEST_PRIMITIVE_TYPE,
35
- } from "./harness.ts";
36
-
37
- const START_TIME = 1_700_000_000_000;
38
- const BLOCK_TIME_MS = 1000;
39
- const STEP_SIZE = 1000;
40
- // A deep backlog the arithmetic fetcher would race through if uncapped (baseline
41
- // hit ~49k–50k). The slow chain (1b) is pinned low so the merge stalls at its tip.
42
- const BIG_TIP = 50_000;
43
- const SLOW_TIP = 50;
44
- // Explicit small cap so the bound is deterministic and far below the backlog.
45
- // The fetch pauses at >= CAP; one in-flight chunk can overshoot by ~stepSize, so
46
- // the buffer settles in [~CAP, CAP + stepSize]. We assert it lands in that band.
47
- const CAP = 3000;
48
- const CAP_FLOOR = CAP - STEP_SIZE; // proves backpressure actually engaged
49
- const BUFFER_BOUND = CAP + STEP_SIZE + 200; // upper bound incl. one overshoot chunk
50
-
51
- // e2e/perf/results lives at the repo root: reproduction → test → runtime →
52
- // node-sdk → packages → <root>/e2e/perf/results.
53
- const RESULTS_DIR = path.resolve(
54
- import.meta.dirname!,
55
- "../../../../../e2e/perf/results",
56
- );
57
-
58
- /** One time-series row; field names mirror e2e/perf/metrics.ts `Sample`. */
59
- type BufSample = {
60
- t: number;
61
- wall: number;
62
- mainBuf: number;
63
- evmBuf: number; // the parallel chain's buffer (report labels it "parallel buf")
64
- mainOwnBlock: number | null;
65
- evmOwnBlock: number | null;
66
- rss: number;
67
- appliedBlock: number | null;
68
- };
69
-
70
- function ownBlock(node: RunningNode, name: string): number | null {
71
- return node.syncProtocols().find((p) => p.name === name)?.lastPage
72
- ?.ownBlockNumber ?? null;
73
- }
74
-
75
- /** Backpressure "fired" counter (rising edges) exposed on SyncState / /debug/metrics. */
76
- function pauses(node: RunningNode, name: string): number {
77
- return node.syncProtocols().find((p) => p.name === name)
78
- ?.backpressurePauses ?? 0;
79
- }
80
-
81
- /** Side-effect-only artifact write; never throws into the test. */
82
- function writeArtifact(
83
- name: string,
84
- samples: BufSample[],
85
- meta: Record<string, unknown>,
86
- ): void {
87
- try {
88
- fs.mkdirSync(RESULTS_DIR, { recursive: true });
89
- const stamp = new Date().toISOString().replace(/[:.]/g, "-");
90
- const file = path.join(RESULTS_DIR, `${name}-${stamp}.json`);
91
- fs.writeFileSync(file, JSON.stringify({ meta, samples }, null, 2));
92
- console.log(`buffering artifact: ${file}`);
93
- } catch (e) {
94
- console.warn(`failed to write buffering artifact: ${String(e)}`);
95
- }
96
- }
97
-
98
- let h: Harness;
99
- beforeEach(async () => {
100
- h = await setupHarness();
101
- });
102
- afterEach(async () => {
103
- await h?.teardown();
104
- });
105
-
106
- // ── 1a — unbounded buffering, now bounded by the cap ────────────────────────────
107
-
108
- const MAIN_1A = "mainClock1a";
109
- const PAR_1A = "parallelP1a";
110
-
111
- function buildConfig1a() {
112
- return new ConfigBuilder()
113
- .setNamespace((b) => b.setSecurityNamespace("sync-buffering-1a"))
114
- .buildNetworks((b) =>
115
- b
116
- .addNetwork({
117
- name: "clock",
118
- type: ConfigNetworkType.TEST,
119
- startTime: START_TIME,
120
- blockTimeMS: BLOCK_TIME_MS,
121
- })
122
- .addNetwork({
123
- name: "chainP",
124
- type: ConfigNetworkType.TEST,
125
- startTime: START_TIME,
126
- blockTimeMS: BLOCK_TIME_MS,
127
- })
128
- )
129
- .buildDeployments((b) => b)
130
- .buildSyncProtocols((b) =>
131
- b
132
- .addMain(
133
- (n) => n.clock,
134
- () => ({
135
- name: MAIN_1A,
136
- type: ConfigSyncProtocolType.TEST_MAIN,
137
- startBlockHeight: 1,
138
- pollingInterval: 30,
139
- stepSize: STEP_SIZE,
140
- maxBufferedPages: CAP,
141
- }),
142
- )
143
- .addParallel(
144
- (n) => (n as any).chainP,
145
- () => ({
146
- name: PAR_1A,
147
- type: ConfigSyncProtocolType.TEST_PARALLEL,
148
- startBlockHeight: 1,
149
- pollingInterval: 30,
150
- stepSize: STEP_SIZE,
151
- delayMs: 0,
152
- events: [],
153
- maxBufferedPages: CAP,
154
- }),
155
- )
156
- )
157
- .buildPrimitives((b) =>
158
- b.addPrimitive(
159
- (sp) => (sp as any)[PAR_1A],
160
- () => ({ name: "noEvt1a", type: TEST_PRIMITIVE_TYPE, startBlockHeight: 1 }),
161
- )
162
- )
163
- .build();
164
- }
165
-
166
- test("1a: backpressure caps the buffer while the node keeps applying blocks", async () => {
167
- // Tips far ahead of where the one-block-per-txn merge can drain to: without a
168
- // cap the fetcher would race to ~50k; with the cap it must plateau near CAP.
169
- TestChainControl.setTip(MAIN_1A, BIG_TIP);
170
- TestChainControl.setTip(PAR_1A, BIG_TIP);
171
-
172
- const node = await h.runNode({ config: buildConfig1a(), apiPort: 19141 });
173
- const samples: BufSample[] = [];
174
- let peakMain = 0;
175
- let peakEvm = 0;
176
- let lastApplied = 0;
177
- let mainPauses = 0;
178
- try {
179
- const startMs = Date.now();
180
- const deadline = startMs + 6000;
181
- while (Date.now() < deadline) {
182
- const wall = Date.now();
183
- const mainBuf = bufferSize(node, MAIN_1A);
184
- const evmBuf = bufferSize(node, PAR_1A);
185
- const appliedBlock = await latestFinalizedHeight(h.pool);
186
- samples.push({
187
- t: wall - startMs,
188
- wall,
189
- mainBuf,
190
- evmBuf,
191
- mainOwnBlock: ownBlock(node, MAIN_1A),
192
- evmOwnBlock: ownBlock(node, PAR_1A),
193
- rss: process.memoryUsage().rss,
194
- appliedBlock,
195
- });
196
- peakMain = Math.max(peakMain, mainBuf);
197
- peakEvm = Math.max(peakEvm, evmBuf);
198
- lastApplied = appliedBlock ?? lastApplied;
199
- // Backpressure engaged (buffer hit the cap) AND the node is still applying
200
- // blocks → we've captured the bounded steady state; stop early.
201
- if (peakMain >= CAP && lastApplied >= 20) break;
202
- await sleep(50);
203
- }
204
- } finally {
205
- mainPauses = pauses(node, MAIN_1A);
206
- writeArtifact("buffering-1a", samples, {
207
- mode: "1a-bounded-by-cap",
208
- stepSize: STEP_SIZE,
209
- bigTip: BIG_TIP,
210
- cap: CAP,
211
- bufferBound: BUFFER_BOUND,
212
- peakMainBuf: peakMain,
213
- peakEvmBuf: peakEvm,
214
- lastApplied,
215
- pauses: mainPauses,
216
- });
217
- await node.stop();
218
- }
219
-
220
- // Backpressure engaged: the buffer reached the cap (not starved)...
221
- expect(peakMain).toBeGreaterThanOrEqual(CAP_FLOOR);
222
- // ...but stayed bounded — nowhere near the uncapped ~49k backlog.
223
- expect(Math.max(peakMain, peakEvm)).toBeLessThanOrEqual(BUFFER_BOUND);
224
- // ...the node kept producing/applying blocks (no deadlock)...
225
- expect(lastApplied).toBeGreaterThanOrEqual(20);
226
- // ...and the cap actually fired (the observability metric populated).
227
- expect(mainPauses).toBeGreaterThan(0);
228
- }, 60_000);
229
-
230
- // ── 1b — head-of-line blocking, sibling buffer now bounded ──────────────────────
231
-
232
- const MAIN_1B = "mainClock1b";
233
- const FAST_1B = "fastP1b";
234
- const SLOW_1B = "slowP1b";
235
-
236
- function buildConfig1b() {
237
- return new ConfigBuilder()
238
- .setNamespace((b) => b.setSecurityNamespace("sync-buffering-1b"))
239
- .buildNetworks((b) =>
240
- b
241
- .addNetwork({
242
- name: "clock",
243
- type: ConfigNetworkType.TEST,
244
- startTime: START_TIME,
245
- blockTimeMS: BLOCK_TIME_MS,
246
- })
247
- .addNetwork({
248
- name: "chainFast",
249
- type: ConfigNetworkType.TEST,
250
- startTime: START_TIME,
251
- blockTimeMS: BLOCK_TIME_MS,
252
- })
253
- .addNetwork({
254
- name: "chainSlow",
255
- type: ConfigNetworkType.TEST,
256
- startTime: START_TIME,
257
- blockTimeMS: BLOCK_TIME_MS,
258
- })
259
- )
260
- .buildDeployments((b) => b)
261
- .buildSyncProtocols((b) =>
262
- b
263
- .addMain(
264
- (n) => n.clock,
265
- () => ({
266
- name: MAIN_1B,
267
- type: ConfigSyncProtocolType.TEST_MAIN,
268
- startBlockHeight: 1,
269
- pollingInterval: 30,
270
- stepSize: STEP_SIZE,
271
- maxBufferedPages: CAP,
272
- }),
273
- )
274
- .addParallel(
275
- (n) => (n as any).chainFast,
276
- () => ({
277
- name: FAST_1B,
278
- type: ConfigSyncProtocolType.TEST_PARALLEL,
279
- startBlockHeight: 1,
280
- pollingInterval: 30,
281
- stepSize: STEP_SIZE,
282
- delayMs: 0,
283
- events: [],
284
- maxBufferedPages: CAP,
285
- }),
286
- )
287
- .addParallel(
288
- (n) => (n as any).chainSlow,
289
- () => ({
290
- name: SLOW_1B,
291
- type: ConfigSyncProtocolType.TEST_PARALLEL,
292
- startBlockHeight: 1,
293
- pollingInterval: 30,
294
- stepSize: STEP_SIZE,
295
- delayMs: 0,
296
- events: [],
297
- maxBufferedPages: CAP,
298
- }),
299
- )
300
- )
301
- .buildPrimitives((b) =>
302
- b
303
- .addPrimitive(
304
- (sp) => (sp as any)[FAST_1B],
305
- () => ({ name: "noEvtFast", type: TEST_PRIMITIVE_TYPE, startBlockHeight: 1 }),
306
- )
307
- .addPrimitive(
308
- (sp) => (sp as any)[SLOW_1B],
309
- () => ({ name: "noEvtSlow", type: TEST_PRIMITIVE_TYPE, startBlockHeight: 1 }),
310
- )
311
- )
312
- .build();
313
- }
314
-
315
- test("1b: a stalled chain still halts production, but the fast chain's buffer stays capped", async () => {
316
- // slowP pinned at a low tip → the merge stalls there forever (correct). fastP and
317
- // the main clock keep fetching toward their high tips, but the cap now bounds them.
318
- TestChainControl.setTip(MAIN_1B, BIG_TIP);
319
- TestChainControl.setTip(FAST_1B, BIG_TIP);
320
- TestChainControl.setTip(SLOW_1B, SLOW_TIP);
321
-
322
- const node = await h.runNode({ config: buildConfig1b(), apiPort: 19142 });
323
- const samples: BufSample[] = [];
324
- let peakMain = 0;
325
- let peakFast = 0;
326
- let stalledHeight = -1;
327
- let fastPauses = 0;
328
- try {
329
- const startMs = Date.now();
330
- const deadline = startMs + 15_000;
331
- let lastHeight = -1;
332
- let lastChange = Date.now();
333
- while (Date.now() < deadline) {
334
- const wall = Date.now();
335
- const mainBuf = bufferSize(node, MAIN_1B);
336
- const fastBuf = bufferSize(node, FAST_1B);
337
- const appliedBlock = await latestFinalizedHeight(h.pool);
338
- samples.push({
339
- t: wall - startMs,
340
- wall,
341
- mainBuf,
342
- evmBuf: fastBuf,
343
- mainOwnBlock: ownBlock(node, MAIN_1B),
344
- evmOwnBlock: ownBlock(node, FAST_1B),
345
- rss: process.memoryUsage().rss,
346
- appliedBlock,
347
- });
348
- peakMain = Math.max(peakMain, mainBuf);
349
- peakFast = Math.max(peakFast, fastBuf);
350
- if (appliedBlock !== lastHeight) {
351
- lastHeight = appliedBlock;
352
- lastChange = wall;
353
- }
354
- // Production has stalled (height flat ≥1.5s) AND the fast chain has hit its
355
- // cap → the head-of-line block holds while the sibling buffer is bounded.
356
- if (wall - lastChange >= 1500 && peakFast >= CAP_FLOOR) {
357
- stalledHeight = appliedBlock ?? -1;
358
- break;
359
- }
360
- await sleep(50);
361
- }
362
- } finally {
363
- fastPauses = pauses(node, FAST_1B);
364
- writeArtifact("buffering-1b", samples, {
365
- mode: "1b-hol-sibling-bounded",
366
- stepSize: STEP_SIZE,
367
- bigTip: BIG_TIP,
368
- slowTip: SLOW_TIP,
369
- cap: CAP,
370
- bufferBound: BUFFER_BOUND,
371
- stalledHeight,
372
- peakMainBuf: peakMain,
373
- peakFastBuf: peakFast,
374
- fastPauses,
375
- });
376
- await node.stop();
377
- }
378
-
379
- // Block production still stalls at ~the slow chain's tip (the merge gate is correct)...
380
- expect(stalledHeight).toBeGreaterThanOrEqual(0);
381
- expect(stalledHeight).toBeLessThanOrEqual(SLOW_TIP + 10);
382
- // ...the fast chain's buffer engaged the cap...
383
- expect(peakFast).toBeGreaterThanOrEqual(CAP_FLOOR);
384
- // ...but stayed bounded behind the head-of-line block (was ~50k uncapped)...
385
- expect(peakFast).toBeLessThanOrEqual(BUFFER_BOUND);
386
- expect(peakMain).toBeLessThanOrEqual(BUFFER_BOUND);
387
- // ...and the cap actually fired on the fast chain (observability metric populated).
388
- expect(fastPauses).toBeGreaterThan(0);
389
- }, 60_000);