@effectstream/runtime 0.100.19 → 0.100.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +13 -13
- package/src/api/http-server.ts +7 -55
- package/src/main.ts +38 -32
- package/src/process-blocks.ts +27 -126
- package/src/types.ts +0 -7
- package/test/reproduction/harness.ts +270 -556
- package/test/reproduction/sanity.test.ts +91 -40
- package/conciseinputhash-determinism.md +0 -114
- package/src/api/apply-status.ts +0 -30
- package/test/reproduction/buffering.test.ts +0 -389
- package/test/reproduction/consistency-snapshot.ts +0 -168
- package/test/reproduction/consistency.test.ts +0 -197
- package/test/reproduction/node-runner.ts +0 -152
- package/test/reproduction/poll.ts +0 -160
- package/test/reproduction/resume.test.ts +0 -77
- package/test/reproduction/scenario.ts +0 -126
|
@@ -2,26 +2,90 @@
|
|
|
2
2
|
* Sanity end-to-end test for the synthetic `test` chain (no fixes required).
|
|
3
3
|
*
|
|
4
4
|
* Proves the foundation this PR adds works: a TEST_MAIN clock + a TEST_PARALLEL
|
|
5
|
-
* chain sync through the real runtime (`start()`), a config-declared
|
|
6
|
-
* processed into `effectstream.primitive_accounting`, and committed
|
|
7
|
-
* survives a restart.
|
|
8
|
-
*
|
|
9
|
-
* Runs on whichever backend `setupHarness` picks: the default in-memory PGLite
|
|
10
|
-
* (no external dependency) or real Postgres when `PGLITE=false`. Either way
|
|
11
|
-
* each node runs in its own subprocess (see harness.ts / node-runner.ts), so a
|
|
12
|
-
* restart is a genuine new OS process booting against the same database.
|
|
5
|
+
* chain sync through the real runtime (`start()` + PGLite), a config-declared
|
|
6
|
+
* event is processed into `effectstream.primitive_accounting`, and committed
|
|
7
|
+
* state survives a restart.
|
|
13
8
|
*
|
|
14
9
|
* (The buffering / resume / fast-catchup reproduction tests land alongside their
|
|
15
10
|
* fixes in follow-up PRs — they assert behaviour that only those fixes provide.)
|
|
16
11
|
*/
|
|
17
12
|
import { afterAll, beforeAll, expect, test } from "bun:test";
|
|
13
|
+
import {
|
|
14
|
+
ConfigBuilder,
|
|
15
|
+
ConfigNetworkType,
|
|
16
|
+
ConfigSyncProtocolType,
|
|
17
|
+
} from "@effectstream/config";
|
|
18
|
+
import { TestChainControl } from "@effectstream/sync";
|
|
18
19
|
import {
|
|
19
20
|
countEventRows,
|
|
20
21
|
type Harness,
|
|
21
22
|
latestFinalizedHeight,
|
|
22
23
|
setupHarness,
|
|
24
|
+
TEST_PRIMITIVE_TYPE,
|
|
25
|
+
waitForHeight,
|
|
26
|
+
waitUntilStable,
|
|
23
27
|
} from "./harness.ts";
|
|
24
28
|
|
|
29
|
+
const START_TIME = 1_700_000_000_000;
|
|
30
|
+
const BLOCK_TIME_MS = 1000;
|
|
31
|
+
|
|
32
|
+
function buildConfig() {
|
|
33
|
+
return new ConfigBuilder()
|
|
34
|
+
.setNamespace((b) => b.setSecurityNamespace("sync-sanity"))
|
|
35
|
+
.buildNetworks((b) =>
|
|
36
|
+
b
|
|
37
|
+
.addNetwork({
|
|
38
|
+
name: "clock",
|
|
39
|
+
type: ConfigNetworkType.TEST,
|
|
40
|
+
startTime: START_TIME,
|
|
41
|
+
blockTimeMS: BLOCK_TIME_MS,
|
|
42
|
+
})
|
|
43
|
+
.addNetwork({
|
|
44
|
+
name: "chainP",
|
|
45
|
+
type: ConfigNetworkType.TEST,
|
|
46
|
+
startTime: START_TIME,
|
|
47
|
+
blockTimeMS: BLOCK_TIME_MS,
|
|
48
|
+
})
|
|
49
|
+
)
|
|
50
|
+
.buildDeployments((b) => b)
|
|
51
|
+
.buildSyncProtocols((b) =>
|
|
52
|
+
b
|
|
53
|
+
.addMain(
|
|
54
|
+
(n) => n.clock,
|
|
55
|
+
() => ({
|
|
56
|
+
name: "mainClock",
|
|
57
|
+
type: ConfigSyncProtocolType.TEST_MAIN,
|
|
58
|
+
startBlockHeight: 1,
|
|
59
|
+
pollingInterval: 30,
|
|
60
|
+
stepSize: 1000,
|
|
61
|
+
}),
|
|
62
|
+
)
|
|
63
|
+
.addParallel(
|
|
64
|
+
(n) => (n as any).chainP,
|
|
65
|
+
() => ({
|
|
66
|
+
name: "parallelP",
|
|
67
|
+
type: ConfigSyncProtocolType.TEST_PARALLEL,
|
|
68
|
+
startBlockHeight: 1,
|
|
69
|
+
pollingInterval: 30,
|
|
70
|
+
stepSize: 1000,
|
|
71
|
+
delayMs: 0,
|
|
72
|
+
events: [{ atBlock: 50, payload: { marker: "evt50" } }],
|
|
73
|
+
}),
|
|
74
|
+
)
|
|
75
|
+
)
|
|
76
|
+
.buildPrimitives((b) =>
|
|
77
|
+
b.addPrimitive(
|
|
78
|
+
(sp) => (sp as any).parallelP,
|
|
79
|
+
() => ({
|
|
80
|
+
name: "testEvt",
|
|
81
|
+
type: TEST_PRIMITIVE_TYPE,
|
|
82
|
+
startBlockHeight: 1,
|
|
83
|
+
}),
|
|
84
|
+
)
|
|
85
|
+
)
|
|
86
|
+
.build();
|
|
87
|
+
}
|
|
88
|
+
|
|
25
89
|
let h: Harness;
|
|
26
90
|
|
|
27
91
|
beforeAll(async () => {
|
|
@@ -33,42 +97,29 @@ afterAll(async () => {
|
|
|
33
97
|
});
|
|
34
98
|
|
|
35
99
|
test("test chain syncs, processes a configured event, and survives restart", async () => {
|
|
36
|
-
const events = [{ atBlock: 50, marker: "evt50" }];
|
|
37
|
-
|
|
38
100
|
// Parallel tip (200) stays ahead of the main clock (100) so the merge is not
|
|
39
101
|
// gated at the chunk boundary.
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
102
|
+
TestChainControl.setTip("mainClock", 100);
|
|
103
|
+
TestChainControl.setTip("parallelP", 200);
|
|
104
|
+
|
|
105
|
+
const n1 = await h.runNode({ config: buildConfig(), apiPort: 19131 });
|
|
106
|
+
await waitForHeight(n1, h.pool, 100);
|
|
107
|
+
await waitUntilStable(n1, h.pool);
|
|
46
108
|
|
|
47
|
-
|
|
48
|
-
const events1 = await countEventRows(h.pool, "evt50");
|
|
49
|
-
console.log(
|
|
50
|
-
`[sanity] after sync: height ${height1} (expect 100), ` +
|
|
51
|
-
`evt50 rows ${events1} (expect 1)`,
|
|
52
|
-
);
|
|
53
|
-
expect(height1).toBe(100);
|
|
109
|
+
expect(await latestFinalizedHeight(h.pool)).toBe(100);
|
|
54
110
|
// The event declared at block 50 was processed into primitive_accounting.
|
|
55
|
-
expect(
|
|
111
|
+
expect(await countEventRows(h.pool, "evt50")).toBe(1);
|
|
112
|
+
await n1.stop();
|
|
113
|
+
|
|
114
|
+
// Restart: committed state must persist (chain already fully caught up).
|
|
115
|
+
await h.simulateRestart();
|
|
116
|
+
TestChainControl.setTip("mainClock", 100);
|
|
117
|
+
TestChainControl.setTip("parallelP", 200);
|
|
56
118
|
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
await h.runToHeight({
|
|
60
|
-
events,
|
|
61
|
-
tips: { mainClock: 100, parallelP: 200 },
|
|
62
|
-
target: 100,
|
|
63
|
-
apiPort: 19132,
|
|
64
|
-
});
|
|
119
|
+
const n2 = await h.runNode({ config: buildConfig(), apiPort: 19132 });
|
|
120
|
+
await waitUntilStable(n2, h.pool);
|
|
65
121
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
`[sanity] after restart: height ${height2} (expect 100, was ${height1}), ` +
|
|
70
|
-
`evt50 rows ${events2} (expect 1, was ${events1})`,
|
|
71
|
-
);
|
|
72
|
-
expect(height2).toBe(100);
|
|
73
|
-
expect(events2).toBe(1);
|
|
122
|
+
expect(await latestFinalizedHeight(h.pool)).toBe(100);
|
|
123
|
+
expect(await countEventRows(h.pool, "evt50")).toBe(1);
|
|
124
|
+
await n2.stop();
|
|
74
125
|
}, 60_000);
|
|
@@ -1,114 +0,0 @@
|
|
|
1
|
-
# Bug Report: non-deterministic `effectstream_tx_hash` (`conciseInputHash`)
|
|
2
|
-
|
|
3
|
-
**Package:** `@effectstream/node-sdk` — runtime
|
|
4
|
-
**Location:** `runtime/src/process-blocks.ts` (scheduled-input loop, STEP 5)
|
|
5
|
-
**Severity:** Medium — silent cross-node state divergence in a derived column; not (currently) consensus-relevant
|
|
6
|
-
**Status:** Open
|
|
7
|
-
|
|
8
|
-
---
|
|
9
|
-
|
|
10
|
-
## Summary
|
|
11
|
-
|
|
12
|
-
When the runtime records the result of a **scheduled** STF input, it fabricates the
|
|
13
|
-
`effectstream_tx_hash` with `Math.random()`. The value is therefore different on every run
|
|
14
|
-
and on every node, so two nodes that replay the exact same source blocks produce **different
|
|
15
|
-
`rollup_input_result.effectstream_tx_hash` values** for the same input. For a framework whose
|
|
16
|
-
core promise is "the database is a pure function of the source blocks," this is a determinism
|
|
17
|
-
hole.
|
|
18
|
-
|
|
19
|
-
## The code
|
|
20
|
-
|
|
21
|
-
`runtime/src/process-blocks.ts`, inside the loop over `scheduledData` (STEP 5):
|
|
22
|
-
|
|
23
|
-
```ts
|
|
24
|
-
const conciseInputHash = `0x${
|
|
25
|
-
Array(64).fill(0).map(() => Math.floor(Math.random() * 16).toString(16)).join("")
|
|
26
|
-
}`;
|
|
27
|
-
yield* until(
|
|
28
|
-
insertGameInputResult.run({
|
|
29
|
-
id: data.id,
|
|
30
|
-
success,
|
|
31
|
-
effectstream_tx_hash: Buffer.from(conciseInputHash),
|
|
32
|
-
index_in_block,
|
|
33
|
-
block_height: value.blockNumber,
|
|
34
|
-
}, dbConn),
|
|
35
|
-
);
|
|
36
|
-
```
|
|
37
|
-
|
|
38
|
-
This runs only for **scheduled** inputs (engine-queued timers / derived inputs that have no
|
|
39
|
-
real on-chain transaction), so the engine has to synthesize a hash. It does so randomly.
|
|
40
|
-
|
|
41
|
-
## Why it matters / why it's currently masked
|
|
42
|
-
|
|
43
|
-
- **Block hash is unaffected.** `generateEffectstreamBlockHash` (`@effectstream/crypto`) hashes
|
|
44
|
-
`blockInfo[].blockHash` (source-chain hashes) + the previous block hash — never
|
|
45
|
-
`effectstream_tx_hash`. So the block-hash chain stays deterministic; the divergence is
|
|
46
|
-
confined to the `rollup_input_result.effectstream_tx_hash` column and anything that reads it.
|
|
47
|
-
- **Masked in tests.** The reproduction tests use a no-op STF (`stateMachinePayload: null`),
|
|
48
|
-
which schedules no STF inputs, so this branch never executes. That is exactly why
|
|
49
|
-
`consistency-snapshot.ts` lists `effectstream_tx_hash` in `VOLATILE_COLUMNS` and excludes it
|
|
50
|
-
— the snapshot would otherwise diff on every run once real STFs are involved.
|
|
51
|
-
|
|
52
|
-
## Two existing quirks to fix alongside
|
|
53
|
-
|
|
54
|
-
1. **Encoding bug.** `Buffer.from(conciseInputHash)` is given the *string* `"0x" + 64 hex`
|
|
55
|
-
with no encoding, so it stores 66 **UTF-8** bytes, not the 32 decoded bytes. Whatever
|
|
56
|
-
derivation replaces `Math.random()`, encode it deliberately (`Buffer.from(hex, "hex")`, or
|
|
57
|
-
commit to the `0x`-string form consistently).
|
|
58
|
-
|
|
59
|
-
2. **Ordering prerequisite (the real root).** The hash inputs that give uniqueness
|
|
60
|
-
(`index_in_block`, and the STF execution order itself) are only deterministic if the
|
|
61
|
-
`scheduledData` order is deterministic. Today it is not fully:
|
|
62
|
-
```ts
|
|
63
|
-
// TODO What should be the order of the scheduled data - per id?
|
|
64
|
-
const scheduledData = [...scheduledData1, ...scheduledData2];
|
|
65
|
-
```
|
|
66
|
-
- `scheduledData1` (block-scheduled) is `ORDER BY id ASC` → deterministic given a
|
|
67
|
-
deterministic scheduling history.
|
|
68
|
-
- `scheduledData2` (timestamp-scheduled) is `ORDER BY future_ms_timestamp ASC` **only** —
|
|
69
|
-
**same-timestamp rows have no tiebreak**, so their order is arbitrary.
|
|
70
|
-
|
|
71
|
-
This is bigger than the hash: the same order drives **STF execution** and the
|
|
72
|
-
**`randomGenerator` (Prando) draw sequence**. If it isn't pinned, game state itself can
|
|
73
|
-
diverge; the random tx-hash is just the most visible symptom.
|
|
74
|
-
|
|
75
|
-
## Suggested fixes (in order)
|
|
76
|
-
|
|
77
|
-
### 1. Pin the scheduled-input order (prerequisite, do first)
|
|
78
|
-
Give the queries a total deterministic sort and a deterministic merge:
|
|
79
|
-
- timestamp list → `ORDER BY future_ms_timestamp ASC, id ASC` (break ties by `id`);
|
|
80
|
-
- keep block-scheduled before timestamp-scheduled (or merge both by a single deterministic
|
|
81
|
-
key). This is required for deterministic STFs regardless of the hash, and it makes
|
|
82
|
-
`index_in_block` deterministic.
|
|
83
|
-
|
|
84
|
-
### 2. Derive the hash deterministically (recommended)
|
|
85
|
-
Replace `Math.random()` with a content-addressed hash over the input's identity + position,
|
|
86
|
-
reusing the `sha512` already used for the block hash:
|
|
87
|
-
|
|
88
|
-
```
|
|
89
|
-
effectstream_tx_hash =
|
|
90
|
-
sha512(block_height ‖ index_in_block ‖ input_data ‖ from_address ‖ from_address_type)
|
|
91
|
-
```
|
|
92
|
-
|
|
93
|
-
- `block_height` + `index_in_block` give **uniqueness** (one result per position; needs fix #1).
|
|
94
|
-
- `input_data` + `from_address` + `from_address_type` make it **content-addressed**
|
|
95
|
-
(tx-hash-like, collision-resistant). All are non-null columns already on the scheduled row.
|
|
96
|
-
|
|
97
|
-
### Alternatives considered
|
|
98
|
-
- **Reuse the seeded `Prando` (`new Prando(blockHash)`):** smallest change and deterministic,
|
|
99
|
-
but the shared `randomGenerator` is also consumed by the STFs — drawing hash bytes from it
|
|
100
|
-
would shift the RNG stream every later STF sees. Would require a *separate*
|
|
101
|
-
`new Prando(blockHash + ":txhash")`. Produces opaque (non-content) hashes.
|
|
102
|
-
- **`H(data.id, block_height)`:** `id` is a global SERIAL counter — a weaker determinism
|
|
103
|
-
guarantee than the block-scoped `index_in_block`, and `block_height` adds nothing once `id`
|
|
104
|
-
is unique. Prefer `index_in_block`.
|
|
105
|
-
- **Provenance: `H(origin_tx_hash, …)`:** ties to the originating real tx, but
|
|
106
|
-
`origin_tx_hash` is `null` for timestamp-scheduled inputs, so it needs a fallback.
|
|
107
|
-
|
|
108
|
-
## Verification
|
|
109
|
-
- Add a determinism test with a **real** STF (one that schedules a derived input) and assert
|
|
110
|
-
two from-scratch syncs produce identical `rollup_input_result.effectstream_tx_hash` —
|
|
111
|
-
i.e. drop `effectstream_tx_hash` from `VOLATILE_COLUMNS` once fixed and confirm
|
|
112
|
-
`consistency.test.ts` Test A still passes.
|
|
113
|
-
- Cover the timestamp-tie case explicitly: schedule ≥2 inputs at the same `future_ms_timestamp`
|
|
114
|
-
and assert identical ordering / hashes across runs.
|
package/src/api/apply-status.ts
DELETED
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Apply-stage liveness for /debug/metrics.
|
|
3
|
-
*
|
|
4
|
-
* The serial finalized-block loop in main.ts calls recordAppliedBlock() right
|
|
5
|
-
* after each block COMMITs; the /debug/metrics handler reads `appliedBlockStatus`
|
|
6
|
-
* to report true lag (now − applied block timestamp). Unlike a protocol's `buf`
|
|
7
|
-
* (fetch backlog), this stays high when the node is write/apply-bound.
|
|
8
|
-
*
|
|
9
|
-
* Process-global: one node runs per process, so a module-level singleton keeps
|
|
10
|
-
* main.ts to a single call instead of threading a ref through startHttpServer.
|
|
11
|
-
*/
|
|
12
|
-
export type AppliedBlockStatus = {
|
|
13
|
-
blockNumber: number | null;
|
|
14
|
-
timestamp: number | null;
|
|
15
|
-
appliedAtMs: number | null;
|
|
16
|
-
};
|
|
17
|
-
|
|
18
|
-
export const appliedBlockStatus: AppliedBlockStatus = {
|
|
19
|
-
blockNumber: null,
|
|
20
|
-
timestamp: null,
|
|
21
|
-
appliedAtMs: null,
|
|
22
|
-
};
|
|
23
|
-
|
|
24
|
-
export function recordAppliedBlock(
|
|
25
|
-
block: { blockNumber: number | bigint; timestamp: number | bigint },
|
|
26
|
-
): void {
|
|
27
|
-
appliedBlockStatus.blockNumber = Number(block.blockNumber);
|
|
28
|
-
appliedBlockStatus.timestamp = Number(block.timestamp);
|
|
29
|
-
appliedBlockStatus.appliedAtMs = Date.now();
|
|
30
|
-
}
|
|
@@ -1,389 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Verifies the fetch-backpressure feature (Option B′) over the synthetic `test`
|
|
3
|
-
* chain through the real runtime (`start()` + in-process PGLite).
|
|
4
|
-
*
|
|
5
|
-
* 1a — unbounded buffering: with a cap, the fetch loop pauses when the buffer is
|
|
6
|
-
* full, so it stays bounded (~cap + stepSize) instead of racing to the whole
|
|
7
|
-
* backlog — while the node keeps applying blocks (liveness).
|
|
8
|
-
* 1b — head-of-line blocking: a stalled slow chain STILL halts block production
|
|
9
|
-
* (the merge gate is correct), but the fast chain's buffer is now bounded by
|
|
10
|
-
* the cap instead of ballooning behind the head-of-line block.
|
|
11
|
-
*
|
|
12
|
-
* Before the fix these buffers reached ~49k–50k (see
|
|
13
|
-
* e2e/perf/results/BACKPRESSURE-BASELINE.md); these tests assert they now
|
|
14
|
-
* stay at the cap. Each test also writes a JSON time-series artifact to
|
|
15
|
-
* e2e/perf/results/ whose field names match e2e/perf/metrics.ts `Sample`, so the
|
|
16
|
-
* perf HTML report's Backpressure section can render the same signals.
|
|
17
|
-
*/
|
|
18
|
-
import { afterEach, beforeEach, expect, test } from "bun:test";
|
|
19
|
-
import {
|
|
20
|
-
ConfigBuilder,
|
|
21
|
-
ConfigNetworkType,
|
|
22
|
-
ConfigSyncProtocolType,
|
|
23
|
-
} from "@effectstream/config";
|
|
24
|
-
import { TestChainControl } from "@effectstream/sync";
|
|
25
|
-
import fs from "node:fs";
|
|
26
|
-
import path from "node:path";
|
|
27
|
-
import {
|
|
28
|
-
bufferSize,
|
|
29
|
-
type Harness,
|
|
30
|
-
latestFinalizedHeight,
|
|
31
|
-
type RunningNode,
|
|
32
|
-
setupHarness,
|
|
33
|
-
sleep,
|
|
34
|
-
TEST_PRIMITIVE_TYPE,
|
|
35
|
-
} from "./harness.ts";
|
|
36
|
-
|
|
37
|
-
const START_TIME = 1_700_000_000_000;
|
|
38
|
-
const BLOCK_TIME_MS = 1000;
|
|
39
|
-
const STEP_SIZE = 1000;
|
|
40
|
-
// A deep backlog the arithmetic fetcher would race through if uncapped (baseline
|
|
41
|
-
// hit ~49k–50k). The slow chain (1b) is pinned low so the merge stalls at its tip.
|
|
42
|
-
const BIG_TIP = 50_000;
|
|
43
|
-
const SLOW_TIP = 50;
|
|
44
|
-
// Explicit small cap so the bound is deterministic and far below the backlog.
|
|
45
|
-
// The fetch pauses at >= CAP; one in-flight chunk can overshoot by ~stepSize, so
|
|
46
|
-
// the buffer settles in [~CAP, CAP + stepSize]. We assert it lands in that band.
|
|
47
|
-
const CAP = 3000;
|
|
48
|
-
const CAP_FLOOR = CAP - STEP_SIZE; // proves backpressure actually engaged
|
|
49
|
-
const BUFFER_BOUND = CAP + STEP_SIZE + 200; // upper bound incl. one overshoot chunk
|
|
50
|
-
|
|
51
|
-
// e2e/perf/results lives at the repo root: reproduction → test → runtime →
|
|
52
|
-
// node-sdk → packages → <root>/e2e/perf/results.
|
|
53
|
-
const RESULTS_DIR = path.resolve(
|
|
54
|
-
import.meta.dirname!,
|
|
55
|
-
"../../../../../e2e/perf/results",
|
|
56
|
-
);
|
|
57
|
-
|
|
58
|
-
/** One time-series row; field names mirror e2e/perf/metrics.ts `Sample`. */
|
|
59
|
-
type BufSample = {
|
|
60
|
-
t: number;
|
|
61
|
-
wall: number;
|
|
62
|
-
mainBuf: number;
|
|
63
|
-
evmBuf: number; // the parallel chain's buffer (report labels it "parallel buf")
|
|
64
|
-
mainOwnBlock: number | null;
|
|
65
|
-
evmOwnBlock: number | null;
|
|
66
|
-
rss: number;
|
|
67
|
-
appliedBlock: number | null;
|
|
68
|
-
};
|
|
69
|
-
|
|
70
|
-
function ownBlock(node: RunningNode, name: string): number | null {
|
|
71
|
-
return node.syncProtocols().find((p) => p.name === name)?.lastPage
|
|
72
|
-
?.ownBlockNumber ?? null;
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
/** Backpressure "fired" counter (rising edges) exposed on SyncState / /debug/metrics. */
|
|
76
|
-
function pauses(node: RunningNode, name: string): number {
|
|
77
|
-
return node.syncProtocols().find((p) => p.name === name)
|
|
78
|
-
?.backpressurePauses ?? 0;
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
/** Side-effect-only artifact write; never throws into the test. */
|
|
82
|
-
function writeArtifact(
|
|
83
|
-
name: string,
|
|
84
|
-
samples: BufSample[],
|
|
85
|
-
meta: Record<string, unknown>,
|
|
86
|
-
): void {
|
|
87
|
-
try {
|
|
88
|
-
fs.mkdirSync(RESULTS_DIR, { recursive: true });
|
|
89
|
-
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
|
|
90
|
-
const file = path.join(RESULTS_DIR, `${name}-${stamp}.json`);
|
|
91
|
-
fs.writeFileSync(file, JSON.stringify({ meta, samples }, null, 2));
|
|
92
|
-
console.log(`buffering artifact: ${file}`);
|
|
93
|
-
} catch (e) {
|
|
94
|
-
console.warn(`failed to write buffering artifact: ${String(e)}`);
|
|
95
|
-
}
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
let h: Harness;
|
|
99
|
-
beforeEach(async () => {
|
|
100
|
-
h = await setupHarness();
|
|
101
|
-
});
|
|
102
|
-
afterEach(async () => {
|
|
103
|
-
await h?.teardown();
|
|
104
|
-
});
|
|
105
|
-
|
|
106
|
-
// ── 1a — unbounded buffering, now bounded by the cap ────────────────────────────
|
|
107
|
-
|
|
108
|
-
const MAIN_1A = "mainClock1a";
|
|
109
|
-
const PAR_1A = "parallelP1a";
|
|
110
|
-
|
|
111
|
-
function buildConfig1a() {
|
|
112
|
-
return new ConfigBuilder()
|
|
113
|
-
.setNamespace((b) => b.setSecurityNamespace("sync-buffering-1a"))
|
|
114
|
-
.buildNetworks((b) =>
|
|
115
|
-
b
|
|
116
|
-
.addNetwork({
|
|
117
|
-
name: "clock",
|
|
118
|
-
type: ConfigNetworkType.TEST,
|
|
119
|
-
startTime: START_TIME,
|
|
120
|
-
blockTimeMS: BLOCK_TIME_MS,
|
|
121
|
-
})
|
|
122
|
-
.addNetwork({
|
|
123
|
-
name: "chainP",
|
|
124
|
-
type: ConfigNetworkType.TEST,
|
|
125
|
-
startTime: START_TIME,
|
|
126
|
-
blockTimeMS: BLOCK_TIME_MS,
|
|
127
|
-
})
|
|
128
|
-
)
|
|
129
|
-
.buildDeployments((b) => b)
|
|
130
|
-
.buildSyncProtocols((b) =>
|
|
131
|
-
b
|
|
132
|
-
.addMain(
|
|
133
|
-
(n) => n.clock,
|
|
134
|
-
() => ({
|
|
135
|
-
name: MAIN_1A,
|
|
136
|
-
type: ConfigSyncProtocolType.TEST_MAIN,
|
|
137
|
-
startBlockHeight: 1,
|
|
138
|
-
pollingInterval: 30,
|
|
139
|
-
stepSize: STEP_SIZE,
|
|
140
|
-
maxBufferedPages: CAP,
|
|
141
|
-
}),
|
|
142
|
-
)
|
|
143
|
-
.addParallel(
|
|
144
|
-
(n) => (n as any).chainP,
|
|
145
|
-
() => ({
|
|
146
|
-
name: PAR_1A,
|
|
147
|
-
type: ConfigSyncProtocolType.TEST_PARALLEL,
|
|
148
|
-
startBlockHeight: 1,
|
|
149
|
-
pollingInterval: 30,
|
|
150
|
-
stepSize: STEP_SIZE,
|
|
151
|
-
delayMs: 0,
|
|
152
|
-
events: [],
|
|
153
|
-
maxBufferedPages: CAP,
|
|
154
|
-
}),
|
|
155
|
-
)
|
|
156
|
-
)
|
|
157
|
-
.buildPrimitives((b) =>
|
|
158
|
-
b.addPrimitive(
|
|
159
|
-
(sp) => (sp as any)[PAR_1A],
|
|
160
|
-
() => ({ name: "noEvt1a", type: TEST_PRIMITIVE_TYPE, startBlockHeight: 1 }),
|
|
161
|
-
)
|
|
162
|
-
)
|
|
163
|
-
.build();
|
|
164
|
-
}
|
|
165
|
-
|
|
166
|
-
test("1a: backpressure caps the buffer while the node keeps applying blocks", async () => {
|
|
167
|
-
// Tips far ahead of where the one-block-per-txn merge can drain to: without a
|
|
168
|
-
// cap the fetcher would race to ~50k; with the cap it must plateau near CAP.
|
|
169
|
-
TestChainControl.setTip(MAIN_1A, BIG_TIP);
|
|
170
|
-
TestChainControl.setTip(PAR_1A, BIG_TIP);
|
|
171
|
-
|
|
172
|
-
const node = await h.runNode({ config: buildConfig1a(), apiPort: 19141 });
|
|
173
|
-
const samples: BufSample[] = [];
|
|
174
|
-
let peakMain = 0;
|
|
175
|
-
let peakEvm = 0;
|
|
176
|
-
let lastApplied = 0;
|
|
177
|
-
let mainPauses = 0;
|
|
178
|
-
try {
|
|
179
|
-
const startMs = Date.now();
|
|
180
|
-
const deadline = startMs + 6000;
|
|
181
|
-
while (Date.now() < deadline) {
|
|
182
|
-
const wall = Date.now();
|
|
183
|
-
const mainBuf = bufferSize(node, MAIN_1A);
|
|
184
|
-
const evmBuf = bufferSize(node, PAR_1A);
|
|
185
|
-
const appliedBlock = await latestFinalizedHeight(h.pool);
|
|
186
|
-
samples.push({
|
|
187
|
-
t: wall - startMs,
|
|
188
|
-
wall,
|
|
189
|
-
mainBuf,
|
|
190
|
-
evmBuf,
|
|
191
|
-
mainOwnBlock: ownBlock(node, MAIN_1A),
|
|
192
|
-
evmOwnBlock: ownBlock(node, PAR_1A),
|
|
193
|
-
rss: process.memoryUsage().rss,
|
|
194
|
-
appliedBlock,
|
|
195
|
-
});
|
|
196
|
-
peakMain = Math.max(peakMain, mainBuf);
|
|
197
|
-
peakEvm = Math.max(peakEvm, evmBuf);
|
|
198
|
-
lastApplied = appliedBlock ?? lastApplied;
|
|
199
|
-
// Backpressure engaged (buffer hit the cap) AND the node is still applying
|
|
200
|
-
// blocks → we've captured the bounded steady state; stop early.
|
|
201
|
-
if (peakMain >= CAP && lastApplied >= 20) break;
|
|
202
|
-
await sleep(50);
|
|
203
|
-
}
|
|
204
|
-
} finally {
|
|
205
|
-
mainPauses = pauses(node, MAIN_1A);
|
|
206
|
-
writeArtifact("buffering-1a", samples, {
|
|
207
|
-
mode: "1a-bounded-by-cap",
|
|
208
|
-
stepSize: STEP_SIZE,
|
|
209
|
-
bigTip: BIG_TIP,
|
|
210
|
-
cap: CAP,
|
|
211
|
-
bufferBound: BUFFER_BOUND,
|
|
212
|
-
peakMainBuf: peakMain,
|
|
213
|
-
peakEvmBuf: peakEvm,
|
|
214
|
-
lastApplied,
|
|
215
|
-
pauses: mainPauses,
|
|
216
|
-
});
|
|
217
|
-
await node.stop();
|
|
218
|
-
}
|
|
219
|
-
|
|
220
|
-
// Backpressure engaged: the buffer reached the cap (not starved)...
|
|
221
|
-
expect(peakMain).toBeGreaterThanOrEqual(CAP_FLOOR);
|
|
222
|
-
// ...but stayed bounded — nowhere near the uncapped ~49k backlog.
|
|
223
|
-
expect(Math.max(peakMain, peakEvm)).toBeLessThanOrEqual(BUFFER_BOUND);
|
|
224
|
-
// ...the node kept producing/applying blocks (no deadlock)...
|
|
225
|
-
expect(lastApplied).toBeGreaterThanOrEqual(20);
|
|
226
|
-
// ...and the cap actually fired (the observability metric populated).
|
|
227
|
-
expect(mainPauses).toBeGreaterThan(0);
|
|
228
|
-
}, 60_000);
|
|
229
|
-
|
|
230
|
-
// ── 1b — head-of-line blocking, sibling buffer now bounded ──────────────────────
|
|
231
|
-
|
|
232
|
-
const MAIN_1B = "mainClock1b";
|
|
233
|
-
const FAST_1B = "fastP1b";
|
|
234
|
-
const SLOW_1B = "slowP1b";
|
|
235
|
-
|
|
236
|
-
function buildConfig1b() {
|
|
237
|
-
return new ConfigBuilder()
|
|
238
|
-
.setNamespace((b) => b.setSecurityNamespace("sync-buffering-1b"))
|
|
239
|
-
.buildNetworks((b) =>
|
|
240
|
-
b
|
|
241
|
-
.addNetwork({
|
|
242
|
-
name: "clock",
|
|
243
|
-
type: ConfigNetworkType.TEST,
|
|
244
|
-
startTime: START_TIME,
|
|
245
|
-
blockTimeMS: BLOCK_TIME_MS,
|
|
246
|
-
})
|
|
247
|
-
.addNetwork({
|
|
248
|
-
name: "chainFast",
|
|
249
|
-
type: ConfigNetworkType.TEST,
|
|
250
|
-
startTime: START_TIME,
|
|
251
|
-
blockTimeMS: BLOCK_TIME_MS,
|
|
252
|
-
})
|
|
253
|
-
.addNetwork({
|
|
254
|
-
name: "chainSlow",
|
|
255
|
-
type: ConfigNetworkType.TEST,
|
|
256
|
-
startTime: START_TIME,
|
|
257
|
-
blockTimeMS: BLOCK_TIME_MS,
|
|
258
|
-
})
|
|
259
|
-
)
|
|
260
|
-
.buildDeployments((b) => b)
|
|
261
|
-
.buildSyncProtocols((b) =>
|
|
262
|
-
b
|
|
263
|
-
.addMain(
|
|
264
|
-
(n) => n.clock,
|
|
265
|
-
() => ({
|
|
266
|
-
name: MAIN_1B,
|
|
267
|
-
type: ConfigSyncProtocolType.TEST_MAIN,
|
|
268
|
-
startBlockHeight: 1,
|
|
269
|
-
pollingInterval: 30,
|
|
270
|
-
stepSize: STEP_SIZE,
|
|
271
|
-
maxBufferedPages: CAP,
|
|
272
|
-
}),
|
|
273
|
-
)
|
|
274
|
-
.addParallel(
|
|
275
|
-
(n) => (n as any).chainFast,
|
|
276
|
-
() => ({
|
|
277
|
-
name: FAST_1B,
|
|
278
|
-
type: ConfigSyncProtocolType.TEST_PARALLEL,
|
|
279
|
-
startBlockHeight: 1,
|
|
280
|
-
pollingInterval: 30,
|
|
281
|
-
stepSize: STEP_SIZE,
|
|
282
|
-
delayMs: 0,
|
|
283
|
-
events: [],
|
|
284
|
-
maxBufferedPages: CAP,
|
|
285
|
-
}),
|
|
286
|
-
)
|
|
287
|
-
.addParallel(
|
|
288
|
-
(n) => (n as any).chainSlow,
|
|
289
|
-
() => ({
|
|
290
|
-
name: SLOW_1B,
|
|
291
|
-
type: ConfigSyncProtocolType.TEST_PARALLEL,
|
|
292
|
-
startBlockHeight: 1,
|
|
293
|
-
pollingInterval: 30,
|
|
294
|
-
stepSize: STEP_SIZE,
|
|
295
|
-
delayMs: 0,
|
|
296
|
-
events: [],
|
|
297
|
-
maxBufferedPages: CAP,
|
|
298
|
-
}),
|
|
299
|
-
)
|
|
300
|
-
)
|
|
301
|
-
.buildPrimitives((b) =>
|
|
302
|
-
b
|
|
303
|
-
.addPrimitive(
|
|
304
|
-
(sp) => (sp as any)[FAST_1B],
|
|
305
|
-
() => ({ name: "noEvtFast", type: TEST_PRIMITIVE_TYPE, startBlockHeight: 1 }),
|
|
306
|
-
)
|
|
307
|
-
.addPrimitive(
|
|
308
|
-
(sp) => (sp as any)[SLOW_1B],
|
|
309
|
-
() => ({ name: "noEvtSlow", type: TEST_PRIMITIVE_TYPE, startBlockHeight: 1 }),
|
|
310
|
-
)
|
|
311
|
-
)
|
|
312
|
-
.build();
|
|
313
|
-
}
|
|
314
|
-
|
|
315
|
-
test("1b: a stalled chain still halts production, but the fast chain's buffer stays capped", async () => {
|
|
316
|
-
// slowP pinned at a low tip → the merge stalls there forever (correct). fastP and
|
|
317
|
-
// the main clock keep fetching toward their high tips, but the cap now bounds them.
|
|
318
|
-
TestChainControl.setTip(MAIN_1B, BIG_TIP);
|
|
319
|
-
TestChainControl.setTip(FAST_1B, BIG_TIP);
|
|
320
|
-
TestChainControl.setTip(SLOW_1B, SLOW_TIP);
|
|
321
|
-
|
|
322
|
-
const node = await h.runNode({ config: buildConfig1b(), apiPort: 19142 });
|
|
323
|
-
const samples: BufSample[] = [];
|
|
324
|
-
let peakMain = 0;
|
|
325
|
-
let peakFast = 0;
|
|
326
|
-
let stalledHeight = -1;
|
|
327
|
-
let fastPauses = 0;
|
|
328
|
-
try {
|
|
329
|
-
const startMs = Date.now();
|
|
330
|
-
const deadline = startMs + 15_000;
|
|
331
|
-
let lastHeight = -1;
|
|
332
|
-
let lastChange = Date.now();
|
|
333
|
-
while (Date.now() < deadline) {
|
|
334
|
-
const wall = Date.now();
|
|
335
|
-
const mainBuf = bufferSize(node, MAIN_1B);
|
|
336
|
-
const fastBuf = bufferSize(node, FAST_1B);
|
|
337
|
-
const appliedBlock = await latestFinalizedHeight(h.pool);
|
|
338
|
-
samples.push({
|
|
339
|
-
t: wall - startMs,
|
|
340
|
-
wall,
|
|
341
|
-
mainBuf,
|
|
342
|
-
evmBuf: fastBuf,
|
|
343
|
-
mainOwnBlock: ownBlock(node, MAIN_1B),
|
|
344
|
-
evmOwnBlock: ownBlock(node, FAST_1B),
|
|
345
|
-
rss: process.memoryUsage().rss,
|
|
346
|
-
appliedBlock,
|
|
347
|
-
});
|
|
348
|
-
peakMain = Math.max(peakMain, mainBuf);
|
|
349
|
-
peakFast = Math.max(peakFast, fastBuf);
|
|
350
|
-
if (appliedBlock !== lastHeight) {
|
|
351
|
-
lastHeight = appliedBlock;
|
|
352
|
-
lastChange = wall;
|
|
353
|
-
}
|
|
354
|
-
// Production has stalled (height flat ≥1.5s) AND the fast chain has hit its
|
|
355
|
-
// cap → the head-of-line block holds while the sibling buffer is bounded.
|
|
356
|
-
if (wall - lastChange >= 1500 && peakFast >= CAP_FLOOR) {
|
|
357
|
-
stalledHeight = appliedBlock ?? -1;
|
|
358
|
-
break;
|
|
359
|
-
}
|
|
360
|
-
await sleep(50);
|
|
361
|
-
}
|
|
362
|
-
} finally {
|
|
363
|
-
fastPauses = pauses(node, FAST_1B);
|
|
364
|
-
writeArtifact("buffering-1b", samples, {
|
|
365
|
-
mode: "1b-hol-sibling-bounded",
|
|
366
|
-
stepSize: STEP_SIZE,
|
|
367
|
-
bigTip: BIG_TIP,
|
|
368
|
-
slowTip: SLOW_TIP,
|
|
369
|
-
cap: CAP,
|
|
370
|
-
bufferBound: BUFFER_BOUND,
|
|
371
|
-
stalledHeight,
|
|
372
|
-
peakMainBuf: peakMain,
|
|
373
|
-
peakFastBuf: peakFast,
|
|
374
|
-
fastPauses,
|
|
375
|
-
});
|
|
376
|
-
await node.stop();
|
|
377
|
-
}
|
|
378
|
-
|
|
379
|
-
// Block production still stalls at ~the slow chain's tip (the merge gate is correct)...
|
|
380
|
-
expect(stalledHeight).toBeGreaterThanOrEqual(0);
|
|
381
|
-
expect(stalledHeight).toBeLessThanOrEqual(SLOW_TIP + 10);
|
|
382
|
-
// ...the fast chain's buffer engaged the cap...
|
|
383
|
-
expect(peakFast).toBeGreaterThanOrEqual(CAP_FLOOR);
|
|
384
|
-
// ...but stayed bounded behind the head-of-line block (was ~50k uncapped)...
|
|
385
|
-
expect(peakFast).toBeLessThanOrEqual(BUFFER_BOUND);
|
|
386
|
-
expect(peakMain).toBeLessThanOrEqual(BUFFER_BOUND);
|
|
387
|
-
// ...and the cap actually fired on the fast chain (observability metric populated).
|
|
388
|
-
expect(fastPauses).toBeGreaterThan(0);
|
|
389
|
-
}, 60_000);
|