@effectstream/runtime 0.100.20 → 0.100.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -34
- package/package.json +13 -13
- package/src/api/http-server.ts +7 -55
- package/src/main.ts +39 -64
- package/src/process-blocks.ts +27 -126
- package/src/types.ts +0 -7
- package/test/reproduction/harness.ts +270 -559
- package/test/reproduction/sanity.test.ts +91 -40
- package/conciseinputhash-determinism.md +0 -114
- package/src/api/apply-status.ts +0 -30
- package/src/coalesce.ts +0 -291
- package/test/coalesce.unit.test.ts +0 -69
- package/test/reproduction/buffering.test.ts +0 -389
- package/test/reproduction/coalesce.test.ts +0 -133
- package/test/reproduction/consistency-snapshot.ts +0 -168
- package/test/reproduction/consistency.test.ts +0 -197
- package/test/reproduction/node-runner.ts +0 -154
- package/test/reproduction/poll.ts +0 -160
- package/test/reproduction/resume.test.ts +0 -77
- package/test/reproduction/scenario.ts +0 -126
|
@@ -2,26 +2,90 @@
|
|
|
2
2
|
* Sanity end-to-end test for the synthetic `test` chain (no fixes required).
|
|
3
3
|
*
|
|
4
4
|
* Proves the foundation this PR adds works: a TEST_MAIN clock + a TEST_PARALLEL
|
|
5
|
-
* chain sync through the real runtime (`start()`), a config-declared
|
|
6
|
-
* processed into `effectstream.primitive_accounting`, and committed
|
|
7
|
-
* survives a restart.
|
|
8
|
-
*
|
|
9
|
-
* Runs on whichever backend `setupHarness` picks: the default in-memory PGLite
|
|
10
|
-
* (no external dependency) or real Postgres when `PGLITE=false`. Either way
|
|
11
|
-
* each node runs in its own subprocess (see harness.ts / node-runner.ts), so a
|
|
12
|
-
* restart is a genuine new OS process booting against the same database.
|
|
5
|
+
* chain sync through the real runtime (`start()` + PGLite), a config-declared
|
|
6
|
+
* event is processed into `effectstream.primitive_accounting`, and committed
|
|
7
|
+
* state survives a restart.
|
|
13
8
|
*
|
|
14
9
|
* (The buffering / resume / fast-catchup reproduction tests land alongside their
|
|
15
10
|
* fixes in follow-up PRs — they assert behaviour that only those fixes provide.)
|
|
16
11
|
*/
|
|
17
12
|
import { afterAll, beforeAll, expect, test } from "bun:test";
|
|
13
|
+
import {
|
|
14
|
+
ConfigBuilder,
|
|
15
|
+
ConfigNetworkType,
|
|
16
|
+
ConfigSyncProtocolType,
|
|
17
|
+
} from "@effectstream/config";
|
|
18
|
+
import { TestChainControl } from "@effectstream/sync";
|
|
18
19
|
import {
|
|
19
20
|
countEventRows,
|
|
20
21
|
type Harness,
|
|
21
22
|
latestFinalizedHeight,
|
|
22
23
|
setupHarness,
|
|
24
|
+
TEST_PRIMITIVE_TYPE,
|
|
25
|
+
waitForHeight,
|
|
26
|
+
waitUntilStable,
|
|
23
27
|
} from "./harness.ts";
|
|
24
28
|
|
|
29
|
+
const START_TIME = 1_700_000_000_000;
|
|
30
|
+
const BLOCK_TIME_MS = 1000;
|
|
31
|
+
|
|
32
|
+
function buildConfig() {
|
|
33
|
+
return new ConfigBuilder()
|
|
34
|
+
.setNamespace((b) => b.setSecurityNamespace("sync-sanity"))
|
|
35
|
+
.buildNetworks((b) =>
|
|
36
|
+
b
|
|
37
|
+
.addNetwork({
|
|
38
|
+
name: "clock",
|
|
39
|
+
type: ConfigNetworkType.TEST,
|
|
40
|
+
startTime: START_TIME,
|
|
41
|
+
blockTimeMS: BLOCK_TIME_MS,
|
|
42
|
+
})
|
|
43
|
+
.addNetwork({
|
|
44
|
+
name: "chainP",
|
|
45
|
+
type: ConfigNetworkType.TEST,
|
|
46
|
+
startTime: START_TIME,
|
|
47
|
+
blockTimeMS: BLOCK_TIME_MS,
|
|
48
|
+
})
|
|
49
|
+
)
|
|
50
|
+
.buildDeployments((b) => b)
|
|
51
|
+
.buildSyncProtocols((b) =>
|
|
52
|
+
b
|
|
53
|
+
.addMain(
|
|
54
|
+
(n) => n.clock,
|
|
55
|
+
() => ({
|
|
56
|
+
name: "mainClock",
|
|
57
|
+
type: ConfigSyncProtocolType.TEST_MAIN,
|
|
58
|
+
startBlockHeight: 1,
|
|
59
|
+
pollingInterval: 30,
|
|
60
|
+
stepSize: 1000,
|
|
61
|
+
}),
|
|
62
|
+
)
|
|
63
|
+
.addParallel(
|
|
64
|
+
(n) => (n as any).chainP,
|
|
65
|
+
() => ({
|
|
66
|
+
name: "parallelP",
|
|
67
|
+
type: ConfigSyncProtocolType.TEST_PARALLEL,
|
|
68
|
+
startBlockHeight: 1,
|
|
69
|
+
pollingInterval: 30,
|
|
70
|
+
stepSize: 1000,
|
|
71
|
+
delayMs: 0,
|
|
72
|
+
events: [{ atBlock: 50, payload: { marker: "evt50" } }],
|
|
73
|
+
}),
|
|
74
|
+
)
|
|
75
|
+
)
|
|
76
|
+
.buildPrimitives((b) =>
|
|
77
|
+
b.addPrimitive(
|
|
78
|
+
(sp) => (sp as any).parallelP,
|
|
79
|
+
() => ({
|
|
80
|
+
name: "testEvt",
|
|
81
|
+
type: TEST_PRIMITIVE_TYPE,
|
|
82
|
+
startBlockHeight: 1,
|
|
83
|
+
}),
|
|
84
|
+
)
|
|
85
|
+
)
|
|
86
|
+
.build();
|
|
87
|
+
}
|
|
88
|
+
|
|
25
89
|
let h: Harness;
|
|
26
90
|
|
|
27
91
|
beforeAll(async () => {
|
|
@@ -33,42 +97,29 @@ afterAll(async () => {
|
|
|
33
97
|
});
|
|
34
98
|
|
|
35
99
|
test("test chain syncs, processes a configured event, and survives restart", async () => {
|
|
36
|
-
const events = [{ atBlock: 50, marker: "evt50" }];
|
|
37
|
-
|
|
38
100
|
// Parallel tip (200) stays ahead of the main clock (100) so the merge is not
|
|
39
101
|
// gated at the chunk boundary.
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
102
|
+
TestChainControl.setTip("mainClock", 100);
|
|
103
|
+
TestChainControl.setTip("parallelP", 200);
|
|
104
|
+
|
|
105
|
+
const n1 = await h.runNode({ config: buildConfig(), apiPort: 19131 });
|
|
106
|
+
await waitForHeight(n1, h.pool, 100);
|
|
107
|
+
await waitUntilStable(n1, h.pool);
|
|
46
108
|
|
|
47
|
-
|
|
48
|
-
const events1 = await countEventRows(h.pool, "evt50");
|
|
49
|
-
console.log(
|
|
50
|
-
`[sanity] after sync: height ${height1} (expect 100), ` +
|
|
51
|
-
`evt50 rows ${events1} (expect 1)`,
|
|
52
|
-
);
|
|
53
|
-
expect(height1).toBe(100);
|
|
109
|
+
expect(await latestFinalizedHeight(h.pool)).toBe(100);
|
|
54
110
|
// The event declared at block 50 was processed into primitive_accounting.
|
|
55
|
-
expect(
|
|
111
|
+
expect(await countEventRows(h.pool, "evt50")).toBe(1);
|
|
112
|
+
await n1.stop();
|
|
113
|
+
|
|
114
|
+
// Restart: committed state must persist (chain already fully caught up).
|
|
115
|
+
await h.simulateRestart();
|
|
116
|
+
TestChainControl.setTip("mainClock", 100);
|
|
117
|
+
TestChainControl.setTip("parallelP", 200);
|
|
56
118
|
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
await h.runToHeight({
|
|
60
|
-
events,
|
|
61
|
-
tips: { mainClock: 100, parallelP: 200 },
|
|
62
|
-
target: 100,
|
|
63
|
-
apiPort: 19132,
|
|
64
|
-
});
|
|
119
|
+
const n2 = await h.runNode({ config: buildConfig(), apiPort: 19132 });
|
|
120
|
+
await waitUntilStable(n2, h.pool);
|
|
65
121
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
`[sanity] after restart: height ${height2} (expect 100, was ${height1}), ` +
|
|
70
|
-
`evt50 rows ${events2} (expect 1, was ${events1})`,
|
|
71
|
-
);
|
|
72
|
-
expect(height2).toBe(100);
|
|
73
|
-
expect(events2).toBe(1);
|
|
122
|
+
expect(await latestFinalizedHeight(h.pool)).toBe(100);
|
|
123
|
+
expect(await countEventRows(h.pool, "evt50")).toBe(1);
|
|
124
|
+
await n2.stop();
|
|
74
125
|
}, 60_000);
|
|
@@ -1,114 +0,0 @@
|
|
|
1
|
-
# Bug Report: non-deterministic `effectstream_tx_hash` (`conciseInputHash`)
|
|
2
|
-
|
|
3
|
-
**Package:** `@effectstream/node-sdk` — runtime
|
|
4
|
-
**Location:** `runtime/src/process-blocks.ts` (scheduled-input loop, STEP 5)
|
|
5
|
-
**Severity:** Medium — silent cross-node state divergence in a derived column; not (currently) consensus-relevant
|
|
6
|
-
**Status:** Open
|
|
7
|
-
|
|
8
|
-
---
|
|
9
|
-
|
|
10
|
-
## Summary
|
|
11
|
-
|
|
12
|
-
When the runtime records the result of a **scheduled** STF input, it fabricates the
|
|
13
|
-
`effectstream_tx_hash` with `Math.random()`. The value is therefore different on every run
|
|
14
|
-
and on every node, so two nodes that replay the exact same source blocks produce **different
|
|
15
|
-
`rollup_input_result.effectstream_tx_hash` values** for the same input. For a framework whose
|
|
16
|
-
core promise is "the database is a pure function of the source blocks," this is a determinism
|
|
17
|
-
hole.
|
|
18
|
-
|
|
19
|
-
## The code
|
|
20
|
-
|
|
21
|
-
`runtime/src/process-blocks.ts`, inside the loop over `scheduledData` (STEP 5):
|
|
22
|
-
|
|
23
|
-
```ts
|
|
24
|
-
const conciseInputHash = `0x${
|
|
25
|
-
Array(64).fill(0).map(() => Math.floor(Math.random() * 16).toString(16)).join("")
|
|
26
|
-
}`;
|
|
27
|
-
yield* until(
|
|
28
|
-
insertGameInputResult.run({
|
|
29
|
-
id: data.id,
|
|
30
|
-
success,
|
|
31
|
-
effectstream_tx_hash: Buffer.from(conciseInputHash),
|
|
32
|
-
index_in_block,
|
|
33
|
-
block_height: value.blockNumber,
|
|
34
|
-
}, dbConn),
|
|
35
|
-
);
|
|
36
|
-
```
|
|
37
|
-
|
|
38
|
-
This runs only for **scheduled** inputs (engine-queued timers / derived inputs that have no
|
|
39
|
-
real on-chain transaction), so the engine has to synthesize a hash. It does so randomly.
|
|
40
|
-
|
|
41
|
-
## Why it matters / why it's currently masked
|
|
42
|
-
|
|
43
|
-
- **Block hash is unaffected.** `generateEffectstreamBlockHash` (`@effectstream/crypto`) hashes
|
|
44
|
-
`blockInfo[].blockHash` (source-chain hashes) + the previous block hash — never
|
|
45
|
-
`effectstream_tx_hash`. So the block-hash chain stays deterministic; the divergence is
|
|
46
|
-
confined to the `rollup_input_result.effectstream_tx_hash` column and anything that reads it.
|
|
47
|
-
- **Masked in tests.** The reproduction tests use a no-op STF (`stateMachinePayload: null`),
|
|
48
|
-
which schedules no STF inputs, so this branch never executes. That is exactly why
|
|
49
|
-
`consistency-snapshot.ts` lists `effectstream_tx_hash` in `VOLATILE_COLUMNS` and excludes it
|
|
50
|
-
— the snapshot would otherwise diff on every run once real STFs are involved.
|
|
51
|
-
|
|
52
|
-
## Two existing quirks to fix alongside
|
|
53
|
-
|
|
54
|
-
1. **Encoding bug.** `Buffer.from(conciseInputHash)` is given the *string* `"0x" + 64 hex`
|
|
55
|
-
with no encoding, so it stores 66 **UTF-8** bytes, not the 32 decoded bytes. Whatever
|
|
56
|
-
derivation replaces `Math.random()`, encode it deliberately (`Buffer.from(hex, "hex")`, or
|
|
57
|
-
commit to the `0x`-string form consistently).
|
|
58
|
-
|
|
59
|
-
2. **Ordering prerequisite (the real root).** The hash inputs that give uniqueness
|
|
60
|
-
(`index_in_block`, and the STF execution order itself) are only deterministic if the
|
|
61
|
-
`scheduledData` order is deterministic. Today it is not fully:
|
|
62
|
-
```ts
|
|
63
|
-
// TODO What should be the order of the scheduled data - per id?
|
|
64
|
-
const scheduledData = [...scheduledData1, ...scheduledData2];
|
|
65
|
-
```
|
|
66
|
-
- `scheduledData1` (block-scheduled) is `ORDER BY id ASC` → deterministic given a
|
|
67
|
-
deterministic scheduling history.
|
|
68
|
-
- `scheduledData2` (timestamp-scheduled) is `ORDER BY future_ms_timestamp ASC` **only** —
|
|
69
|
-
**same-timestamp rows have no tiebreak**, so their order is arbitrary.
|
|
70
|
-
|
|
71
|
-
This is bigger than the hash: the same order drives **STF execution** and the
|
|
72
|
-
**`randomGenerator` (Prando) draw sequence**. If it isn't pinned, game state itself can
|
|
73
|
-
diverge; the random tx-hash is just the most visible symptom.
|
|
74
|
-
|
|
75
|
-
## Suggested fixes (in order)
|
|
76
|
-
|
|
77
|
-
### 1. Pin the scheduled-input order (prerequisite, do first)
|
|
78
|
-
Give the queries a total deterministic sort and a deterministic merge:
|
|
79
|
-
- timestamp list → `ORDER BY future_ms_timestamp ASC, id ASC` (break ties by `id`);
|
|
80
|
-
- keep block-scheduled before timestamp-scheduled (or merge both by a single deterministic
|
|
81
|
-
key). This is required for deterministic STFs regardless of the hash, and it makes
|
|
82
|
-
`index_in_block` deterministic.
|
|
83
|
-
|
|
84
|
-
### 2. Derive the hash deterministically (recommended)
|
|
85
|
-
Replace `Math.random()` with a content-addressed hash over the input's identity + position,
|
|
86
|
-
reusing the `sha512` already used for the block hash:
|
|
87
|
-
|
|
88
|
-
```
|
|
89
|
-
effectstream_tx_hash =
|
|
90
|
-
sha512(block_height ‖ index_in_block ‖ input_data ‖ from_address ‖ from_address_type)
|
|
91
|
-
```
|
|
92
|
-
|
|
93
|
-
- `block_height` + `index_in_block` give **uniqueness** (one result per position; needs fix #1).
|
|
94
|
-
- `input_data` + `from_address` + `from_address_type` make it **content-addressed**
|
|
95
|
-
(tx-hash-like, collision-resistant). All are non-null columns already on the scheduled row.
|
|
96
|
-
|
|
97
|
-
### Alternatives considered
|
|
98
|
-
- **Reuse the seeded `Prando` (`new Prando(blockHash)`):** smallest change and deterministic,
|
|
99
|
-
but the shared `randomGenerator` is also consumed by the STFs — drawing hash bytes from it
|
|
100
|
-
would shift the RNG stream every later STF sees. Would require a *separate*
|
|
101
|
-
`new Prando(blockHash + ":txhash")`. Produces opaque (non-content) hashes.
|
|
102
|
-
- **`H(data.id, block_height)`:** `id` is a global SERIAL counter — a weaker determinism
|
|
103
|
-
guarantee than the block-scoped `index_in_block`, and `block_height` adds nothing once `id`
|
|
104
|
-
is unique. Prefer `index_in_block`.
|
|
105
|
-
- **Provenance: `H(origin_tx_hash, …)`:** ties to the originating real tx, but
|
|
106
|
-
`origin_tx_hash` is `null` for timestamp-scheduled inputs, so it needs a fallback.
|
|
107
|
-
|
|
108
|
-
## Verification
|
|
109
|
-
- Add a determinism test with a **real** STF (one that schedules a derived input) and assert
|
|
110
|
-
two from-scratch syncs produce identical `rollup_input_result.effectstream_tx_hash` —
|
|
111
|
-
i.e. drop `effectstream_tx_hash` from `VOLATILE_COLUMNS` once fixed and confirm
|
|
112
|
-
`consistency.test.ts` Test A still passes.
|
|
113
|
-
- Cover the timestamp-tie case explicitly: schedule ≥2 inputs at the same `future_ms_timestamp`
|
|
114
|
-
and assert identical ordering / hashes across runs.
|
package/src/api/apply-status.ts
DELETED
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Apply-stage liveness for /debug/metrics.
|
|
3
|
-
*
|
|
4
|
-
* The serial finalized-block loop in main.ts calls recordAppliedBlock() right
|
|
5
|
-
* after each block COMMITs; the /debug/metrics handler reads `appliedBlockStatus`
|
|
6
|
-
* to report true lag (now − applied block timestamp). Unlike a protocol's `buf`
|
|
7
|
-
* (fetch backlog), this stays high when the node is write/apply-bound.
|
|
8
|
-
*
|
|
9
|
-
* Process-global: one node runs per process, so a module-level singleton keeps
|
|
10
|
-
* main.ts to a single call instead of threading a ref through startHttpServer.
|
|
11
|
-
*/
|
|
12
|
-
export type AppliedBlockStatus = {
|
|
13
|
-
blockNumber: number | null;
|
|
14
|
-
timestamp: number | null;
|
|
15
|
-
appliedAtMs: number | null;
|
|
16
|
-
};
|
|
17
|
-
|
|
18
|
-
export const appliedBlockStatus: AppliedBlockStatus = {
|
|
19
|
-
blockNumber: null,
|
|
20
|
-
timestamp: null,
|
|
21
|
-
appliedAtMs: null,
|
|
22
|
-
};
|
|
23
|
-
|
|
24
|
-
export function recordAppliedBlock(
|
|
25
|
-
block: { blockNumber: number | bigint; timestamp: number | bigint },
|
|
26
|
-
): void {
|
|
27
|
-
appliedBlockStatus.blockNumber = Number(block.blockNumber);
|
|
28
|
-
appliedBlockStatus.timestamp = Number(block.timestamp);
|
|
29
|
-
appliedBlockStatus.appliedAtMs = Date.now();
|
|
30
|
-
}
|
package/src/coalesce.ts
DELETED
|
@@ -1,291 +0,0 @@
|
|
|
1
|
-
/** Empty-block coalescing during deep catch-up: fold consecutive no-op blocks into one transaction. */
|
|
2
|
-
import type { ChainBlock } from "@effectstream/sync";
|
|
3
|
-
import { call, type Operation, race, sleep, until } from "effection";
|
|
4
|
-
import type { Pool, PoolClient } from "pg";
|
|
5
|
-
import {
|
|
6
|
-
acquireDBMutex,
|
|
7
|
-
blockHeightDone,
|
|
8
|
-
deleteEmptyBlocks,
|
|
9
|
-
getEarliestScheduledBlockHeight,
|
|
10
|
-
getEarliestScheduledTimestamp,
|
|
11
|
-
getMigrationsForBlockHeight,
|
|
12
|
-
isTransientPgError,
|
|
13
|
-
poolErrors,
|
|
14
|
-
releaseDBMutex,
|
|
15
|
-
removePages,
|
|
16
|
-
saveLastBlock,
|
|
17
|
-
upsertPage,
|
|
18
|
-
} from "@effectstream/db";
|
|
19
|
-
import { Buffer } from "node:buffer";
|
|
20
|
-
import { ComponentNames, log, SeverityNumber } from "@effectstream/log";
|
|
21
|
-
import { generateEffectstreamBlockHash, Prando } from "@effectstream/crypto";
|
|
22
|
-
import type { EffectstreamBlockHash } from "@effectstream/utils";
|
|
23
|
-
import type { DBMigrations } from "@effectstream/runtime";
|
|
24
|
-
|
|
25
|
-
/** Scheduled-input boundaries sampled once at run start (stable until the next non-empty block). */
|
|
26
|
-
export type EmptyRunBoundaries = {
|
|
27
|
-
minScheduledBlockHeight: number | null;
|
|
28
|
-
minScheduledTimestampMs: number | null;
|
|
29
|
-
};
|
|
30
|
-
|
|
31
|
-
export const MAX_EMPTY_RUN = 5_000;
|
|
32
|
-
|
|
33
|
-
const MAX_RETRY_BACKOFF_MS = 5_000;
|
|
34
|
-
|
|
35
|
-
export function* loadEmptyRunBoundaries(
|
|
36
|
-
dbConn: Pool,
|
|
37
|
-
): Operation<EmptyRunBoundaries> {
|
|
38
|
-
const [bh] = yield* until(getEarliestScheduledBlockHeight.run(undefined, dbConn));
|
|
39
|
-
const [ts] = yield* until(getEarliestScheduledTimestamp.run(undefined, dbConn));
|
|
40
|
-
return {
|
|
41
|
-
minScheduledBlockHeight: bh?.min_block_height ?? null,
|
|
42
|
-
minScheduledTimestampMs: ts?.min_timestamp != null
|
|
43
|
-
? new Date(ts.min_timestamp).getTime()
|
|
44
|
-
: null,
|
|
45
|
-
};
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
/** Mirrors the empty-block checks in process-blocks.ts without DB/STF work. */
|
|
49
|
-
export function isCoalescableEmptyBlock(
|
|
50
|
-
value: ChainBlock,
|
|
51
|
-
boundaries: EmptyRunBoundaries,
|
|
52
|
-
migrations: DBMigrations[] | undefined,
|
|
53
|
-
): boolean {
|
|
54
|
-
if (value.primitives.length > 0) return false;
|
|
55
|
-
if (
|
|
56
|
-
getMigrationsForBlockHeight(migrations, value.blockNumber, value.blockNumber)
|
|
57
|
-
.length > 0
|
|
58
|
-
) {
|
|
59
|
-
return false;
|
|
60
|
-
}
|
|
61
|
-
if (
|
|
62
|
-
boundaries.minScheduledBlockHeight != null &&
|
|
63
|
-
value.blockNumber >= boundaries.minScheduledBlockHeight
|
|
64
|
-
) {
|
|
65
|
-
return false;
|
|
66
|
-
}
|
|
67
|
-
if (
|
|
68
|
-
boundaries.minScheduledTimestampMs != null &&
|
|
69
|
-
boundaries.minScheduledTimestampMs <= value.timestamp
|
|
70
|
-
) {
|
|
71
|
-
return false;
|
|
72
|
-
}
|
|
73
|
-
return true;
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
export function* processCoalescedEmptyRun(
|
|
77
|
-
endpoint: ChainBlock,
|
|
78
|
-
pool: Pool,
|
|
79
|
-
previousBlockHash: EffectstreamBlockHash | null,
|
|
80
|
-
): Operation<void> {
|
|
81
|
-
const lockName = `processing-blocks:${endpoint.blockNumber}`;
|
|
82
|
-
let attempt = 0;
|
|
83
|
-
while (true) {
|
|
84
|
-
let dbClient: PoolClient | undefined;
|
|
85
|
-
let transientErr: unknown;
|
|
86
|
-
try {
|
|
87
|
-
yield* acquireDBMutex(lockName);
|
|
88
|
-
dbClient = yield* until(pool.connect());
|
|
89
|
-
yield* writeEndpoint(
|
|
90
|
-
endpoint,
|
|
91
|
-
dbClient as unknown as Pool,
|
|
92
|
-
previousBlockHash,
|
|
93
|
-
);
|
|
94
|
-
return;
|
|
95
|
-
} catch (err) {
|
|
96
|
-
if (!isTransientPgError(err)) throw err;
|
|
97
|
-
transientErr = err;
|
|
98
|
-
} finally {
|
|
99
|
-
releaseDBMutex(lockName);
|
|
100
|
-
if (dbClient) {
|
|
101
|
-
try {
|
|
102
|
-
dbClient.release(transientErr as Error | undefined);
|
|
103
|
-
} catch {
|
|
104
|
-
/* already released/destroyed */
|
|
105
|
-
}
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
attempt++;
|
|
110
|
-
poolErrors.log(transientErr, ["sync-write"]);
|
|
111
|
-
const backoffMs = Math.min(500 * 2 ** (attempt - 1), MAX_RETRY_BACKOFF_MS);
|
|
112
|
-
log.local(
|
|
113
|
-
ComponentNames.EFFECTSTREAM_SYNC,
|
|
114
|
-
"block-processing",
|
|
115
|
-
SeverityNumber.WARN,
|
|
116
|
-
(l) =>
|
|
117
|
-
l(
|
|
118
|
-
`transient db error on coalesced run → block ${endpoint.blockNumber}; retry #${attempt} in ${backoffMs}ms: ${
|
|
119
|
-
String(transientErr)
|
|
120
|
-
}`,
|
|
121
|
-
),
|
|
122
|
-
);
|
|
123
|
-
yield* sleep(backoffMs);
|
|
124
|
-
}
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
/** One transaction for the run endpoint; matches process-blocks.ts empty-block writes. */
|
|
128
|
-
function* writeEndpoint(
|
|
129
|
-
endpoint: ChainBlock,
|
|
130
|
-
dbConn: Pool,
|
|
131
|
-
previousBlockHash: EffectstreamBlockHash | null,
|
|
132
|
-
): Operation<void> {
|
|
133
|
-
const blockHash = generateEffectstreamBlockHash(endpoint, previousBlockHash);
|
|
134
|
-
const seed = new Prando(blockHash).seed.toString();
|
|
135
|
-
try {
|
|
136
|
-
yield* until(dbConn.query("BEGIN"));
|
|
137
|
-
|
|
138
|
-
yield* call(() =>
|
|
139
|
-
saveLastBlock.run({
|
|
140
|
-
block_height: endpoint.blockNumber,
|
|
141
|
-
ver: 0,
|
|
142
|
-
main_chain_block_hash: Buffer.from(endpoint.blockNumber.toString()),
|
|
143
|
-
seed,
|
|
144
|
-
ms_timestamp: new Date(endpoint.timestamp),
|
|
145
|
-
}, dbConn)
|
|
146
|
-
);
|
|
147
|
-
|
|
148
|
-
yield* call(() => deleteEmptyBlocks.run(undefined, dbConn));
|
|
149
|
-
yield* call(() =>
|
|
150
|
-
blockHeightDone.run({
|
|
151
|
-
block_hash: Buffer.from("0x0"),
|
|
152
|
-
block_height: endpoint.blockNumber,
|
|
153
|
-
}, dbConn)
|
|
154
|
-
);
|
|
155
|
-
|
|
156
|
-
for (const { protocol_name, lastPage } of endpoint.resumePages) {
|
|
157
|
-
yield* until(
|
|
158
|
-
upsertPage.run({
|
|
159
|
-
protocol_name,
|
|
160
|
-
page_number: lastPage.ownBlockNumber,
|
|
161
|
-
page: JSON.stringify(lastPage),
|
|
162
|
-
}, dbConn),
|
|
163
|
-
);
|
|
164
|
-
yield* until(
|
|
165
|
-
removePages.run({
|
|
166
|
-
protocol_name,
|
|
167
|
-
page_number: lastPage.ownBlockNumber,
|
|
168
|
-
}, dbConn),
|
|
169
|
-
);
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
yield* until(dbConn.query("COMMIT"));
|
|
173
|
-
poolErrors.markHealthy();
|
|
174
|
-
} catch (err) {
|
|
175
|
-
try {
|
|
176
|
-
yield* until(dbConn.query("ROLLBACK"));
|
|
177
|
-
} catch (rollbackErr) {
|
|
178
|
-
log.remote(
|
|
179
|
-
ComponentNames.EFFECTSTREAM_SYNC,
|
|
180
|
-
"block-processing",
|
|
181
|
-
SeverityNumber.WARN,
|
|
182
|
-
(l) =>
|
|
183
|
-
l(
|
|
184
|
-
`ROLLBACK failed for coalesced run → block ${endpoint.blockNumber} (connection likely dead): ${
|
|
185
|
-
String(rollbackErr)
|
|
186
|
-
}`,
|
|
187
|
-
),
|
|
188
|
-
);
|
|
189
|
-
}
|
|
190
|
-
throw err;
|
|
191
|
-
}
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
export type FinalizedBlockSubscription = {
|
|
195
|
-
next(): Operation<IteratorResult<ChainBlock, void>>;
|
|
196
|
-
};
|
|
197
|
-
|
|
198
|
-
function* readFinalizedBlock(
|
|
199
|
-
subscription: FinalizedBlockSubscription,
|
|
200
|
-
): Operation<{ tag: "item"; value: ChainBlock } | { tag: "done" }> {
|
|
201
|
-
const n = yield* subscription.next();
|
|
202
|
-
if (n.done) return { tag: "done" };
|
|
203
|
-
// IteratorResult value type does not narrow on `done` under non-strict tsconfig.
|
|
204
|
-
return { tag: "item", value: n.value as ChainBlock };
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
export type EmptyBlockCoalescerOptions = {
|
|
208
|
-
enabled: boolean;
|
|
209
|
-
subscription: FinalizedBlockSubscription;
|
|
210
|
-
pool: Pool;
|
|
211
|
-
migrations: DBMigrations[] | undefined;
|
|
212
|
-
/** Coalesce only while `now - block.timestamp` exceeds this. Undefined disables lag gating. */
|
|
213
|
-
lagThresholdMs: number | undefined;
|
|
214
|
-
getPreviousBlockHash: () => EffectstreamBlockHash | null;
|
|
215
|
-
onFlush: (endpoint: ChainBlock, length: number) => void;
|
|
216
|
-
idleFlushMs?: number;
|
|
217
|
-
};
|
|
218
|
-
|
|
219
|
-
export type EmptyBlockCoalescer = {
|
|
220
|
-
advance(): Operation<ChainBlock | undefined>;
|
|
221
|
-
};
|
|
222
|
-
|
|
223
|
-
export function createEmptyBlockCoalescer(
|
|
224
|
-
opts: EmptyBlockCoalescerOptions,
|
|
225
|
-
): EmptyBlockCoalescer {
|
|
226
|
-
const idleFlushMs = opts.idleFlushMs ?? 250;
|
|
227
|
-
let pendingRun:
|
|
228
|
-
| { endpoint: ChainBlock; boundaries: EmptyRunBoundaries; length: number }
|
|
229
|
-
| null = null;
|
|
230
|
-
|
|
231
|
-
function* flush(): Operation<void> {
|
|
232
|
-
if (pendingRun == null) return;
|
|
233
|
-
const { endpoint, length } = pendingRun;
|
|
234
|
-
yield* processCoalescedEmptyRun(
|
|
235
|
-
endpoint,
|
|
236
|
-
opts.pool,
|
|
237
|
-
opts.getPreviousBlockHash(),
|
|
238
|
-
);
|
|
239
|
-
pendingRun = null;
|
|
240
|
-
opts.onFlush(endpoint, length);
|
|
241
|
-
}
|
|
242
|
-
|
|
243
|
-
function* nextItem(): Operation<
|
|
244
|
-
{ tag: "item"; value: ChainBlock } | { tag: "done" } | { tag: "idle" }
|
|
245
|
-
> {
|
|
246
|
-
if (pendingRun == null) return yield* readFinalizedBlock(opts.subscription);
|
|
247
|
-
return yield* race([
|
|
248
|
-
readFinalizedBlock(opts.subscription),
|
|
249
|
-
(function* () {
|
|
250
|
-
yield* sleep(idleFlushMs);
|
|
251
|
-
return { tag: "idle" as const };
|
|
252
|
-
})(),
|
|
253
|
-
]);
|
|
254
|
-
}
|
|
255
|
-
|
|
256
|
-
function* advance(): Operation<ChainBlock | undefined> {
|
|
257
|
-
if (!opts.enabled) {
|
|
258
|
-
const n = yield* readFinalizedBlock(opts.subscription);
|
|
259
|
-
return n.tag === "done" ? undefined : n.value;
|
|
260
|
-
}
|
|
261
|
-
while (true) {
|
|
262
|
-
const next = yield* nextItem();
|
|
263
|
-
if (next.tag === "idle") {
|
|
264
|
-
yield* flush();
|
|
265
|
-
continue;
|
|
266
|
-
}
|
|
267
|
-
if (next.tag === "done") {
|
|
268
|
-
yield* flush();
|
|
269
|
-
return undefined;
|
|
270
|
-
}
|
|
271
|
-
const value = next.value;
|
|
272
|
-
const behindTip = opts.lagThresholdMs != null &&
|
|
273
|
-
Date.now() - value.timestamp > opts.lagThresholdMs;
|
|
274
|
-
if (behindTip && value.primitives.length === 0) {
|
|
275
|
-
const boundaries = pendingRun?.boundaries ??
|
|
276
|
-
(yield* loadEmptyRunBoundaries(opts.pool));
|
|
277
|
-
if (isCoalescableEmptyBlock(value, boundaries, opts.migrations)) {
|
|
278
|
-
pendingRun = pendingRun
|
|
279
|
-
? { boundaries, endpoint: value, length: pendingRun.length + 1 }
|
|
280
|
-
: { boundaries, endpoint: value, length: 1 };
|
|
281
|
-
if (pendingRun.length >= MAX_EMPTY_RUN) yield* flush();
|
|
282
|
-
continue;
|
|
283
|
-
}
|
|
284
|
-
}
|
|
285
|
-
yield* flush();
|
|
286
|
-
return value;
|
|
287
|
-
}
|
|
288
|
-
}
|
|
289
|
-
|
|
290
|
-
return { advance };
|
|
291
|
-
}
|
|
@@ -1,69 +0,0 @@
|
|
|
1
|
-
import { expect, test } from "bun:test";
|
|
2
|
-
import {
|
|
3
|
-
type EmptyRunBoundaries,
|
|
4
|
-
isCoalescableEmptyBlock,
|
|
5
|
-
} from "../src/coalesce.ts";
|
|
6
|
-
|
|
7
|
-
const NO_BOUNDARIES: EmptyRunBoundaries = {
|
|
8
|
-
minScheduledBlockHeight: null,
|
|
9
|
-
minScheduledTimestampMs: null,
|
|
10
|
-
};
|
|
11
|
-
|
|
12
|
-
function block(
|
|
13
|
-
blockNumber: number,
|
|
14
|
-
timestamp: number,
|
|
15
|
-
primitives: unknown[] = [],
|
|
16
|
-
): any {
|
|
17
|
-
return { blockNumber, timestamp, blockInfo: [], resumePages: [], primitives };
|
|
18
|
-
}
|
|
19
|
-
|
|
20
|
-
test("empty block with no boundaries is coalescable", () => {
|
|
21
|
-
expect(isCoalescableEmptyBlock(block(10, 10_000), NO_BOUNDARIES, undefined))
|
|
22
|
-
.toBe(true);
|
|
23
|
-
});
|
|
24
|
-
|
|
25
|
-
test("block with on-chain content is NOT coalescable", () => {
|
|
26
|
-
expect(
|
|
27
|
-
isCoalescableEmptyBlock(block(10, 10_000, [{}]), NO_BOUNDARIES, undefined),
|
|
28
|
-
).toBe(false);
|
|
29
|
-
});
|
|
30
|
-
|
|
31
|
-
test("block where a migration fires is NOT coalescable", () => {
|
|
32
|
-
const migrations = [{ name: "m1", sql: "", blockHeight: 10 }];
|
|
33
|
-
expect(isCoalescableEmptyBlock(block(10, 10_000), NO_BOUNDARIES, migrations))
|
|
34
|
-
.toBe(false);
|
|
35
|
-
expect(isCoalescableEmptyBlock(block(9, 9_000), NO_BOUNDARIES, migrations))
|
|
36
|
-
.toBe(true);
|
|
37
|
-
});
|
|
38
|
-
|
|
39
|
-
test("block at/after a pending block-height-scheduled input is NOT coalescable", () => {
|
|
40
|
-
const boundaries: EmptyRunBoundaries = {
|
|
41
|
-
minScheduledBlockHeight: 10,
|
|
42
|
-
minScheduledTimestampMs: null,
|
|
43
|
-
};
|
|
44
|
-
expect(isCoalescableEmptyBlock(block(9, 9_000), boundaries, undefined)).toBe(
|
|
45
|
-
true,
|
|
46
|
-
);
|
|
47
|
-
expect(isCoalescableEmptyBlock(block(10, 10_000), boundaries, undefined)).toBe(
|
|
48
|
-
false,
|
|
49
|
-
);
|
|
50
|
-
expect(isCoalescableEmptyBlock(block(11, 11_000), boundaries, undefined)).toBe(
|
|
51
|
-
false,
|
|
52
|
-
);
|
|
53
|
-
});
|
|
54
|
-
|
|
55
|
-
test("block at/after a due timestamp-scheduled input is NOT coalescable", () => {
|
|
56
|
-
const boundaries: EmptyRunBoundaries = {
|
|
57
|
-
minScheduledBlockHeight: null,
|
|
58
|
-
minScheduledTimestampMs: 10_000,
|
|
59
|
-
};
|
|
60
|
-
expect(isCoalescableEmptyBlock(block(9, 9_999), boundaries, undefined)).toBe(
|
|
61
|
-
true,
|
|
62
|
-
);
|
|
63
|
-
expect(isCoalescableEmptyBlock(block(10, 10_000), boundaries, undefined)).toBe(
|
|
64
|
-
false,
|
|
65
|
-
);
|
|
66
|
-
expect(isCoalescableEmptyBlock(block(11, 10_001), boundaries, undefined)).toBe(
|
|
67
|
-
false,
|
|
68
|
-
);
|
|
69
|
-
});
|