@effectstream/runtime 0.100.19 → 0.100.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -69,6 +69,40 @@ exercised when this loop is running. The `StartConfig` shape determines
69
69
  which DB migrations apply, which state machines fire, and whether the
70
70
  runtime exposes a Fastify router.
71
71
 
72
+ ## Empty-block coalescing (catch-up)
73
+
74
+ The runtime replays one Effectstream block per main-clock tick, each in its
75
+ own DB transaction. During a deep catch-up over an idle gap (e.g. an NTP main
76
+ chain at one block/second across days), most of those blocks are **empty** — no
77
+ on-chain content, no scheduled input, no migration — yet still cost a full
78
+ `BEGIN…COMMIT` round-trip.
79
+ Empty-block coalescing folds a run of consecutive empty blocks into a **single
80
+ transaction** that writes only the run's endpoint. It is **opt-in** and off by
81
+ default:
82
+
83
+ ```bash
84
+ EFFECTSTREAM_COALESCE_EMPTY_BLOCKS=true
85
+ ```
86
+
87
+ Behavior and guarantees:
88
+
89
+ - **Catch-up only.** Coalescing engages only while behind the chain tip
90
+ (`now - block.timestamp` greater than 10× the main clock's block time) and
91
+ disengages at the tip, so steady-state stays one block per transaction.
92
+ - **Identical database.** A coalesced sync produces a byte-for-byte identical
93
+ database to a non-coalesced one. Empty blocks already never advance the block
94
+ hash or RNG seed, so only the endpoint's block row and the per-protocol resume
95
+ watermark are written — exactly as the normal path would leave them.
96
+ - **Never skips work.** A run is flushed before any block that has on-chain
97
+ content, a migration at its height, or a due block-height/timestamp-scheduled
98
+ input — so nothing that mutates state is ever folded away.
99
+ - **Tip stays visible.** If the producer goes quiet on an empty tail, the run is
100
+ flushed on a short idle timeout so the latest height still commits.
101
+ - A run is also capped (`MAX_EMPTY_RUN`) to bound transaction size and recovery.
102
+
103
+ Implementation: [`src/coalesce.ts`](./src/coalesce.ts) (the `createEmptyBlockCoalescer`
104
+ coordinator the main loop drives).
105
+
72
106
  ## Key exports
73
107
 
74
108
  - `init()` - `Operation<void>`. One-shot setup: OpenTelemetry, config validation, version pinning. Call before `start`.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@effectstream/runtime",
3
- "version": "0.100.19",
3
+ "version": "0.100.20",
4
4
  "license": "MIT",
5
5
  "homepage": "https://effectstream.github.io/docs/",
6
6
  "repository": {
@@ -13,17 +13,17 @@
13
13
  },
14
14
  "exports": "./src/mod.ts",
15
15
  "dependencies": {
16
- "@effectstream/concise": "0.100.19",
17
- "@effectstream/config": "0.100.19",
18
- "@effectstream/coroutine": "0.100.19",
19
- "@effectstream/crypto": "0.100.19",
20
- "@effectstream/db": "0.100.19",
21
- "@effectstream/event-client": "0.100.19",
22
- "@effectstream/event-server": "0.100.19",
23
- "@effectstream/log": "0.100.19",
24
- "@effectstream/sm": "0.100.19",
25
- "@effectstream/sync": "0.100.19",
26
- "@effectstream/utils": "0.100.19",
16
+ "@effectstream/concise": "0.100.20",
17
+ "@effectstream/config": "0.100.20",
18
+ "@effectstream/coroutine": "0.100.20",
19
+ "@effectstream/crypto": "0.100.20",
20
+ "@effectstream/db": "0.100.20",
21
+ "@effectstream/event-client": "0.100.20",
22
+ "@effectstream/event-server": "0.100.20",
23
+ "@effectstream/log": "0.100.20",
24
+ "@effectstream/sm": "0.100.20",
25
+ "@effectstream/sync": "0.100.20",
26
+ "@effectstream/utils": "0.100.20",
27
27
  "@fastify/cors": "^11.0.1",
28
28
  "@metamask/json-rpc-engine": "^10.0.3",
29
29
  "pg": "8.17.2",
@@ -0,0 +1,291 @@
1
+ /** Empty-block coalescing during deep catch-up: fold consecutive no-op blocks into one transaction. */
2
+ import type { ChainBlock } from "@effectstream/sync";
3
+ import { call, type Operation, race, sleep, until } from "effection";
4
+ import type { Pool, PoolClient } from "pg";
5
+ import {
6
+ acquireDBMutex,
7
+ blockHeightDone,
8
+ deleteEmptyBlocks,
9
+ getEarliestScheduledBlockHeight,
10
+ getEarliestScheduledTimestamp,
11
+ getMigrationsForBlockHeight,
12
+ isTransientPgError,
13
+ poolErrors,
14
+ releaseDBMutex,
15
+ removePages,
16
+ saveLastBlock,
17
+ upsertPage,
18
+ } from "@effectstream/db";
19
+ import { Buffer } from "node:buffer";
20
+ import { ComponentNames, log, SeverityNumber } from "@effectstream/log";
21
+ import { generateEffectstreamBlockHash, Prando } from "@effectstream/crypto";
22
+ import type { EffectstreamBlockHash } from "@effectstream/utils";
23
+ import type { DBMigrations } from "@effectstream/runtime";
24
+
25
+ /** Scheduled-input boundaries sampled once at run start (stable until the next non-empty block). */
26
+ export type EmptyRunBoundaries = {
27
+ minScheduledBlockHeight: number | null;
28
+ minScheduledTimestampMs: number | null;
29
+ };
30
+
31
+ export const MAX_EMPTY_RUN = 5_000;
32
+
33
+ const MAX_RETRY_BACKOFF_MS = 5_000;
34
+
35
+ export function* loadEmptyRunBoundaries(
36
+ dbConn: Pool,
37
+ ): Operation<EmptyRunBoundaries> {
38
+ const [bh] = yield* until(getEarliestScheduledBlockHeight.run(undefined, dbConn));
39
+ const [ts] = yield* until(getEarliestScheduledTimestamp.run(undefined, dbConn));
40
+ return {
41
+ minScheduledBlockHeight: bh?.min_block_height ?? null,
42
+ minScheduledTimestampMs: ts?.min_timestamp != null
43
+ ? new Date(ts.min_timestamp).getTime()
44
+ : null,
45
+ };
46
+ }
47
+
48
+ /** Mirrors the empty-block checks in process-blocks.ts without DB/STF work. */
49
+ export function isCoalescableEmptyBlock(
50
+ value: ChainBlock,
51
+ boundaries: EmptyRunBoundaries,
52
+ migrations: DBMigrations[] | undefined,
53
+ ): boolean {
54
+ if (value.primitives.length > 0) return false;
55
+ if (
56
+ getMigrationsForBlockHeight(migrations, value.blockNumber, value.blockNumber)
57
+ .length > 0
58
+ ) {
59
+ return false;
60
+ }
61
+ if (
62
+ boundaries.minScheduledBlockHeight != null &&
63
+ value.blockNumber >= boundaries.minScheduledBlockHeight
64
+ ) {
65
+ return false;
66
+ }
67
+ if (
68
+ boundaries.minScheduledTimestampMs != null &&
69
+ boundaries.minScheduledTimestampMs <= value.timestamp
70
+ ) {
71
+ return false;
72
+ }
73
+ return true;
74
+ }
75
+
76
+ export function* processCoalescedEmptyRun(
77
+ endpoint: ChainBlock,
78
+ pool: Pool,
79
+ previousBlockHash: EffectstreamBlockHash | null,
80
+ ): Operation<void> {
81
+ const lockName = `processing-blocks:${endpoint.blockNumber}`;
82
+ let attempt = 0;
83
+ while (true) {
84
+ let dbClient: PoolClient | undefined;
85
+ let transientErr: unknown;
86
+ try {
87
+ yield* acquireDBMutex(lockName);
88
+ dbClient = yield* until(pool.connect());
89
+ yield* writeEndpoint(
90
+ endpoint,
91
+ dbClient as unknown as Pool,
92
+ previousBlockHash,
93
+ );
94
+ return;
95
+ } catch (err) {
96
+ if (!isTransientPgError(err)) throw err;
97
+ transientErr = err;
98
+ } finally {
99
+ releaseDBMutex(lockName);
100
+ if (dbClient) {
101
+ try {
102
+ dbClient.release(transientErr as Error | undefined);
103
+ } catch {
104
+ /* already released/destroyed */
105
+ }
106
+ }
107
+ }
108
+
109
+ attempt++;
110
+ poolErrors.log(transientErr, ["sync-write"]);
111
+ const backoffMs = Math.min(500 * 2 ** (attempt - 1), MAX_RETRY_BACKOFF_MS);
112
+ log.local(
113
+ ComponentNames.EFFECTSTREAM_SYNC,
114
+ "block-processing",
115
+ SeverityNumber.WARN,
116
+ (l) =>
117
+ l(
118
+ `transient db error on coalesced run → block ${endpoint.blockNumber}; retry #${attempt} in ${backoffMs}ms: ${
119
+ String(transientErr)
120
+ }`,
121
+ ),
122
+ );
123
+ yield* sleep(backoffMs);
124
+ }
125
+ }
126
+
127
+ /** One transaction for the run endpoint; matches process-blocks.ts empty-block writes. */
128
+ function* writeEndpoint(
129
+ endpoint: ChainBlock,
130
+ dbConn: Pool,
131
+ previousBlockHash: EffectstreamBlockHash | null,
132
+ ): Operation<void> {
133
+ const blockHash = generateEffectstreamBlockHash(endpoint, previousBlockHash);
134
+ const seed = new Prando(blockHash).seed.toString();
135
+ try {
136
+ yield* until(dbConn.query("BEGIN"));
137
+
138
+ yield* call(() =>
139
+ saveLastBlock.run({
140
+ block_height: endpoint.blockNumber,
141
+ ver: 0,
142
+ main_chain_block_hash: Buffer.from(endpoint.blockNumber.toString()),
143
+ seed,
144
+ ms_timestamp: new Date(endpoint.timestamp),
145
+ }, dbConn)
146
+ );
147
+
148
+ yield* call(() => deleteEmptyBlocks.run(undefined, dbConn));
149
+ yield* call(() =>
150
+ blockHeightDone.run({
151
+ block_hash: Buffer.from("0x0"),
152
+ block_height: endpoint.blockNumber,
153
+ }, dbConn)
154
+ );
155
+
156
+ for (const { protocol_name, lastPage } of endpoint.resumePages) {
157
+ yield* until(
158
+ upsertPage.run({
159
+ protocol_name,
160
+ page_number: lastPage.ownBlockNumber,
161
+ page: JSON.stringify(lastPage),
162
+ }, dbConn),
163
+ );
164
+ yield* until(
165
+ removePages.run({
166
+ protocol_name,
167
+ page_number: lastPage.ownBlockNumber,
168
+ }, dbConn),
169
+ );
170
+ }
171
+
172
+ yield* until(dbConn.query("COMMIT"));
173
+ poolErrors.markHealthy();
174
+ } catch (err) {
175
+ try {
176
+ yield* until(dbConn.query("ROLLBACK"));
177
+ } catch (rollbackErr) {
178
+ log.remote(
179
+ ComponentNames.EFFECTSTREAM_SYNC,
180
+ "block-processing",
181
+ SeverityNumber.WARN,
182
+ (l) =>
183
+ l(
184
+ `ROLLBACK failed for coalesced run → block ${endpoint.blockNumber} (connection likely dead): ${
185
+ String(rollbackErr)
186
+ }`,
187
+ ),
188
+ );
189
+ }
190
+ throw err;
191
+ }
192
+ }
193
+
194
+ export type FinalizedBlockSubscription = {
195
+ next(): Operation<IteratorResult<ChainBlock, void>>;
196
+ };
197
+
198
+ function* readFinalizedBlock(
199
+ subscription: FinalizedBlockSubscription,
200
+ ): Operation<{ tag: "item"; value: ChainBlock } | { tag: "done" }> {
201
+ const n = yield* subscription.next();
202
+ if (n.done) return { tag: "done" };
203
+ // IteratorResult value type does not narrow on `done` under non-strict tsconfig.
204
+ return { tag: "item", value: n.value as ChainBlock };
205
+ }
206
+
207
+ export type EmptyBlockCoalescerOptions = {
208
+ enabled: boolean;
209
+ subscription: FinalizedBlockSubscription;
210
+ pool: Pool;
211
+ migrations: DBMigrations[] | undefined;
212
+ /** Coalesce only while `now - block.timestamp` exceeds this. Undefined disables lag gating. */
213
+ lagThresholdMs: number | undefined;
214
+ getPreviousBlockHash: () => EffectstreamBlockHash | null;
215
+ onFlush: (endpoint: ChainBlock, length: number) => void;
216
+ idleFlushMs?: number;
217
+ };
218
+
219
+ export type EmptyBlockCoalescer = {
220
+ advance(): Operation<ChainBlock | undefined>;
221
+ };
222
+
223
+ export function createEmptyBlockCoalescer(
224
+ opts: EmptyBlockCoalescerOptions,
225
+ ): EmptyBlockCoalescer {
226
+ const idleFlushMs = opts.idleFlushMs ?? 250;
227
+ let pendingRun:
228
+ | { endpoint: ChainBlock; boundaries: EmptyRunBoundaries; length: number }
229
+ | null = null;
230
+
231
+ function* flush(): Operation<void> {
232
+ if (pendingRun == null) return;
233
+ const { endpoint, length } = pendingRun;
234
+ yield* processCoalescedEmptyRun(
235
+ endpoint,
236
+ opts.pool,
237
+ opts.getPreviousBlockHash(),
238
+ );
239
+ pendingRun = null;
240
+ opts.onFlush(endpoint, length);
241
+ }
242
+
243
+ function* nextItem(): Operation<
244
+ { tag: "item"; value: ChainBlock } | { tag: "done" } | { tag: "idle" }
245
+ > {
246
+ if (pendingRun == null) return yield* readFinalizedBlock(opts.subscription);
247
+ return yield* race([
248
+ readFinalizedBlock(opts.subscription),
249
+ (function* () {
250
+ yield* sleep(idleFlushMs);
251
+ return { tag: "idle" as const };
252
+ })(),
253
+ ]);
254
+ }
255
+
256
+ function* advance(): Operation<ChainBlock | undefined> {
257
+ if (!opts.enabled) {
258
+ const n = yield* readFinalizedBlock(opts.subscription);
259
+ return n.tag === "done" ? undefined : n.value;
260
+ }
261
+ while (true) {
262
+ const next = yield* nextItem();
263
+ if (next.tag === "idle") {
264
+ yield* flush();
265
+ continue;
266
+ }
267
+ if (next.tag === "done") {
268
+ yield* flush();
269
+ return undefined;
270
+ }
271
+ const value = next.value;
272
+ const behindTip = opts.lagThresholdMs != null &&
273
+ Date.now() - value.timestamp > opts.lagThresholdMs;
274
+ if (behindTip && value.primitives.length === 0) {
275
+ const boundaries = pendingRun?.boundaries ??
276
+ (yield* loadEmptyRunBoundaries(opts.pool));
277
+ if (isCoalescableEmptyBlock(value, boundaries, opts.migrations)) {
278
+ pendingRun = pendingRun
279
+ ? { boundaries, endpoint: value, length: pendingRun.length + 1 }
280
+ : { boundaries, endpoint: value, length: 1 };
281
+ if (pendingRun.length >= MAX_EMPTY_RUN) yield* flush();
282
+ continue;
283
+ }
284
+ }
285
+ yield* flush();
286
+ return value;
287
+ }
288
+ }
289
+
290
+ return { advance };
291
+ }
package/src/main.ts CHANGED
@@ -32,6 +32,7 @@ import {
32
32
  type PendingEvent,
33
33
  processFinalizedBlockWithRetry,
34
34
  } from "./process-blocks.ts";
35
+ import { createEmptyBlockCoalescer } from "./coalesce.ts";
35
36
  import { startHttpServer } from "./api/http-server.ts";
36
37
  import { recordAppliedBlock } from "./api/apply-status.ts";
37
38
  import type { StartConfig } from "./types.ts";
@@ -92,9 +93,12 @@ export function* start(config: StartConfig): Operation<void> {
92
93
  );
93
94
  });
94
95
 
96
+ // 10× main clock block time (NTP if present, else protocol 0). Undefined disables lag gating.
95
97
  const ntpConfig = syncInfo.find(s => s.networkType === ConfigNetworkType.NTP);
96
- const ntpBlockTimeMS = (ntpConfig?.network as { blockTimeMS: number } | undefined)?.blockTimeMS;
97
- const lagThresholdMs = ntpBlockTimeMS != null ? ntpBlockTimeMS * 10 : undefined;
98
+ const clockBlockTimeMS =
99
+ (ntpConfig?.network as { blockTimeMS?: number } | undefined)?.blockTimeMS ??
100
+ (syncInfo[0]?.network as { blockTimeMS?: number } | undefined)?.blockTimeMS;
101
+ const lagThresholdMs = clockBlockTimeMS != null ? clockBlockTimeMS * 10 : undefined;
98
102
 
99
103
  const finalizedBlockStream = createChannel<ChainBlock>();
100
104
 
@@ -148,9 +152,36 @@ export function* start(config: StartConfig): Operation<void> {
148
152
  yield* spawn(() => runSnapshotLoop(config.snapshotConfig!));
149
153
  }
150
154
 
151
- let nextBlock = yield* finalizedBlocks.next();
152
- for (; !nextBlock.done; nextBlock = yield* finalizedBlocks.next()) {
153
- const value = nextBlock.value;
155
+ const coalescer = createEmptyBlockCoalescer({
156
+ enabled: ENV.EFFECTSTREAM_COALESCE_EMPTY_BLOCKS,
157
+ subscription: finalizedBlocks,
158
+ pool: dbConn as any, // Pool,
159
+ migrations: config.migrations,
160
+ lagThresholdMs,
161
+ getPreviousBlockHash: () => blockHash,
162
+ onFlush: (endpoint, length) => {
163
+ recordAppliedBlock(endpoint);
164
+ emitLatestBlocks(
165
+ endpoint.blockNumber,
166
+ endpoint.timestamp,
167
+ getRangesForSyncProtocols(endpoint),
168
+ );
169
+ log.local(
170
+ ComponentNames.EFFECTSTREAM_SYNC,
171
+ "block-merge",
172
+ SeverityNumber.INFO,
173
+ (l) =>
174
+ l(
175
+ `coalesced ${length} empty block(s) → block ${endpoint.blockNumber}`,
176
+ ),
177
+ );
178
+ },
179
+ });
180
+
181
+ while (true) {
182
+ const value = yield* coalescer.advance();
183
+ if (value === undefined) break;
184
+
154
185
  // processFinalizedBlockWithRetry owns connection checkout, the per-block
155
186
  // DB mutex (PGLite), and transient-pg retry/backoff. App events it
156
187
  // collects are flushed below ONLY after it returns — i.e. strictly after
@@ -0,0 +1,69 @@
1
+ import { expect, test } from "bun:test";
2
+ import {
3
+ type EmptyRunBoundaries,
4
+ isCoalescableEmptyBlock,
5
+ } from "../src/coalesce.ts";
6
+
7
+ const NO_BOUNDARIES: EmptyRunBoundaries = {
8
+ minScheduledBlockHeight: null,
9
+ minScheduledTimestampMs: null,
10
+ };
11
+
12
+ function block(
13
+ blockNumber: number,
14
+ timestamp: number,
15
+ primitives: unknown[] = [],
16
+ ): any {
17
+ return { blockNumber, timestamp, blockInfo: [], resumePages: [], primitives };
18
+ }
19
+
20
+ test("empty block with no boundaries is coalescable", () => {
21
+ expect(isCoalescableEmptyBlock(block(10, 10_000), NO_BOUNDARIES, undefined))
22
+ .toBe(true);
23
+ });
24
+
25
+ test("block with on-chain content is NOT coalescable", () => {
26
+ expect(
27
+ isCoalescableEmptyBlock(block(10, 10_000, [{}]), NO_BOUNDARIES, undefined),
28
+ ).toBe(false);
29
+ });
30
+
31
+ test("block where a migration fires is NOT coalescable", () => {
32
+ const migrations = [{ name: "m1", sql: "", blockHeight: 10 }];
33
+ expect(isCoalescableEmptyBlock(block(10, 10_000), NO_BOUNDARIES, migrations))
34
+ .toBe(false);
35
+ expect(isCoalescableEmptyBlock(block(9, 9_000), NO_BOUNDARIES, migrations))
36
+ .toBe(true);
37
+ });
38
+
39
+ test("block at/after a pending block-height-scheduled input is NOT coalescable", () => {
40
+ const boundaries: EmptyRunBoundaries = {
41
+ minScheduledBlockHeight: 10,
42
+ minScheduledTimestampMs: null,
43
+ };
44
+ expect(isCoalescableEmptyBlock(block(9, 9_000), boundaries, undefined)).toBe(
45
+ true,
46
+ );
47
+ expect(isCoalescableEmptyBlock(block(10, 10_000), boundaries, undefined)).toBe(
48
+ false,
49
+ );
50
+ expect(isCoalescableEmptyBlock(block(11, 11_000), boundaries, undefined)).toBe(
51
+ false,
52
+ );
53
+ });
54
+
55
+ test("block at/after a due timestamp-scheduled input is NOT coalescable", () => {
56
+ const boundaries: EmptyRunBoundaries = {
57
+ minScheduledBlockHeight: null,
58
+ minScheduledTimestampMs: 10_000,
59
+ };
60
+ expect(isCoalescableEmptyBlock(block(9, 9_999), boundaries, undefined)).toBe(
61
+ true,
62
+ );
63
+ expect(isCoalescableEmptyBlock(block(10, 10_000), boundaries, undefined)).toBe(
64
+ false,
65
+ );
66
+ expect(isCoalescableEmptyBlock(block(11, 10_001), boundaries, undefined)).toBe(
67
+ false,
68
+ );
69
+ });
@@ -0,0 +1,133 @@
1
+ /** E2E: coalesced vs non-coalesced syncs produce the same DB; resume through coalescing matches a clean run. */
2
+ import { expect, test } from "bun:test";
3
+ import { setupHarness } from "./harness.ts";
4
+ import {
5
+ compareConsistencySnapshots,
6
+ type ConsistencySnapshot,
7
+ dumpConsistencySnapshot,
8
+ type SnapshotDiff,
9
+ type SnapshotOpts,
10
+ SYNC_BOOKKEEPING_TABLES,
11
+ } from "./consistency-snapshot.ts";
12
+
13
+ function summarize(snap: ConsistencySnapshot): string {
14
+ let rows = 0;
15
+ for (const f of snap.values()) rows += f.rowCount;
16
+ return `${snap.size} tables, ${rows} rows`;
17
+ }
18
+
19
+ function formatDiffs(diffs: SnapshotDiff[]): string {
20
+ return diffs
21
+ .map((d) => `${d.table} [${d.reason} A=${d.aRows ?? "—"} B=${d.bRows ?? "—"}]`)
22
+ .join(", ");
23
+ }
24
+
25
+ function logComparison(
26
+ tag: string,
27
+ aLabel: string,
28
+ a: ConsistencySnapshot,
29
+ bLabel: string,
30
+ b: ConsistencySnapshot,
31
+ diffs: SnapshotDiff[],
32
+ ): void {
33
+ console.log(tag);
34
+ console.log(`${tag} ${aLabel}: ${summarize(a)}`);
35
+ console.log(`${tag} ${bLabel}: ${summarize(b)}`);
36
+ console.log(
37
+ diffs.length === 0
38
+ ? `${tag} → IDENTICAL (0 differences)`
39
+ : `${tag} → ${diffs.length} diff(s): ${formatDiffs(diffs)}`,
40
+ );
41
+ }
42
+
43
+ test(
44
+ "coalesced and non-coalesced from-scratch syncs produce identical databases",
45
+ async () => {
46
+ const events = [
47
+ { atBlock: 25, marker: "e25" },
48
+ { atBlock: 50, marker: "e50" },
49
+ { atBlock: 75, marker: "e75" },
50
+ ];
51
+
52
+ const runOnce = async (
53
+ coalesce: boolean,
54
+ apiPort: number,
55
+ ): Promise<ConsistencySnapshot> => {
56
+ const h = await setupHarness();
57
+ try {
58
+ await h.runToHeight({
59
+ events,
60
+ tips: { mainClock: 100, parallelP: 200 },
61
+ target: 100,
62
+ apiPort,
63
+ coalesce,
64
+ });
65
+ return await dumpConsistencySnapshot(h.pool);
66
+ } finally {
67
+ await h.teardown();
68
+ }
69
+ };
70
+
71
+ const off = await runOnce(false, 19160);
72
+ const on = await runOnce(true, 19161);
73
+
74
+ const { diffs } = compareConsistencySnapshots(off, on);
75
+ logComparison("[coalesce A] from-scratch sync to height 100:", "coalesce OFF", off, "coalesce ON", on, diffs);
76
+ expect(diffs).toEqual([]);
77
+ },
78
+ 120_000,
79
+ );
80
+
81
+ test(
82
+ "a coalesced sync that resumes matches a clean coalesced sync",
83
+ async () => {
84
+ const events = [{ atBlock: 20, marker: "gap20" }];
85
+ const parallelStepSize = 1000;
86
+ const dumpOpts: SnapshotOpts = { excludeTables: SYNC_BOOKKEEPING_TABLES };
87
+
88
+ const ref = await setupHarness();
89
+ let refSnap: ConsistencySnapshot;
90
+ try {
91
+ await ref.runToHeight({
92
+ events,
93
+ parallelStepSize,
94
+ tips: { mainClock: 30, parallelP: 50 },
95
+ target: 30,
96
+ apiPort: 19170,
97
+ coalesce: true,
98
+ });
99
+ refSnap = await dumpConsistencySnapshot(ref.pool, dumpOpts);
100
+ } finally {
101
+ await ref.teardown();
102
+ }
103
+
104
+ const resumed = await setupHarness();
105
+ let resumeSnap: ConsistencySnapshot;
106
+ try {
107
+ await resumed.runToHeight({
108
+ events,
109
+ parallelStepSize,
110
+ tips: { mainClock: 10, parallelP: 50 },
111
+ target: 10,
112
+ apiPort: 19171,
113
+ coalesce: true,
114
+ });
115
+ await resumed.runToHeight({
116
+ events,
117
+ parallelStepSize,
118
+ tips: { mainClock: 30, parallelP: 50 },
119
+ target: 30,
120
+ apiPort: 19172,
121
+ coalesce: true,
122
+ });
123
+ resumeSnap = await dumpConsistencySnapshot(resumed.pool, dumpOpts);
124
+ } finally {
125
+ await resumed.teardown();
126
+ }
127
+
128
+ const { diffs } = compareConsistencySnapshots(refSnap, resumeSnap);
129
+ logComparison("[coalesce B] clean vs resumed coalesced sync to height 30:", "clean ref", refSnap, "resumed", resumeSnap, diffs);
130
+ expect(diffs).toEqual([]);
131
+ },
132
+ 120_000,
133
+ );
@@ -305,6 +305,8 @@ export type RunToHeightOpts = {
305
305
  /** Optional per-protocol page watermarks to also wait for before exiting. */
306
306
  waitPages?: Record<string, number>;
307
307
  securityNamespace?: string;
308
+ /** Enable/disable empty-block coalescing. */
309
+ coalesce?: boolean;
308
310
  };
309
311
 
310
312
  const RUNNER = join(import.meta.dir, "node-runner.ts");
@@ -329,6 +331,7 @@ function makeRunToHeight(
329
331
  target: opts.target,
330
332
  apiPort: opts.apiPort,
331
333
  waitPages: opts.waitPages,
334
+ coalesce: opts.coalesce,
332
335
  };
333
336
  return new Promise<void>((resolve, reject) => {
334
337
  const child = spawn("bun", [RUNNER, JSON.stringify(spec)], {
@@ -48,6 +48,7 @@ export type RunSpec = {
48
48
  * ahead and persisted its page row before we kill the process.
49
49
  */
50
50
  waitPages?: Record<string, number>;
51
+ coalesce?: boolean;
51
52
  };
52
53
 
53
54
  const noopStf: StartConfigGameStateTransitions = function* () {};
@@ -60,6 +61,7 @@ async function main() {
60
61
  // broker (it binds fixed ports and isn't freed on halt) and a per-run API port.
61
62
  process.env.MQTT_BROKER = "false";
62
63
  process.env.EFFECTSTREAM_API_PORT = String(spec.apiPort);
64
+ process.env.EFFECTSTREAM_COALESCE_EMPTY_BLOCKS = spec.coalesce ? "true" : "false";
63
65
  const isPglite = process.env.PGLITE === "true";
64
66
 
65
67
  for (const [name, tip] of Object.entries(spec.tips)) {