@effectstream/runtime 0.100.22 → 0.100.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -0
- package/conciseinputhash-determinism.md +114 -0
- package/package.json +12 -12
- package/src/api/apply-status.ts +30 -0
- package/src/api/http-server.ts +68 -7
- package/src/api/stream-status.ts +35 -0
- package/src/coalesce.ts +291 -0
- package/src/finalized-stream.ts +73 -0
- package/src/main.ts +111 -89
- package/src/process-blocks.ts +126 -27
- package/src/types.ts +7 -0
- package/test/coalesce.unit.test.ts +69 -0
- package/test/reproduction/buffering.test.ts +650 -0
- package/test/reproduction/coalesce-memory.test.ts +181 -0
- package/test/reproduction/coalesce.test.ts +133 -0
- package/test/reproduction/consistency-snapshot.ts +168 -0
- package/test/reproduction/consistency.test.ts +197 -0
- package/test/reproduction/harness.ts +573 -270
- package/test/reproduction/node-runner.ts +154 -0
- package/test/reproduction/poll.ts +160 -0
- package/test/reproduction/resume.test.ts +77 -0
- package/test/reproduction/sanity.test.ts +40 -91
- package/test/reproduction/scenario.ts +126 -0
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bounded hand-off queue between the merge (producer) and the apply loop (consumer).
|
|
3
|
+
*
|
|
4
|
+
* The merge sends finalized blocks into an effection channel; the apply loop drains
|
|
5
|
+
* them. effection channel subscribers buffer unbounded, so during deep catch-up the
|
|
6
|
+
* merge (no I/O) outpaces the apply loop and the queue grows toward the whole backlog
|
|
7
|
+
* → OOM (sync/CLAUDE.md Finding #1) — especially with empty-block coalescing removing
|
|
8
|
+
* the per-block apply brake.
|
|
9
|
+
*
|
|
10
|
+
* This wraps the channel with backpressure: the producer blocks once `cap` produced
|
|
11
|
+
* blocks are still unconsumed, and the consumer wakes it as it drains. The cap only
|
|
12
|
+
* ever throttles the producer (the consumer always keeps pulling), so it can't
|
|
13
|
+
* deadlock. `inFlight = produced − consumed` is the live queue depth, surfaced on
|
|
14
|
+
* /debug/metrics.
|
|
15
|
+
*/
|
|
16
|
+
import { type Channel, createChannel, type Operation } from "effection";
|
|
17
|
+
import { conditionVariable } from "@effectstream/utils";
|
|
18
|
+
import type { ChainBlock } from "@effectstream/sync";
|
|
19
|
+
import {
|
|
20
|
+
finalizedStreamStatus,
|
|
21
|
+
recordConsumed,
|
|
22
|
+
recordProduced,
|
|
23
|
+
} from "./api/stream-status.ts";
|
|
24
|
+
import type { FinalizedBlockSubscription } from "./coalesce.ts";
|
|
25
|
+
|
|
26
|
+
export type BoundedFinalizedStream = {
|
|
27
|
+
/** Send side — hand to `startMerge`. `send` blocks the producer at the cap. */
|
|
28
|
+
stream: Channel<ChainBlock, void>;
|
|
29
|
+
/** Read side — hand to the coalescer / apply loop. Counts drains and wakes the producer. */
|
|
30
|
+
subscription: FinalizedBlockSubscription;
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
export function* createBoundedFinalizedStream(
|
|
34
|
+
cap: number,
|
|
35
|
+
): Operation<BoundedFinalizedStream> {
|
|
36
|
+
const channel = createChannel<ChainBlock>();
|
|
37
|
+
// Subscribe BEFORE the caller spawns the merge: effection channels drop sends
|
|
38
|
+
// with no subscriber, so a fast restart could emit blocks before we subscribe
|
|
39
|
+
// and stall at the pre-restart height.
|
|
40
|
+
const subscription = yield* channel;
|
|
41
|
+
|
|
42
|
+
const drained = conditionVariable<void>();
|
|
43
|
+
const inFlight = () =>
|
|
44
|
+
finalizedStreamStatus.produced - finalizedStreamStatus.consumed;
|
|
45
|
+
|
|
46
|
+
// `startMerge` only uses `.send`; gate it. check-then-wait is atomic under
|
|
47
|
+
// effection's cooperative scheduling, so there's no lost wakeup.
|
|
48
|
+
const stream: Channel<ChainBlock, void> = {
|
|
49
|
+
...channel,
|
|
50
|
+
*send(block: ChainBlock) {
|
|
51
|
+
while (inFlight() >= cap) {
|
|
52
|
+
yield* drained.wait();
|
|
53
|
+
}
|
|
54
|
+
recordProduced();
|
|
55
|
+
yield* channel.send(block);
|
|
56
|
+
},
|
|
57
|
+
};
|
|
58
|
+
|
|
59
|
+
// Count consumed at the pull point (exact channel depth, decoupled from the
|
|
60
|
+
// coalescer's run length) and wake the merge once the queue drops below cap.
|
|
61
|
+
const countedSubscription: FinalizedBlockSubscription = {
|
|
62
|
+
*next() {
|
|
63
|
+
const result = yield* subscription.next();
|
|
64
|
+
if (!result.done) {
|
|
65
|
+
recordConsumed(1);
|
|
66
|
+
if (inFlight() < cap) drained.wake();
|
|
67
|
+
}
|
|
68
|
+
return result;
|
|
69
|
+
},
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
return { stream, subscription: countedSubscription };
|
|
73
|
+
}
|
package/src/main.ts
CHANGED
|
@@ -23,16 +23,21 @@ import {
|
|
|
23
23
|
import { startMerge, startSync } from "@effectstream/sync";
|
|
24
24
|
import { ComponentNames, log, SeverityNumber } from "@effectstream/log";
|
|
25
25
|
import {
|
|
26
|
-
createChannel,
|
|
27
|
-
each,
|
|
28
26
|
type Operation,
|
|
29
27
|
sleep,
|
|
30
28
|
spawn,
|
|
31
29
|
until,
|
|
32
30
|
} from "effection";
|
|
33
31
|
import { initTelemetry } from "./telemetry.ts";
|
|
34
|
-
import {
|
|
32
|
+
import {
|
|
33
|
+
type PendingEvent,
|
|
34
|
+
processFinalizedBlockWithRetry,
|
|
35
|
+
} from "./process-blocks.ts";
|
|
36
|
+
import { createEmptyBlockCoalescer } from "./coalesce.ts";
|
|
35
37
|
import { startHttpServer } from "./api/http-server.ts";
|
|
38
|
+
import { recordAppliedBlock } from "./api/apply-status.ts";
|
|
39
|
+
import { recordCoalesced } from "./api/stream-status.ts";
|
|
40
|
+
import { createBoundedFinalizedStream } from "./finalized-stream.ts";
|
|
36
41
|
import type { StartConfig } from "./types.ts";
|
|
37
42
|
import type { Client } from "pg";
|
|
38
43
|
import type { EffectstreamBlockHash } from "@effectstream/utils";
|
|
@@ -91,13 +96,20 @@ export function* start(config: StartConfig): Operation<void> {
|
|
|
91
96
|
);
|
|
92
97
|
});
|
|
93
98
|
|
|
99
|
+
// 10× main clock block time (NTP if present, else protocol 0). Undefined disables lag gating.
|
|
94
100
|
const ntpConfig = syncInfo.find(s => s.networkType === ConfigNetworkType.NTP);
|
|
95
|
-
const
|
|
96
|
-
|
|
101
|
+
const clockBlockTimeMS =
|
|
102
|
+
(ntpConfig?.network as { blockTimeMS?: number } | undefined)?.blockTimeMS ??
|
|
103
|
+
(syncInfo[0]?.network as { blockTimeMS?: number } | undefined)?.blockTimeMS;
|
|
104
|
+
const lagThresholdMs = clockBlockTimeMS != null ? clockBlockTimeMS * 10 : undefined;
|
|
97
105
|
|
|
98
|
-
|
|
106
|
+
// Bounded hand-off queue between the merge and the apply loop. Backpressure caps
|
|
107
|
+
// the in-memory queue so deep catch-up can't grow it toward the whole backlog
|
|
108
|
+
// (see finalized-stream.ts / sync/CLAUDE.md Finding #1).
|
|
109
|
+
const { stream: finalizedStream, subscription: finalizedBlocks } =
|
|
110
|
+
yield* createBoundedFinalizedStream(ENV.EFFECTSTREAM_FINALIZED_STREAM_CAP);
|
|
99
111
|
|
|
100
|
-
yield* spawn(() => startMerge(syncProtocols,
|
|
112
|
+
yield* spawn(() => startMerge(syncProtocols, finalizedStream));
|
|
101
113
|
|
|
102
114
|
const heartbeatIntervalMs = 60_000;
|
|
103
115
|
yield* spawn(function* () {
|
|
@@ -139,55 +151,66 @@ export function* start(config: StartConfig): Operation<void> {
|
|
|
139
151
|
yield* spawn(() => runSnapshotLoop(config.snapshotConfig!));
|
|
140
152
|
}
|
|
141
153
|
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
154
|
+
const coalescer = createEmptyBlockCoalescer({
|
|
155
|
+
enabled: ENV.EFFECTSTREAM_COALESCE_EMPTY_BLOCKS,
|
|
156
|
+
subscription: finalizedBlocks,
|
|
157
|
+
pool: dbConn as any, // Pool,
|
|
158
|
+
migrations: config.migrations,
|
|
159
|
+
lagThresholdMs,
|
|
160
|
+
getPreviousBlockHash: () => blockHash,
|
|
161
|
+
onFlush: (endpoint, length) => {
|
|
162
|
+
// `consumed` is counted at the subscription pull point (above), not here, so
|
|
163
|
+
// it reflects the channel depth regardless of how many blocks a run folds.
|
|
164
|
+
recordCoalesced(length);
|
|
165
|
+
recordAppliedBlock(endpoint);
|
|
166
|
+
emitLatestBlocks(
|
|
167
|
+
endpoint.blockNumber,
|
|
168
|
+
endpoint.timestamp,
|
|
169
|
+
getRangesForSyncProtocols(endpoint),
|
|
170
|
+
);
|
|
171
|
+
log.local(
|
|
172
|
+
ComponentNames.EFFECTSTREAM_SYNC,
|
|
173
|
+
"block-merge",
|
|
174
|
+
SeverityNumber.INFO,
|
|
175
|
+
(l) =>
|
|
176
|
+
l(
|
|
177
|
+
`coalesced ${length} empty block(s) → block ${endpoint.blockNumber}`,
|
|
178
|
+
),
|
|
160
179
|
);
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
180
|
+
},
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
while (true) {
|
|
184
|
+
const value = yield* coalescer.advance();
|
|
185
|
+
if (value === undefined) break;
|
|
186
|
+
|
|
187
|
+
// Owns connection checkout, the per-block DB mutex (PGLite), and
|
|
188
|
+
// transient-pg retry/backoff. App events flush below only after it
|
|
189
|
+
// returns — strictly after the block's COMMIT.
|
|
190
|
+
const result = yield* processFinalizedBlockWithRetry(
|
|
191
|
+
value,
|
|
192
|
+
config,
|
|
193
|
+
dbConn as any, // Pool,
|
|
194
|
+
blockHash,
|
|
195
|
+
);
|
|
196
|
+
const blockAppEvents: PendingEvent[] = result.events;
|
|
197
|
+
if (result.blockHash !== "0x0") {
|
|
198
|
+
blockHash = result.blockHash;
|
|
171
199
|
}
|
|
172
200
|
|
|
201
|
+
recordAppliedBlock(value); // apply-stage liveness for /debug/metrics
|
|
202
|
+
|
|
173
203
|
// Used to emit & log the block range for each protocol.
|
|
174
204
|
const contentBlocksForProtocol = getRangesForSyncProtocols(value);
|
|
175
205
|
|
|
176
|
-
// Fire-and-forget:
|
|
206
|
+
// Fire-and-forget: don't stall the sync loop on broker acks. Ordering is
|
|
207
|
+
// safe without a wrapper queue — Opifex assigns packet IDs synchronously in
|
|
208
|
+
// publish() and net.Socket serializes writes in call order, so MQTT 3.1.1
|
|
209
|
+
// §4.6 in-order delivery holds per (publisher, topic, QoS).
|
|
177
210
|
//
|
|
178
|
-
//
|
|
179
|
-
//
|
|
180
|
-
//
|
|
181
|
-
// MQTT 3.1.1 §4.6 then guarantees in-order delivery to each subscriber
|
|
182
|
-
// per (publisher, topic, QoS). We don't need a wrapper queue.
|
|
183
|
-
//
|
|
184
|
-
// TODO(scaling): the broker runs in-process (see `new EventBroker(...)`
|
|
185
|
-
// above). Fan-out to N subscribers consumes the same event loop as block
|
|
186
|
-
// processing. At ~thousands of subscribers, consider:
|
|
187
|
-
// 1) moving the broker to a worker / separate process,
|
|
188
|
-
// 2) a publisher-side circuit breaker if ctx.unresolvedPublish.size > N
|
|
189
|
-
// (drop events with a counter for observability).
|
|
190
|
-
// Today: localhost, in-process, expected O(10²) subscribers — non-issue.
|
|
211
|
+
// TODO(scaling): the broker is in-process, so fan-out shares this event
|
|
212
|
+
// loop. At ~thousands of subscribers, move it to a worker and/or add a
|
|
213
|
+
// publisher-side circuit breaker. Today: localhost, O(10²) subs — non-issue.
|
|
191
214
|
emitLatestBlocks(
|
|
192
215
|
value.blockNumber,
|
|
193
216
|
value.timestamp,
|
|
@@ -220,7 +243,7 @@ export function* start(config: StartConfig): Operation<void> {
|
|
|
220
243
|
}...${lagSuffix} | ${JSON.stringify(contentBlocksForProtocol)}`,
|
|
221
244
|
),
|
|
222
245
|
);
|
|
223
|
-
yield*
|
|
246
|
+
if (config.dev?.applyDelayMs) yield* sleep(config.dev.applyDelayMs);
|
|
224
247
|
}
|
|
225
248
|
}
|
|
226
249
|
|
|
@@ -239,13 +262,9 @@ function getRangesForSyncProtocols(value: ChainBlock): Record<string, [number, n
|
|
|
239
262
|
}
|
|
240
263
|
|
|
241
264
|
/**
|
|
242
|
-
* Publish built-in block-level events.
|
|
243
|
-
*
|
|
244
|
-
*
|
|
245
|
-
* (do not stall the sync loop; ordering preserved by Opifex + MQTT 3.1.1).
|
|
246
|
-
*
|
|
247
|
-
* Errors are caught locally so they don't escape to the global
|
|
248
|
-
* `unhandledrejection` handler without context.
|
|
265
|
+
* Publish built-in block-level events. Fire-and-forget like the app-event flush
|
|
266
|
+
* above (don't stall the sync loop; ordering held by Opifex + MQTT 3.1.1).
|
|
267
|
+
* Errors are caught locally so they don't reach the global handler contextless.
|
|
249
268
|
*/
|
|
250
269
|
function emitLatestBlocks(
|
|
251
270
|
rollUpBlockHeight: number,
|
|
@@ -296,45 +315,48 @@ function* startup(
|
|
|
296
315
|
});
|
|
297
316
|
});
|
|
298
317
|
|
|
299
|
-
|
|
300
318
|
yield* acquireDBMutex(`startup-node`);
|
|
319
|
+
// `finally` releases the mutex on error/cancellation too; a throw in any step
|
|
320
|
+
// below would otherwise leak it and deadlock every later DB operation.
|
|
321
|
+
try {
|
|
322
|
+
// Dev-only reset of user-owned public tables
|
|
323
|
+
if (config.dev?.resetPublicData) {
|
|
324
|
+
yield* resetPublicTables(dbConn as any); // Client,
|
|
325
|
+
}
|
|
301
326
|
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
yield*
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
versionInfo,
|
|
312
|
-
lastBlockHeight,
|
|
313
|
-
dbConn,
|
|
314
|
-
config.migrations,
|
|
315
|
-
);
|
|
327
|
+
// When the node is started, we apply system migrations.
|
|
328
|
+
// Either system initial migrations, or migrations given a Paima Engine Update.
|
|
329
|
+
yield* applySystemMigrations(
|
|
330
|
+
config.appVersion,
|
|
331
|
+
versionInfo,
|
|
332
|
+
lastBlockHeight,
|
|
333
|
+
dbConn,
|
|
334
|
+
config.migrations,
|
|
335
|
+
);
|
|
316
336
|
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
337
|
+
// Validate that immutable config fields (e.g. NTP startTime, startBlockHeight)
|
|
338
|
+
// have not changed since the last run. Persists a snapshot on first start.
|
|
339
|
+
// Must run after system migrations so the snapshot table exists.
|
|
340
|
+
yield* validateAndSnapshotConfig(syncInfo, dbConn);
|
|
321
341
|
|
|
322
|
-
|
|
323
|
-
|
|
342
|
+
const syncProtocols = yield* genSyncProtocols(dbConn as any, // Client,
|
|
343
|
+
syncInfo);
|
|
324
344
|
|
|
325
|
-
|
|
326
|
-
|
|
345
|
+
const capabilities = yield* until(detectCapabilities(dbConn as any));
|
|
346
|
+
const viewStrategy = selectViewStrategy(capabilities);
|
|
327
347
|
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
348
|
+
yield* createDynamicTables(
|
|
349
|
+
versionInfo,
|
|
350
|
+
lastBlockHeight,
|
|
351
|
+
dbConn as any, // Client,
|
|
352
|
+
syncProtocols,
|
|
353
|
+
viewStrategy,
|
|
354
|
+
);
|
|
335
355
|
|
|
336
|
-
|
|
337
|
-
|
|
356
|
+
return syncProtocols;
|
|
357
|
+
} finally {
|
|
358
|
+
releaseDBMutex(`startup-node`);
|
|
359
|
+
}
|
|
338
360
|
}
|
|
339
361
|
|
|
340
362
|
// Convert the primitive config to the final primitive instance
|
package/src/process-blocks.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { ChainBlock } from "@effectstream/sync";
|
|
2
|
-
import { call, type Operation, until } from "effection";
|
|
3
|
-
import type { Pool } from "pg";
|
|
2
|
+
import { call, type Operation, sleep, until } from "effection";
|
|
3
|
+
import type { Pool, PoolClient } from "pg";
|
|
4
4
|
import { type BaseStfInput, type EmitFn, primitiveTransitionFunction } from "@effectstream/sm";
|
|
5
5
|
import { PreparedQuery } from "@pgtyped/runtime";
|
|
6
6
|
import type {
|
|
@@ -9,19 +9,24 @@ import type {
|
|
|
9
9
|
SyncStateUpdateStream,
|
|
10
10
|
} from "@effectstream/coroutine";
|
|
11
11
|
import {
|
|
12
|
+
acquireDBMutex,
|
|
12
13
|
blockHeightDone,
|
|
13
14
|
deleteEmptyBlocks,
|
|
14
15
|
deleteScheduled,
|
|
15
16
|
getFutureGameInputByBlockHeight,
|
|
16
17
|
getFutureGameInputByMaxTimestamp,
|
|
17
18
|
insertGameInputResult,
|
|
19
|
+
isTransientPgError,
|
|
20
|
+
poolErrors,
|
|
21
|
+
releaseDBMutex,
|
|
18
22
|
removePages,
|
|
19
23
|
saveLastBlock,
|
|
24
|
+
upsertPage,
|
|
20
25
|
} from "@effectstream/db";
|
|
21
26
|
import { Buffer } from "node:buffer";
|
|
22
27
|
import { ComponentNames, log, SeverityNumber } from "@effectstream/log";
|
|
23
28
|
import type { StartConfig } from "./types.ts";
|
|
24
|
-
import type {
|
|
29
|
+
import type { EffectstreamBlockHash } from "@effectstream/utils";
|
|
25
30
|
import { generateEffectstreamBlockHash, Prando } from "@effectstream/crypto";
|
|
26
31
|
import { applyUserMigrations } from "./version-migrations.ts";
|
|
27
32
|
import type { EventPathAndDef } from "@effectstream/event-client";
|
|
@@ -308,36 +313,53 @@ export function* processFinalizedBlock(
|
|
|
308
313
|
}, dbConn)
|
|
309
314
|
);
|
|
310
315
|
|
|
311
|
-
/* STEP 7:
|
|
312
|
-
*
|
|
313
|
-
*
|
|
314
|
-
*
|
|
316
|
+
/* STEP 7: Persist the block-accurate resume marker per protocol, inside this
|
|
317
|
+
* block's transaction (the runtime is the sole writer of the resume
|
|
318
|
+
* position). Upsert at page_number = ownBlockNumber, then prune
|
|
319
|
+
* everything below → one row per protocol = the committed watermark.
|
|
320
|
+
* Full mechanism: @effectstream/sync CLAUDE.md, design idea #5.
|
|
315
321
|
*/
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
)
|
|
324
|
-
latestBlocks[blockInfo.protocol_name] = blockInfo.block_number;
|
|
325
|
-
}
|
|
326
|
-
}
|
|
327
|
-
|
|
328
|
-
for (const [protocolName, blockNumber] of Object.entries(latestBlocks)) {
|
|
322
|
+
for (const { protocol_name, lastPage } of value.resumePages) {
|
|
323
|
+
yield* until(
|
|
324
|
+
upsertPage.run({
|
|
325
|
+
protocol_name,
|
|
326
|
+
page_number: lastPage.ownBlockNumber,
|
|
327
|
+
page: JSON.stringify(lastPage),
|
|
328
|
+
}, dbConn),
|
|
329
|
+
);
|
|
329
330
|
yield* until(
|
|
330
331
|
removePages.run({
|
|
331
|
-
protocol_name
|
|
332
|
-
page_number:
|
|
332
|
+
protocol_name,
|
|
333
|
+
page_number: lastPage.ownBlockNumber,
|
|
333
334
|
}, dbConn),
|
|
334
335
|
);
|
|
335
336
|
}
|
|
336
337
|
|
|
337
338
|
/* STEP 8: Commit the transaction. */
|
|
338
339
|
yield* until(dbConn.query("COMMIT"));
|
|
340
|
+
// A successful COMMIT proves the DB is reachable end-to-end — reset the
|
|
341
|
+
// pool failure clock so the write path also feeds the health tracker
|
|
342
|
+
// (it otherwise only sees floating pool errors and the read-side API).
|
|
343
|
+
poolErrors.markHealthy();
|
|
339
344
|
} catch (err) {
|
|
340
|
-
|
|
345
|
+
// Best-effort rollback. If the socket is dead the ROLLBACK throws too;
|
|
346
|
+
// swallow that so it doesn't mask the original error — the caller's retry
|
|
347
|
+
// loop keys off the original (transient) error, not a rollback failure.
|
|
348
|
+
try {
|
|
349
|
+
yield* until(dbConn.query("ROLLBACK"));
|
|
350
|
+
} catch (rollbackErr) {
|
|
351
|
+
log.remote(
|
|
352
|
+
ComponentNames.EFFECTSTREAM_SYNC,
|
|
353
|
+
"block-processing",
|
|
354
|
+
SeverityNumber.WARN,
|
|
355
|
+
(log) =>
|
|
356
|
+
log(
|
|
357
|
+
`ROLLBACK failed for block ${value.blockNumber} (connection likely dead): ${
|
|
358
|
+
String(rollbackErr)
|
|
359
|
+
}`,
|
|
360
|
+
),
|
|
361
|
+
);
|
|
362
|
+
}
|
|
341
363
|
log.remote(
|
|
342
364
|
ComponentNames.EFFECTSTREAM_SYNC,
|
|
343
365
|
"block-processing",
|
|
@@ -345,12 +367,89 @@ export function* processFinalizedBlock(
|
|
|
345
367
|
(log) =>
|
|
346
368
|
log(`Error processing block ${value.blockNumber}: ${String(err)}`),
|
|
347
369
|
);
|
|
348
|
-
//
|
|
370
|
+
// Re-throw: `processFinalizedBlockWithRetry` decides whether this is a
|
|
371
|
+
// retryable transient pg failure or a fatal bug.
|
|
349
372
|
throw err;
|
|
350
373
|
}
|
|
351
374
|
|
|
352
|
-
return {
|
|
353
|
-
blockHash: hasOperations ? blockHash : "0x0" as PaimaBlockHash,
|
|
354
|
-
events: blockEvents
|
|
375
|
+
return {
|
|
376
|
+
blockHash: hasOperations ? blockHash : "0x0" as PaimaBlockHash,
|
|
377
|
+
events: blockEvents
|
|
355
378
|
};
|
|
356
379
|
}
|
|
380
|
+
|
|
381
|
+
/** Caps the exponential backoff between block-retry attempts. */
|
|
382
|
+
const MAX_RETRY_BACKOFF_MS = 5_000;
|
|
383
|
+
|
|
384
|
+
/**
|
|
385
|
+
* Run {@link processFinalizedBlock} against a freshly checked-out pool client,
|
|
386
|
+
* retrying the whole block on transient pg failures (Neon cold-starts,
|
|
387
|
+
* idle-socket evictions, TLS blips).
|
|
388
|
+
*
|
|
389
|
+
* The previous socket is dead after such a failure, so each attempt
|
|
390
|
+
* re-acquires a client and reprocesses the block from `BEGIN` — safe because
|
|
391
|
+
* the failed tx never committed and the caller only advances the block stream
|
|
392
|
+
* after this returns.
|
|
393
|
+
*
|
|
394
|
+
* `poolErrors` governs escalation: it logs WARN→ERROR as the outage persists
|
|
395
|
+
* and calls `process.exit(1)` once it crosses the fatal threshold, letting the
|
|
396
|
+
* orchestrator restart us. A non-transient error (a real SQL/logic bug) is
|
|
397
|
+
* re-thrown and crashes, as before.
|
|
398
|
+
*
|
|
399
|
+
* Owns the per-block DB mutex (for PGLite) and client lifecycle so the caller
|
|
400
|
+
* doesn't have to.
|
|
401
|
+
*/
|
|
402
|
+
export function* processFinalizedBlockWithRetry(
|
|
403
|
+
value: ChainBlock,
|
|
404
|
+
config: StartConfig,
|
|
405
|
+
pool: Pool,
|
|
406
|
+
previousBlockHash: EffectstreamBlockHash | null,
|
|
407
|
+
): Operation<ProcessFinalizedBlockResult> {
|
|
408
|
+
const lockName = `processing-blocks:${value.blockNumber}`;
|
|
409
|
+
let attempt = 0;
|
|
410
|
+
while (true) {
|
|
411
|
+
let dbClient: PoolClient | undefined;
|
|
412
|
+
let transientErr: unknown;
|
|
413
|
+
try {
|
|
414
|
+
yield* acquireDBMutex(lockName);
|
|
415
|
+
dbClient = yield* until(pool.connect());
|
|
416
|
+
return yield* processFinalizedBlock(
|
|
417
|
+
value,
|
|
418
|
+
config,
|
|
419
|
+
dbClient as unknown as Pool, // a checked-out client, used as the tx conn
|
|
420
|
+
previousBlockHash,
|
|
421
|
+
);
|
|
422
|
+
} catch (err) {
|
|
423
|
+
if (!isTransientPgError(err)) throw err;
|
|
424
|
+
transientErr = err;
|
|
425
|
+
} finally {
|
|
426
|
+
releaseDBMutex(lockName);
|
|
427
|
+
if (dbClient) {
|
|
428
|
+
// Passing the error destroys the client instead of recycling it, so
|
|
429
|
+
// the pool never hands the dead socket back out. On success
|
|
430
|
+
// `transientErr` is undefined and the client returns to the pool.
|
|
431
|
+
try {
|
|
432
|
+
dbClient.release(transientErr as Error | undefined);
|
|
433
|
+
} catch {
|
|
434
|
+
/* already released/destroyed */
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
attempt++;
|
|
440
|
+
poolErrors.log(transientErr, ["sync-write"]);
|
|
441
|
+
const backoffMs = Math.min(500 * 2 ** (attempt - 1), MAX_RETRY_BACKOFF_MS);
|
|
442
|
+
log.local(
|
|
443
|
+
ComponentNames.EFFECTSTREAM_SYNC,
|
|
444
|
+
"block-processing",
|
|
445
|
+
SeverityNumber.WARN,
|
|
446
|
+
(l) =>
|
|
447
|
+
l(
|
|
448
|
+
`transient db error on block ${value.blockNumber}; retry #${attempt} in ${backoffMs}ms: ${
|
|
449
|
+
String(transientErr)
|
|
450
|
+
}`,
|
|
451
|
+
),
|
|
452
|
+
);
|
|
453
|
+
yield* sleep(backoffMs);
|
|
454
|
+
}
|
|
455
|
+
}
|
package/src/types.ts
CHANGED
|
@@ -79,5 +79,12 @@ export type StartConfig = {
|
|
|
79
79
|
* state such as `bufferedData.size()`. No effect in production.
|
|
80
80
|
*/
|
|
81
81
|
onStarted?: (ctx: { syncProtocols: AllSyncProtocols[] }) => void;
|
|
82
|
+
/**
|
|
83
|
+
* Test/diagnostic only: sleep this many ms after each applied block, slowing
|
|
84
|
+
* the drain so the per-chain fetch buffers fill to their cap and the
|
|
85
|
+
* backpressure fix becomes observable (otherwise drain ≈ fetch and the buffer
|
|
86
|
+
* never grows). No effect in production (leave unset / 0).
|
|
87
|
+
*/
|
|
88
|
+
applyDelayMs?: number;
|
|
82
89
|
};
|
|
83
90
|
};
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { expect, test } from "bun:test";
|
|
2
|
+
import {
|
|
3
|
+
type EmptyRunBoundaries,
|
|
4
|
+
isCoalescableEmptyBlock,
|
|
5
|
+
} from "../src/coalesce.ts";
|
|
6
|
+
|
|
7
|
+
const NO_BOUNDARIES: EmptyRunBoundaries = {
|
|
8
|
+
minScheduledBlockHeight: null,
|
|
9
|
+
minScheduledTimestampMs: null,
|
|
10
|
+
};
|
|
11
|
+
|
|
12
|
+
function block(
|
|
13
|
+
blockNumber: number,
|
|
14
|
+
timestamp: number,
|
|
15
|
+
primitives: unknown[] = [],
|
|
16
|
+
): any {
|
|
17
|
+
return { blockNumber, timestamp, blockInfo: [], resumePages: [], primitives };
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
test("empty block with no boundaries is coalescable", () => {
|
|
21
|
+
expect(isCoalescableEmptyBlock(block(10, 10_000), NO_BOUNDARIES, undefined))
|
|
22
|
+
.toBe(true);
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
test("block with on-chain content is NOT coalescable", () => {
|
|
26
|
+
expect(
|
|
27
|
+
isCoalescableEmptyBlock(block(10, 10_000, [{}]), NO_BOUNDARIES, undefined),
|
|
28
|
+
).toBe(false);
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
test("block where a migration fires is NOT coalescable", () => {
|
|
32
|
+
const migrations = [{ name: "m1", sql: "", blockHeight: 10 }];
|
|
33
|
+
expect(isCoalescableEmptyBlock(block(10, 10_000), NO_BOUNDARIES, migrations))
|
|
34
|
+
.toBe(false);
|
|
35
|
+
expect(isCoalescableEmptyBlock(block(9, 9_000), NO_BOUNDARIES, migrations))
|
|
36
|
+
.toBe(true);
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
test("block at/after a pending block-height-scheduled input is NOT coalescable", () => {
|
|
40
|
+
const boundaries: EmptyRunBoundaries = {
|
|
41
|
+
minScheduledBlockHeight: 10,
|
|
42
|
+
minScheduledTimestampMs: null,
|
|
43
|
+
};
|
|
44
|
+
expect(isCoalescableEmptyBlock(block(9, 9_000), boundaries, undefined)).toBe(
|
|
45
|
+
true,
|
|
46
|
+
);
|
|
47
|
+
expect(isCoalescableEmptyBlock(block(10, 10_000), boundaries, undefined)).toBe(
|
|
48
|
+
false,
|
|
49
|
+
);
|
|
50
|
+
expect(isCoalescableEmptyBlock(block(11, 11_000), boundaries, undefined)).toBe(
|
|
51
|
+
false,
|
|
52
|
+
);
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
test("block at/after a due timestamp-scheduled input is NOT coalescable", () => {
|
|
56
|
+
const boundaries: EmptyRunBoundaries = {
|
|
57
|
+
minScheduledBlockHeight: null,
|
|
58
|
+
minScheduledTimestampMs: 10_000,
|
|
59
|
+
};
|
|
60
|
+
expect(isCoalescableEmptyBlock(block(9, 9_999), boundaries, undefined)).toBe(
|
|
61
|
+
true,
|
|
62
|
+
);
|
|
63
|
+
expect(isCoalescableEmptyBlock(block(10, 10_000), boundaries, undefined)).toBe(
|
|
64
|
+
false,
|
|
65
|
+
);
|
|
66
|
+
expect(isCoalescableEmptyBlock(block(11, 10_001), boundaries, undefined)).toBe(
|
|
67
|
+
false,
|
|
68
|
+
);
|
|
69
|
+
});
|