@rocicorp/zero 1.10.0-head-23da5f1c-20260822 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/shared/src/promise-race.d.ts +14 -0
- package/out/shared/src/promise-race.d.ts.map +1 -1
- package/out/shared/src/promise-race.js +32 -1
- package/out/shared/src/promise-race.js.map +1 -1
- package/out/zero/package.js +1 -1
- package/out/zero/package.js.map +1 -1
- package/out/zero-cache/src/config/normalize.d.ts +5 -0
- package/out/zero-cache/src/config/normalize.d.ts.map +1 -1
- package/out/zero-cache/src/config/normalize.js +10 -2
- package/out/zero-cache/src/config/normalize.js.map +1 -1
- package/out/zero-cache/src/config/zero-config.d.ts +3 -1
- package/out/zero-cache/src/config/zero-config.d.ts.map +1 -1
- package/out/zero-cache/src/config/zero-config.js +10 -12
- package/out/zero-cache/src/config/zero-config.js.map +1 -1
- package/out/zero-cache/src/db/migration-lite.d.ts.map +1 -1
- package/out/zero-cache/src/db/migration-lite.js +5 -3
- package/out/zero-cache/src/db/migration-lite.js.map +1 -1
- package/out/zero-cache/src/db/transaction-pool.d.ts +0 -13
- package/out/zero-cache/src/db/transaction-pool.d.ts.map +1 -1
- package/out/zero-cache/src/db/transaction-pool.js +34 -70
- package/out/zero-cache/src/db/transaction-pool.js.map +1 -1
- package/out/zero-cache/src/server/change-streamer.d.ts.map +1 -1
- package/out/zero-cache/src/server/change-streamer.js +1 -1
- package/out/zero-cache/src/server/change-streamer.js.map +1 -1
- package/out/zero-cache/src/server/main.d.ts.map +1 -1
- package/out/zero-cache/src/server/main.js +4 -4
- package/out/zero-cache/src/server/main.js.map +1 -1
- package/out/zero-cache/src/server/replicator.d.ts.map +1 -1
- package/out/zero-cache/src/server/replicator.js +2 -2
- package/out/zero-cache/src/server/replicator.js.map +1 -1
- package/out/zero-cache/src/services/analyze.d.ts.map +1 -1
- package/out/zero-cache/src/services/analyze.js +1 -0
- package/out/zero-cache/src/services/analyze.js.map +1 -1
- package/out/zero-cache/src/services/change-source/common/backfill-manager.d.ts +12 -2
- package/out/zero-cache/src/services/change-source/common/backfill-manager.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/common/backfill-manager.js +14 -3
- package/out/zero-cache/src/services/change-source/common/backfill-manager.js.map +1 -1
- package/out/zero-cache/src/services/change-source/common/replica-schema.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/common/replica-schema.js +10 -41
- package/out/zero-cache/src/services/change-source/common/replica-schema.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/backfill-stream.d.ts +3 -2
- package/out/zero-cache/src/services/change-source/pg/backfill-stream.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/backfill-stream.js +41 -15
- package/out/zero-cache/src/services/change-source/pg/backfill-stream.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/change-source.d.ts +23 -1
- package/out/zero-cache/src/services/change-source/pg/change-source.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/change-source.js +25 -16
- package/out/zero-cache/src/services/change-source/pg/change-source.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/broadcast.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/broadcast.js +4 -1
- package/out/zero-cache/src/services/change-streamer/broadcast.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js +16 -9
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/forwarder.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/forwarder.js +12 -3
- package/out/zero-cache/src/services/change-streamer/forwarder.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/litestream3-prometheus-poller.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/litestream3-prometheus-poller.js +11 -5
- package/out/zero-cache/src/services/change-streamer/litestream3-prometheus-poller.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/storer.d.ts +40 -1
- package/out/zero-cache/src/services/change-streamer/storer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/storer.js +179 -51
- package/out/zero-cache/src/services/change-streamer/storer.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/subscriber.d.ts +8 -1
- package/out/zero-cache/src/services/change-streamer/subscriber.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/subscriber.js +10 -1
- package/out/zero-cache/src/services/change-streamer/subscriber.js.map +1 -1
- package/out/zero-cache/src/services/litestream/commands.d.ts.map +1 -1
- package/out/zero-cache/src/services/litestream/commands.js +8 -4
- package/out/zero-cache/src/services/litestream/commands.js.map +1 -1
- package/out/zero-cache/src/services/replicator/change-log-db.js.map +1 -1
- package/out/zero-cache/src/services/replicator/change-processor.js +2 -5
- package/out/zero-cache/src/services/replicator/change-processor.js.map +1 -1
- package/out/zero-cache/src/services/replicator/incremental-sync.d.ts +1 -1
- package/out/zero-cache/src/services/replicator/incremental-sync.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/incremental-sync.js +1 -6
- package/out/zero-cache/src/services/replicator/incremental-sync.js.map +1 -1
- package/out/zero-cache/src/services/replicator/replicator.d.ts +1 -1
- package/out/zero-cache/src/services/replicator/replicator.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/replicator.js +2 -2
- package/out/zero-cache/src/services/replicator/replicator.js.map +1 -1
- package/out/zero-cache/src/types/pg.d.ts +15 -2
- package/out/zero-cache/src/types/pg.d.ts.map +1 -1
- package/out/zero-cache/src/types/pg.js +7 -4
- package/out/zero-cache/src/types/pg.js.map +1 -1
- package/out/zero-cache/src/types/streams.d.ts +4 -9
- package/out/zero-cache/src/types/streams.d.ts.map +1 -1
- package/out/zero-cache/src/types/streams.js.map +1 -1
- package/out/zero-cache/src/types/subscription.d.ts +2 -11
- package/out/zero-cache/src/types/subscription.d.ts.map +1 -1
- package/out/zero-cache/src/types/subscription.js +4 -25
- package/out/zero-cache/src/types/subscription.js.map +1 -1
- package/out/zero-client/src/client/inspector/lazy-inspector.d.ts.map +1 -1
- package/out/zero-client/src/client/inspector/lazy-inspector.js +17 -10
- package/out/zero-client/src/client/inspector/lazy-inspector.js.map +1 -1
- package/out/zero-client/src/client/mutation-tracker.d.ts.map +1 -1
- package/out/zero-client/src/client/mutation-tracker.js +1 -1
- package/out/zero-client/src/client/mutation-tracker.js.map +1 -1
- package/out/zero-client/src/client/version.js +1 -1
- package/out/zero-protocol/src/custom-queries.d.ts +1 -1
- package/out/zero-protocol/src/error.d.ts +4 -4
- package/out/zero-protocol/src/inspect-down.js +1 -1
- package/out/zero-protocol/src/mutate-server.d.ts +2 -2
- package/out/zero-protocol/src/push.d.ts +1 -1
- package/out/zero-protocol/src/query-server.d.ts +2 -2
- package/package.json +1 -1
|
@@ -2,6 +2,7 @@ import { assert, unreachable } from "../../../../shared/src/asserts.js";
|
|
|
2
2
|
import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
|
|
3
3
|
import { AbortError } from "../../../../shared/src/abort-error.js";
|
|
4
4
|
import { parse, valita_exports } from "../../../../shared/src/valita.js";
|
|
5
|
+
import { BigIntJSON } from "../../../../shared/src/bigint-json.js";
|
|
5
6
|
import { cdcSchema } from "../../types/shards.js";
|
|
6
7
|
import { isDataChange, isSchemaChange } from "../change-source/protocol/current/data.js";
|
|
7
8
|
import "../change-source/protocol/current/downstream.js";
|
|
@@ -13,7 +14,7 @@ import { READONLY, READ_COMMITTED } from "../../db/mode-enum.js";
|
|
|
13
14
|
import { runTx } from "../../db/run-transaction.js";
|
|
14
15
|
import { Queue } from "../../../../shared/src/queue.js";
|
|
15
16
|
import { AutoResetSignal, markResetRequired } from "./schema/tables.js";
|
|
16
|
-
import { TransactionPool } from "../../db/transaction-pool.js";
|
|
17
|
+
import { TransactionPool, sharedSnapshot } from "../../db/transaction-pool.js";
|
|
17
18
|
import { orTimeout } from "../../types/timeout.js";
|
|
18
19
|
import { extractChangeSubstring, reconstructWatermarkedChange, serializeChangeStreamData } from "./change-log-codec.js";
|
|
19
20
|
import { resolver } from "@rocicorp/resolver";
|
|
@@ -58,7 +59,10 @@ var Storer = class {
|
|
|
58
59
|
#taskID;
|
|
59
60
|
#discoveryAddress;
|
|
60
61
|
#discoveryProtocol;
|
|
62
|
+
#makeConnectionPool;
|
|
61
63
|
#db;
|
|
64
|
+
#inserter;
|
|
65
|
+
#purger;
|
|
62
66
|
#replicaVersion;
|
|
63
67
|
#onCommitted;
|
|
64
68
|
#onFatal;
|
|
@@ -67,21 +71,26 @@ var Storer = class {
|
|
|
67
71
|
#statementTimeoutMs;
|
|
68
72
|
#changeLogBatchSize;
|
|
69
73
|
#drainTimeoutMs;
|
|
74
|
+
#progressMonitor;
|
|
70
75
|
#approximateQueuedBytes = 0;
|
|
71
76
|
#running = false;
|
|
72
|
-
constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol,
|
|
77
|
+
constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol, dbProvider, replicaVersion, onCommitted, onFatal, { backPressureLimitHeapProportion, statementTimeoutMs, changeLogBatchSize, drainTimeoutMs = 3e4 }) {
|
|
73
78
|
this.#lc = lc.withContext("component", "change-log");
|
|
74
79
|
this.#shard = shard;
|
|
75
80
|
this.#taskID = taskID;
|
|
76
81
|
this.#discoveryAddress = discoveryAddress;
|
|
77
82
|
this.#discoveryProtocol = discoveryProtocol;
|
|
78
|
-
this.#
|
|
83
|
+
this.#makeConnectionPool = dbProvider;
|
|
84
|
+
this.#db = dbProvider("change-stream-init", 3);
|
|
85
|
+
this.#inserter = dbProvider("change-log-inserter", 1);
|
|
86
|
+
this.#purger = dbProvider("change-log-purger", 1);
|
|
79
87
|
this.#replicaVersion = replicaVersion;
|
|
80
88
|
this.#onCommitted = onCommitted;
|
|
81
89
|
this.#onFatal = onFatal;
|
|
82
90
|
this.#statementTimeoutMs = statementTimeoutMs;
|
|
83
91
|
this.#changeLogBatchSize = Math.max(1, changeLogBatchSize);
|
|
84
92
|
this.#drainTimeoutMs = drainTimeoutMs;
|
|
93
|
+
this.#progressMonitor = new ProgressMonitor(lc, statementTimeoutMs, onFatal);
|
|
85
94
|
const heapStats = getHeapStatistics();
|
|
86
95
|
this.#backPressureThresholdBytes = (heapStats.heap_size_limit - heapStats.used_heap_size) * backPressureLimitHeapProportion;
|
|
87
96
|
this.#lc.info?.(`Using up to ${(this.#backPressureThresholdBytes / 1024 ** 2).toFixed(2)} MB of --max-old-space-size (~${(heapStats.heap_size_limit / 1024 ** 2).toFixed(2)} MB) to absorb upstream spikes`, { heapStats });
|
|
@@ -89,6 +98,28 @@ var Storer = class {
|
|
|
89
98
|
#cdc(table) {
|
|
90
99
|
return this.#db(`${cdcSchema(this.#shard)}.${table}`);
|
|
91
100
|
}
|
|
101
|
+
/**
|
|
102
|
+
* Bounds a one-off db call (i.e. not part of the main storer loop or the
|
|
103
|
+
* background catchup read, which are tracked by the ProgressMonitor
|
|
104
|
+
* instead) with a plain timeout. This covers calls like
|
|
105
|
+
* {@link assumeOwnership} and {@link getStartStreamInitializationParameters}
|
|
106
|
+
* that are made by the caller *before* {@link run()} -- and thus before the
|
|
107
|
+
* ProgressMonitor's polling starts -- as well as ones made well after,
|
|
108
|
+
* where a continuously-polling watchdog would be overkill for a single
|
|
109
|
+
* bounded db round trip.
|
|
110
|
+
*
|
|
111
|
+
* Does not cancel `promise` on timeout (there is no cancellation mechanism
|
|
112
|
+
* for a postgres.js query); it just stops waiting for it, and swallows a
|
|
113
|
+
* later rejection so it doesn't surface as an unhandled rejection.
|
|
114
|
+
*/
|
|
115
|
+
async #withTimeout(name, promise) {
|
|
116
|
+
const result = await orTimeout(promise, this.#statementTimeoutMs);
|
|
117
|
+
if (result === "timed-out") {
|
|
118
|
+
promise.catch(() => {});
|
|
119
|
+
throw new AbortError(`${name} did not complete within ${this.#statementTimeoutMs}ms`);
|
|
120
|
+
}
|
|
121
|
+
return result;
|
|
122
|
+
}
|
|
92
123
|
async assumeOwnership(purgeLock) {
|
|
93
124
|
const db = this.#db;
|
|
94
125
|
const owner = this.#taskID;
|
|
@@ -97,10 +128,10 @@ var Storer = class {
|
|
|
97
128
|
const addressWithProtocol = ownerProtocol === "ws" ? ownerAddress : `${ownerProtocol}://${ownerAddress}`;
|
|
98
129
|
this.#lc.info?.(`assuming ownership at ${addressWithProtocol}`);
|
|
99
130
|
const start = performance.now();
|
|
100
|
-
await db`UPDATE ${this.#cdc("replicationState")} SET ${db({
|
|
131
|
+
await this.#withTimeout("assume-ownership", db`UPDATE ${this.#cdc("replicationState")} SET ${db({
|
|
101
132
|
owner,
|
|
102
133
|
ownerAddress: addressWithProtocol
|
|
103
|
-
})}
|
|
134
|
+
})}`);
|
|
104
135
|
const elapsed = (performance.now() - start).toFixed(2);
|
|
105
136
|
this.#lc.info?.(`assumed ownership at ${addressWithProtocol} (${elapsed} ms)`);
|
|
106
137
|
if (purgeLock) purgeLock.release();
|
|
@@ -123,7 +154,7 @@ var Storer = class {
|
|
|
123
154
|
* snapshot that the SQLite change log continues folding onto.
|
|
124
155
|
*/
|
|
125
156
|
async getStartStreamInitializationParameters() {
|
|
126
|
-
const [[{ lastWatermark }], result, tableMetadata, backfilling] = await runTx(this.#db, (sql) => [
|
|
157
|
+
const [[{ lastWatermark }], result, tableMetadata, backfilling] = await this.#withTimeout("get-stream-params", runTx(this.#db, (sql) => [
|
|
127
158
|
sql`
|
|
128
159
|
SELECT "lastWatermark" FROM ${this.#cdc("replicationState")}`,
|
|
129
160
|
sql`
|
|
@@ -146,7 +177,7 @@ var Storer = class {
|
|
|
146
177
|
sql`
|
|
147
178
|
SELECT "schema", "table", "column", "backfill" FROM ${this.#cdc("backfilling")}
|
|
148
179
|
ORDER BY "schema" COLLATE "C", "table" COLLATE "C", "column" COLLATE "C"`
|
|
149
|
-
], { mode: READONLY });
|
|
180
|
+
], { mode: READONLY }));
|
|
150
181
|
return {
|
|
151
182
|
lastWatermark,
|
|
152
183
|
backfillRequests: parse(result, backfillRequestsSchema),
|
|
@@ -157,8 +188,8 @@ var Storer = class {
|
|
|
157
188
|
};
|
|
158
189
|
}
|
|
159
190
|
async getMinWatermarkForCatchup() {
|
|
160
|
-
const [{ minWatermark }] = await this.#db`
|
|
161
|
-
SELECT min(watermark) as "minWatermark" FROM ${this.#cdc("changeLog")}
|
|
191
|
+
const [{ minWatermark }] = await this.#withTimeout("get-min-watermark", this.#db`
|
|
192
|
+
SELECT min(watermark) as "minWatermark" FROM ${this.#cdc("changeLog")}`);
|
|
162
193
|
return minWatermark;
|
|
163
194
|
}
|
|
164
195
|
/**
|
|
@@ -167,10 +198,10 @@ var Storer = class {
|
|
|
167
198
|
* `lastWatermark` is the durable head.
|
|
168
199
|
*/
|
|
169
200
|
async getCatchupBounds() {
|
|
170
|
-
const [bounds] = await this.#db`
|
|
201
|
+
const [bounds] = await this.#withTimeout("get-catchup-bounds", this.#db`
|
|
171
202
|
SELECT
|
|
172
203
|
(SELECT min(watermark) FROM ${this.#cdc("changeLog")}) as "minWatermark",
|
|
173
|
-
(SELECT "lastWatermark" FROM ${this.#cdc("replicationState")}) as "lastWatermark"
|
|
204
|
+
(SELECT "lastWatermark" FROM ${this.#cdc("replicationState")}) as "lastWatermark"`);
|
|
174
205
|
return bounds;
|
|
175
206
|
}
|
|
176
207
|
/**
|
|
@@ -202,7 +233,7 @@ var Storer = class {
|
|
|
202
233
|
ORDER BY watermark, pos`.cursor(batchRows);
|
|
203
234
|
}
|
|
204
235
|
purgeRecordsBefore(watermark) {
|
|
205
|
-
return runTx(this.#
|
|
236
|
+
return runTx(this.#purger, async (sql) => {
|
|
206
237
|
await sql`
|
|
207
238
|
SELECT watermark FROM ${this.#cdc("changeLog")}
|
|
208
239
|
ORDER BY watermark, pos LIMIT 1
|
|
@@ -254,30 +285,22 @@ var Storer = class {
|
|
|
254
285
|
}]);
|
|
255
286
|
}
|
|
256
287
|
#readyForMore = null;
|
|
257
|
-
#backpressureTimeout;
|
|
258
288
|
readyForMore() {
|
|
259
289
|
if (!this.#running) return;
|
|
260
290
|
if (this.#readyForMore === null && this.#approximateQueuedBytes > this.#backPressureThresholdBytes) {
|
|
261
291
|
this.#lc.warn?.(`applying back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)\n\nTo inspect changeLog backlog in your change DB:\n SELECT\n (change->'relation'->>'schema') || '.' || (change->'relation'->>'name') AS table_name,\n change->>'tag' AS operation,\n COUNT(*) AS count\n FROM "<app_id>/cdc"."changeLog"\n GROUP BY 1, 2\n ORDER BY 3 DESC\n LIMIT 20;`);
|
|
262
292
|
this.#readyForMore = resolver();
|
|
263
|
-
this.#resetBackpressureTimeout(true);
|
|
264
293
|
}
|
|
265
294
|
return this.#readyForMore?.promise;
|
|
266
295
|
}
|
|
267
|
-
#resetBackpressureTimeout(reschedule) {
|
|
268
|
-
clearTimeout(this.#backpressureTimeout);
|
|
269
|
-
this.#backpressureTimeout = reschedule ? setTimeout(() => {
|
|
270
|
-
this.#readyForMore?.reject(new AbortError(`No statement processed within the last ${this.#statementTimeoutMs}ms. Considering the change-log to be wedged.`));
|
|
271
|
-
this.#readyForMore = null;
|
|
272
|
-
}, this.#statementTimeoutMs) : void 0;
|
|
273
|
-
}
|
|
274
296
|
#maybeReleaseBackPressure() {
|
|
275
|
-
if (this.#readyForMore !== null)
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
297
|
+
if (this.#readyForMore !== null) {
|
|
298
|
+
if (this.#approximateQueuedBytes < this.#backPressureThresholdBytes * .8) {
|
|
299
|
+
this.#lc.info?.(`releasing back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)`);
|
|
300
|
+
this.#readyForMore.resolve();
|
|
301
|
+
this.#readyForMore = null;
|
|
302
|
+
}
|
|
303
|
+
}
|
|
281
304
|
}
|
|
282
305
|
/**
|
|
283
306
|
* Flushes any buffered {@link PendingTransaction.batch} rows to the changeLog.
|
|
@@ -316,6 +339,7 @@ var Storer = class {
|
|
|
316
339
|
this.#running = true;
|
|
317
340
|
this.#stopped = stopped;
|
|
318
341
|
this.#lc.info?.("starting storer");
|
|
342
|
+
this.#progressMonitor.start();
|
|
319
343
|
let err;
|
|
320
344
|
try {
|
|
321
345
|
await this.#processQueue();
|
|
@@ -327,7 +351,6 @@ var Storer = class {
|
|
|
327
351
|
this.#readyForMore.resolve();
|
|
328
352
|
this.#readyForMore = null;
|
|
329
353
|
}
|
|
330
|
-
this.#resetBackpressureTimeout(false);
|
|
331
354
|
this.#cancelQueueEntries(this.#queue.drain().filter((entry) => entry !== void 0), err);
|
|
332
355
|
this.#running = false;
|
|
333
356
|
signalStopped();
|
|
@@ -353,9 +376,20 @@ var Storer = class {
|
|
|
353
376
|
async #processQueue() {
|
|
354
377
|
let tx = null;
|
|
355
378
|
let msg;
|
|
379
|
+
let lastTaskDone;
|
|
380
|
+
const nextMessage = async () => {
|
|
381
|
+
lastTaskDone?.();
|
|
382
|
+
lastTaskDone = void 0;
|
|
383
|
+
const entry = await this.#queue.dequeue();
|
|
384
|
+
if (entry !== "stop") lastTaskDone = this.#progressMonitor.trackTask({
|
|
385
|
+
task: "queue-entry",
|
|
386
|
+
type: entry[0]
|
|
387
|
+
});
|
|
388
|
+
return entry;
|
|
389
|
+
};
|
|
356
390
|
const catchupQueue = [];
|
|
357
391
|
try {
|
|
358
|
-
while ((msg = await
|
|
392
|
+
while ((msg = await nextMessage()) !== "stop") {
|
|
359
393
|
const [msgType] = msg;
|
|
360
394
|
switch (msgType) {
|
|
361
395
|
case "ready": {
|
|
@@ -385,10 +419,7 @@ var Storer = class {
|
|
|
385
419
|
const { promise, resolve, reject } = resolver();
|
|
386
420
|
promise.catch(() => {});
|
|
387
421
|
tx = {
|
|
388
|
-
pool: new TransactionPool(this.#lc.withContext("watermark", watermark), {
|
|
389
|
-
mode: READ_COMMITTED,
|
|
390
|
-
statementResponseTimeout: this.#statementTimeoutMs
|
|
391
|
-
}),
|
|
422
|
+
pool: new TransactionPool(this.#lc.withContext("watermark", watermark), { mode: READ_COMMITTED }),
|
|
392
423
|
preCommitWatermark: watermark,
|
|
393
424
|
pos: 0,
|
|
394
425
|
startingReplicationState: promise,
|
|
@@ -396,7 +427,7 @@ var Storer = class {
|
|
|
396
427
|
batch: [],
|
|
397
428
|
lastFlush: void 0
|
|
398
429
|
};
|
|
399
|
-
tx.pool.run(this.#
|
|
430
|
+
tx.pool.run(this.#inserter);
|
|
400
431
|
tx.pool.process((tx) => {
|
|
401
432
|
tx`
|
|
402
433
|
SELECT "owner" FROM ${this.#cdc("replicationState")} FOR UPDATE`.then(([result]) => resolve(result), reject);
|
|
@@ -455,22 +486,46 @@ var Storer = class {
|
|
|
455
486
|
}
|
|
456
487
|
}
|
|
457
488
|
async #startCatchup(subs) {
|
|
458
|
-
|
|
489
|
+
const numCatchups = subs.length;
|
|
490
|
+
if (numCatchups === 0) return;
|
|
459
491
|
const lc = this.#lc.withContext("pool", "catchup");
|
|
460
|
-
const
|
|
461
|
-
reader
|
|
492
|
+
const { init, cleanup, snapshotID } = sharedSnapshot();
|
|
493
|
+
const reader = new TransactionPool(lc, {
|
|
494
|
+
mode: READONLY,
|
|
495
|
+
init,
|
|
496
|
+
cleanup,
|
|
497
|
+
initialWorkers: subs.length
|
|
498
|
+
});
|
|
499
|
+
const catchupConns = this.#makeConnectionPool("subscriber-catchup", numCatchups);
|
|
500
|
+
reader.run(catchupConns);
|
|
462
501
|
let lastWatermark;
|
|
502
|
+
const catchupSnapshotted = this.#progressMonitor.trackTask({
|
|
503
|
+
task: "capture-catchup-snapshot",
|
|
504
|
+
subscribers: subs.map(({ subscriber: s }) => s.id)
|
|
505
|
+
});
|
|
463
506
|
try {
|
|
464
507
|
[{lastWatermark}] = await reader.processReadTask((sql) => sql`
|
|
465
508
|
SELECT "lastWatermark" FROM ${this.#cdc("replicationState")}
|
|
466
509
|
`);
|
|
510
|
+
lc.info?.(`snapshotted db at ${lastWatermark} (${await snapshotID}) to catchup ${numCatchups} subscriber(s)`);
|
|
467
511
|
} catch (e) {
|
|
468
512
|
subs.map(({ subscriber }) => subscriber.fail(e));
|
|
469
513
|
throw e;
|
|
514
|
+
} finally {
|
|
515
|
+
catchupSnapshotted();
|
|
470
516
|
}
|
|
471
|
-
Promise.
|
|
517
|
+
Promise.allSettled(subs.map((sub) => this.#catchup(lc.withContext("subscriber", sub.subscriber.id), sub, lastWatermark, reader))).finally(() => {
|
|
518
|
+
reader.setDone();
|
|
519
|
+
catchupConns.end().catch(() => {});
|
|
520
|
+
});
|
|
472
521
|
}
|
|
473
522
|
async #catchup(lc, { subscriber: sub, mode }, lastWatermark, reader) {
|
|
523
|
+
let entriesReceived = 0;
|
|
524
|
+
let catchupProgressed = this.#progressMonitor.trackTask({
|
|
525
|
+
task: "catchup",
|
|
526
|
+
subscriber: sub.id,
|
|
527
|
+
entriesReceived
|
|
528
|
+
});
|
|
474
529
|
try {
|
|
475
530
|
lc.info?.(`starting catchup`);
|
|
476
531
|
await reader.processReadTask(async (tx) => {
|
|
@@ -484,21 +539,32 @@ var Storer = class {
|
|
|
484
539
|
WHERE watermark >= ${sub.watermark}
|
|
485
540
|
AND watermark <= ${lastWatermark}
|
|
486
541
|
ORDER BY watermark, pos`.cursor(2e3)) {
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
lastBatchConsumed
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
542
|
+
catchupProgressed();
|
|
543
|
+
entriesReceived += entries.length;
|
|
544
|
+
try {
|
|
545
|
+
const start = performance.now();
|
|
546
|
+
await lastBatchConsumed;
|
|
547
|
+
const elapsed = performance.now() - start;
|
|
548
|
+
if (lastBatchConsumed) lc[elapsed > 100 ? "info" : "debug"]?.(`waited ${elapsed.toFixed(3)} ms for ${sub.id} to consume last batch of catchup entries`);
|
|
549
|
+
for (const entry of entries) if (entry.watermark === sub.watermark) watermarkFound = true;
|
|
550
|
+
else if (watermarkFound) {
|
|
551
|
+
lastBatchConsumed = sub.catchup(reconstructWatermarkedChange(entry));
|
|
552
|
+
count++;
|
|
553
|
+
} else if (mode === "backup") throw new AutoResetSignal(`backup replica at watermark ${sub.watermark} is behind change db: ${entry.watermark})`);
|
|
554
|
+
else {
|
|
555
|
+
lc.warn?.(`rejecting subscriber at watermark ${sub.watermark} (earliest watermark: ${entry.watermark})`);
|
|
556
|
+
sub.close(2, `earliest supported watermark is ${entry.watermark} (requested ${sub.watermark})`);
|
|
557
|
+
return;
|
|
558
|
+
}
|
|
559
|
+
} finally {
|
|
560
|
+
catchupProgressed = this.#progressMonitor.trackTask({
|
|
561
|
+
task: "catchup",
|
|
562
|
+
subscriber: sub.id,
|
|
563
|
+
entriesReceived
|
|
564
|
+
});
|
|
500
565
|
}
|
|
501
566
|
}
|
|
567
|
+
catchupProgressed();
|
|
502
568
|
if (watermarkFound) {
|
|
503
569
|
await lastBatchConsumed;
|
|
504
570
|
lc.info?.(`caught up ${sub.id} with ${count} changes (${Date.now() - start} ms)`);
|
|
@@ -512,6 +578,8 @@ var Storer = class {
|
|
|
512
578
|
this.#onFatal(err);
|
|
513
579
|
}
|
|
514
580
|
sub.fail(err);
|
|
581
|
+
} finally {
|
|
582
|
+
catchupProgressed();
|
|
515
583
|
}
|
|
516
584
|
}
|
|
517
585
|
/**
|
|
@@ -609,6 +677,7 @@ var Storer = class {
|
|
|
609
677
|
*/
|
|
610
678
|
async stop() {
|
|
611
679
|
if (this.#running) {
|
|
680
|
+
this.#progressMonitor.stop();
|
|
612
681
|
this.#lc.info?.(`draining ${this.#queue.size()} changeLog entries`);
|
|
613
682
|
this.abort();
|
|
614
683
|
this.#queue.enqueue("stop");
|
|
@@ -669,6 +738,65 @@ var PurgeLocker = class {
|
|
|
669
738
|
return new PurgeLock(this.#lc, tx, replicaVersion, watermark);
|
|
670
739
|
}
|
|
671
740
|
};
|
|
741
|
+
/**
|
|
742
|
+
* Periodic monitor for detecting the pathological condition in which a db
|
|
743
|
+
* query hangs forever because of a connection problem, such as a half-closed
|
|
744
|
+
* socket.
|
|
745
|
+
*
|
|
746
|
+
* Use this for db calls that recur or accumulate progress within a single
|
|
747
|
+
* logical operation on the main storer loop (e.g. the queue-processing loop,
|
|
748
|
+
* or a multi-batch catchup read), where "no progress for N ms" is the
|
|
749
|
+
* meaningful signal. For a one-off db call -- including ones made before
|
|
750
|
+
* {@link Storer.run} starts this monitor's polling -- use a plain timeout
|
|
751
|
+
* (see `Storer.#withTimeout`) instead.
|
|
752
|
+
*
|
|
753
|
+
* The failure handler should stop the server. The ProgressMonitor will
|
|
754
|
+
* assume this and stop itself after calling it.
|
|
755
|
+
*
|
|
756
|
+
* Internal to the Storer, but exported for testing.
|
|
757
|
+
*/
|
|
758
|
+
var ProgressMonitor = class {
|
|
759
|
+
#lc;
|
|
760
|
+
#failureThresholdMs;
|
|
761
|
+
#onFailure;
|
|
762
|
+
#taskUpdates = /* @__PURE__ */ new Map();
|
|
763
|
+
#timer;
|
|
764
|
+
constructor(lc, failureThresholdMs, onFailure) {
|
|
765
|
+
this.#lc = lc.withContext("component", "storer-progress-monitor");
|
|
766
|
+
this.#failureThresholdMs = failureThresholdMs;
|
|
767
|
+
this.#onFailure = onFailure;
|
|
768
|
+
}
|
|
769
|
+
start() {
|
|
770
|
+
clearInterval(this.#timer);
|
|
771
|
+
this.#timer = setInterval(() => this.checkTaskProgress(/* @__PURE__ */ new Date()), this.#failureThresholdMs).unref();
|
|
772
|
+
}
|
|
773
|
+
checkTaskProgress(now) {
|
|
774
|
+
for (const [task, lastUpdate] of this.#taskUpdates.entries()) if (now.getTime() - lastUpdate.getTime() >= this.#failureThresholdMs) {
|
|
775
|
+
this.#lc.error?.(`Last task update was more than ${this.#failureThresholdMs}ms ago`, {
|
|
776
|
+
task,
|
|
777
|
+
lastUpdate
|
|
778
|
+
});
|
|
779
|
+
this.#onFailure(/* @__PURE__ */ new Error(`Task failed to progress for over ${this.#failureThresholdMs}ms: ${BigIntJSON.stringify(task)}`));
|
|
780
|
+
this.stop();
|
|
781
|
+
return;
|
|
782
|
+
}
|
|
783
|
+
}
|
|
784
|
+
/**
|
|
785
|
+
* Starts a task to be tracked.
|
|
786
|
+
*
|
|
787
|
+
* @return a callback to invoke when the task is done.
|
|
788
|
+
*/
|
|
789
|
+
trackTask(task, now = /* @__PURE__ */ new Date()) {
|
|
790
|
+
this.#taskUpdates.set(task, now);
|
|
791
|
+
return () => {
|
|
792
|
+
this.#taskUpdates.delete(task);
|
|
793
|
+
};
|
|
794
|
+
}
|
|
795
|
+
stop() {
|
|
796
|
+
clearInterval(this.#timer);
|
|
797
|
+
this.#taskUpdates.clear();
|
|
798
|
+
}
|
|
799
|
+
};
|
|
672
800
|
//#endregion
|
|
673
801
|
export { PurgeLocker, Storer };
|
|
674
802
|
|