@rocicorp/zero 1.10.0-canary.1 → 1.10.0-canary.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/shared/src/promise-race.d.ts +14 -0
- package/out/shared/src/promise-race.d.ts.map +1 -1
- package/out/shared/src/promise-race.js +32 -1
- package/out/shared/src/promise-race.js.map +1 -1
- package/out/zero/package.js +1 -1
- package/out/zero/package.js.map +1 -1
- package/out/zero-cache/src/auth/write-authorizer.d.ts.map +1 -1
- package/out/zero-cache/src/auth/write-authorizer.js +8 -1
- package/out/zero-cache/src/auth/write-authorizer.js.map +1 -1
- package/out/zero-cache/src/config/normalize.d.ts +5 -0
- package/out/zero-cache/src/config/normalize.d.ts.map +1 -1
- package/out/zero-cache/src/config/normalize.js +16 -3
- package/out/zero-cache/src/config/normalize.js.map +1 -1
- package/out/zero-cache/src/config/zero-config.d.ts +14 -4
- package/out/zero-cache/src/config/zero-config.d.ts.map +1 -1
- package/out/zero-cache/src/config/zero-config.js +31 -13
- package/out/zero-cache/src/config/zero-config.js.map +1 -1
- package/out/zero-cache/src/db/migration-lite.d.ts.map +1 -1
- package/out/zero-cache/src/db/migration-lite.js +5 -3
- package/out/zero-cache/src/db/migration-lite.js.map +1 -1
- package/out/zero-cache/src/db/transaction-pool.d.ts +5 -15
- package/out/zero-cache/src/db/transaction-pool.d.ts.map +1 -1
- package/out/zero-cache/src/db/transaction-pool.js +8 -72
- package/out/zero-cache/src/db/transaction-pool.js.map +1 -1
- package/out/zero-cache/src/server/change-streamer.d.ts.map +1 -1
- package/out/zero-cache/src/server/change-streamer.js +24 -13
- package/out/zero-cache/src/server/change-streamer.js.map +1 -1
- package/out/zero-cache/src/server/main.d.ts.map +1 -1
- package/out/zero-cache/src/server/main.js +4 -4
- package/out/zero-cache/src/server/main.js.map +1 -1
- package/out/zero-cache/src/server/replicator.d.ts.map +1 -1
- package/out/zero-cache/src/server/replicator.js +1 -1
- package/out/zero-cache/src/server/replicator.js.map +1 -1
- package/out/zero-cache/src/server/runner/run-worker.js +1 -1
- package/out/zero-cache/src/server/worker-urls.d.ts +0 -1
- package/out/zero-cache/src/server/worker-urls.d.ts.map +1 -1
- package/out/zero-cache/src/server/worker-urls.js +1 -2
- package/out/zero-cache/src/server/worker-urls.js.map +1 -1
- package/out/zero-cache/src/services/change-source/common/replica-schema.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/common/replica-schema.js +8 -2
- package/out/zero-cache/src/services/change-source/common/replica-schema.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/backfill-stream.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/backfill-stream.js +18 -4
- package/out/zero-cache/src/services/change-source/pg/backfill-stream.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/change-source.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/change-source.js +12 -10
- package/out/zero-cache/src/services/change-source/pg/change-source.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/decommission.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/decommission.js +20 -11
- package/out/zero-cache/src/services/change-source/pg/decommission.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/initial-sync.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/initial-sync.js +51 -74
- package/out/zero-cache/src/services/change-source/pg/initial-sync.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.d.ts +20 -0
- package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.js +43 -3
- package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/replication-slots.d.ts +24 -1
- package/out/zero-cache/src/services/change-source/pg/replication-slots.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/replication-slots.js +35 -5
- package/out/zero-cache/src/services/change-source/pg/replication-slots.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.d.ts +1 -4
- package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.js +11 -18
- package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/backup-monitor.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/backup-monitor.js +8 -3
- package/out/zero-cache/src/services/change-streamer/backup-monitor.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-log-compare-digest.d.ts +34 -0
- package/out/zero-cache/src/services/change-streamer/change-log-compare-digest.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/change-log-compare-digest.js +67 -0
- package/out/zero-cache/src/services/change-streamer/change-log-compare-digest.js.map +1 -0
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts +2 -4
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.js +8 -2
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts +34 -13
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js +152 -29
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts +3 -9
- package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/forwarder.js +1 -1
- package/out/zero-cache/src/services/change-streamer/forwarder.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/litestream3-prometheus-poller.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/litestream3-prometheus-poller.js +11 -5
- package/out/zero-cache/src/services/change-streamer/litestream3-prometheus-poller.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/shard-sampling.d.ts +11 -0
- package/out/zero-cache/src/services/change-streamer/shard-sampling.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/shard-sampling.js +19 -0
- package/out/zero-cache/src/services/change-streamer/shard-sampling.js.map +1 -0
- package/out/zero-cache/src/services/change-streamer/snapshot-reservations.d.ts +11 -1
- package/out/zero-cache/src/services/change-streamer/snapshot-reservations.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/snapshot-reservations.js +38 -5
- package/out/zero-cache/src/services/change-streamer/snapshot-reservations.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-catchup.d.ts +25 -1
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-catchup.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-catchup.js +95 -12
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-catchup.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-comparator.d.ts +83 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-comparator.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-comparator.js +417 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-comparator.js.map +1 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-read-router.d.ts +73 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-read-router.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-read-router.js +157 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-read-router.js.map +1 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-reader.d.ts +1 -1
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-reader.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-reader.js +8 -3
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-reader.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/storer.d.ts +57 -0
- package/out/zero-cache/src/services/change-streamer/storer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/storer.js +190 -40
- package/out/zero-cache/src/services/change-streamer/storer.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/vfs-watermark-poller.d.ts +29 -0
- package/out/zero-cache/src/services/change-streamer/vfs-watermark-poller.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/vfs-watermark-poller.js +154 -0
- package/out/zero-cache/src/services/change-streamer/vfs-watermark-poller.js.map +1 -0
- package/out/zero-cache/src/services/http-service.d.ts +1 -0
- package/out/zero-cache/src/services/http-service.d.ts.map +1 -1
- package/out/zero-cache/src/services/http-service.js +7 -3
- package/out/zero-cache/src/services/http-service.js.map +1 -1
- package/out/zero-cache/src/services/litestream/commands.d.ts.map +1 -1
- package/out/zero-cache/src/services/litestream/commands.js +8 -4
- package/out/zero-cache/src/services/litestream/commands.js.map +1 -1
- package/out/zero-cache/src/services/litestream/metrics.d.ts +1 -0
- package/out/zero-cache/src/services/litestream/metrics.d.ts.map +1 -1
- package/out/zero-cache/src/services/litestream/metrics.js +4 -1
- package/out/zero-cache/src/services/litestream/metrics.js.map +1 -1
- package/out/zero-cache/src/services/mutagen/mutagen.d.ts.map +1 -1
- package/out/zero-cache/src/services/mutagen/mutagen.js +1 -1
- package/out/zero-cache/src/services/mutagen/mutagen.js.map +1 -1
- package/out/zero-cache/src/services/replicator/change-log-db.d.ts +10 -0
- package/out/zero-cache/src/services/replicator/change-log-db.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/change-log-db.js +15 -1
- package/out/zero-cache/src/services/replicator/change-log-db.js.map +1 -1
- package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.d.ts +3 -2
- package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.js +8 -6
- package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/client-handler.d.ts +3 -2
- package/out/zero-cache/src/services/view-syncer/client-handler.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/client-handler.js +25 -17
- package/out/zero-cache/src/services/view-syncer/client-handler.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/cvr-store.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/cvr-store.js +14 -1
- package/out/zero-cache/src/services/view-syncer/cvr-store.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts +3 -3
- package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.js +6 -2
- package/out/zero-cache/src/services/view-syncer/view-syncer.js.map +1 -1
- package/out/zero-cache/src/types/downstream.d.ts +11 -0
- package/out/zero-cache/src/types/downstream.d.ts.map +1 -0
- package/out/zero-cache/src/types/pg.d.ts.map +1 -1
- package/out/zero-cache/src/types/pg.js +14 -4
- package/out/zero-cache/src/types/pg.js.map +1 -1
- package/out/zero-cache/src/types/poke-chunk.d.ts +18 -0
- package/out/zero-cache/src/types/poke-chunk.d.ts.map +1 -0
- package/out/zero-cache/src/types/poke-chunk.js +57 -0
- package/out/zero-cache/src/types/poke-chunk.js.map +1 -0
- package/out/zero-cache/src/types/processes.js +1 -4
- package/out/zero-cache/src/types/processes.js.map +1 -1
- package/out/zero-cache/src/types/streams.d.ts +4 -9
- package/out/zero-cache/src/types/streams.d.ts.map +1 -1
- package/out/zero-cache/src/types/streams.js.map +1 -1
- package/out/zero-cache/src/types/subscription.d.ts +2 -11
- package/out/zero-cache/src/types/subscription.d.ts.map +1 -1
- package/out/zero-cache/src/types/subscription.js +4 -25
- package/out/zero-cache/src/types/subscription.js.map +1 -1
- package/out/zero-cache/src/workers/connection.d.ts +22 -4
- package/out/zero-cache/src/workers/connection.d.ts.map +1 -1
- package/out/zero-cache/src/workers/connection.js +112 -14
- package/out/zero-cache/src/workers/connection.js.map +1 -1
- package/out/zero-client/src/client/version.js +1 -1
- package/out/zero-client/src/client/zero-poke-handler.d.ts +7 -2
- package/out/zero-client/src/client/zero-poke-handler.d.ts.map +1 -1
- package/out/zero-client/src/client/zero-poke-handler.js +52 -2
- package/out/zero-client/src/client/zero-poke-handler.js.map +1 -1
- package/out/zero-client/src/client/zero.d.ts.map +1 -1
- package/out/zero-client/src/client/zero.js +38 -6
- package/out/zero-client/src/client/zero.js.map +1 -1
- package/out/zero-protocol/src/down.d.ts +1 -1
- package/out/zero-protocol/src/poke.d.ts +89 -7
- package/out/zero-protocol/src/poke.d.ts.map +1 -1
- package/out/zero-protocol/src/poke.js +16 -8
- package/out/zero-protocol/src/poke.js.map +1 -1
- package/out/zero-protocol/src/protocol-version.d.ts +12 -7
- package/out/zero-protocol/src/protocol-version.d.ts.map +1 -1
- package/out/zero-protocol/src/protocol-version.js.map +1 -1
- package/out/zql/src/ivm/memory-source.d.ts +10 -2
- package/out/zql/src/ivm/memory-source.d.ts.map +1 -1
- package/out/zql/src/ivm/memory-source.js +14 -6
- package/out/zql/src/ivm/memory-source.js.map +1 -1
- package/out/zql/src/ivm/stream.d.ts +0 -1
- package/out/zql/src/ivm/stream.d.ts.map +1 -1
- package/out/zql/src/ivm/stream.js +1 -7
- package/out/zql/src/ivm/stream.js.map +1 -1
- package/out/zql/src/ivm/union-fan-in.d.ts +1 -1
- package/out/zql/src/ivm/union-fan-in.d.ts.map +1 -1
- package/out/zql/src/ivm/union-fan-in.js +11 -2
- package/out/zql/src/ivm/union-fan-in.js.map +1 -1
- package/out/zqlite/src/db.js +2 -2
- package/out/zqlite/src/sqlite-cost-model.js +5 -5
- package/out/zqlite/src/table-source.d.ts.map +1 -1
- package/out/zqlite/src/table-source.js +4 -4
- package/out/zqlite/src/table-source.js.map +1 -1
- package/package.json +1 -1
- package/out/zero-cache/src/server/backup-watermark-poller.js +0 -31
- package/out/zero-cache/src/server/backup-watermark-poller.js.map +0 -1
- package/out/zero-cache/src/services/litestream/vfs-watermark-poller.d.ts +0 -57
- package/out/zero-cache/src/services/litestream/vfs-watermark-poller.d.ts.map +0 -1
- package/out/zero-cache/src/services/litestream/vfs-watermark-poller.js +0 -137
- package/out/zero-cache/src/services/litestream/vfs-watermark-poller.js.map +0 -1
|
@@ -2,6 +2,7 @@ import { assert, unreachable } from "../../../../shared/src/asserts.js";
|
|
|
2
2
|
import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
|
|
3
3
|
import { AbortError } from "../../../../shared/src/abort-error.js";
|
|
4
4
|
import { parse, valita_exports } from "../../../../shared/src/valita.js";
|
|
5
|
+
import { BigIntJSON } from "../../../../shared/src/bigint-json.js";
|
|
5
6
|
import { cdcSchema } from "../../types/shards.js";
|
|
6
7
|
import { isDataChange, isSchemaChange } from "../change-source/protocol/current/data.js";
|
|
7
8
|
import "../change-source/protocol/current/downstream.js";
|
|
@@ -67,6 +68,7 @@ var Storer = class {
|
|
|
67
68
|
#statementTimeoutMs;
|
|
68
69
|
#changeLogBatchSize;
|
|
69
70
|
#drainTimeoutMs;
|
|
71
|
+
#progressMonitor;
|
|
70
72
|
#approximateQueuedBytes = 0;
|
|
71
73
|
#running = false;
|
|
72
74
|
constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol, db, replicaVersion, onCommitted, onFatal, { backPressureLimitHeapProportion, statementTimeoutMs, changeLogBatchSize, drainTimeoutMs = 3e4 }) {
|
|
@@ -82,6 +84,7 @@ var Storer = class {
|
|
|
82
84
|
this.#statementTimeoutMs = statementTimeoutMs;
|
|
83
85
|
this.#changeLogBatchSize = Math.max(1, changeLogBatchSize);
|
|
84
86
|
this.#drainTimeoutMs = drainTimeoutMs;
|
|
87
|
+
this.#progressMonitor = new ProgressMonitor(lc, statementTimeoutMs, onFatal);
|
|
85
88
|
const heapStats = getHeapStatistics();
|
|
86
89
|
this.#backPressureThresholdBytes = (heapStats.heap_size_limit - heapStats.used_heap_size) * backPressureLimitHeapProportion;
|
|
87
90
|
this.#lc.info?.(`Using up to ${(this.#backPressureThresholdBytes / 1024 ** 2).toFixed(2)} MB of --max-old-space-size (~${(heapStats.heap_size_limit / 1024 ** 2).toFixed(2)} MB) to absorb upstream spikes`, { heapStats });
|
|
@@ -89,6 +92,28 @@ var Storer = class {
|
|
|
89
92
|
#cdc(table) {
|
|
90
93
|
return this.#db(`${cdcSchema(this.#shard)}.${table}`);
|
|
91
94
|
}
|
|
95
|
+
/**
|
|
96
|
+
* Bounds a one-off db call (i.e. not part of the main storer loop or the
|
|
97
|
+
* background catchup read, which are tracked by the ProgressMonitor
|
|
98
|
+
* instead) with a plain timeout. This covers calls like
|
|
99
|
+
* {@link assumeOwnership} and {@link getStartStreamInitializationParameters}
|
|
100
|
+
* that are made by the caller *before* {@link run()} -- and thus before the
|
|
101
|
+
* ProgressMonitor's polling starts -- as well as ones made well after,
|
|
102
|
+
* where a continuously-polling watchdog would be overkill for a single
|
|
103
|
+
* bounded db round trip.
|
|
104
|
+
*
|
|
105
|
+
* Does not cancel `promise` on timeout (there is no cancellation mechanism
|
|
106
|
+
* for a postgres.js query); it just stops waiting for it, and swallows a
|
|
107
|
+
* later rejection so it doesn't surface as an unhandled rejection.
|
|
108
|
+
*/
|
|
109
|
+
async #withTimeout(name, promise) {
|
|
110
|
+
const result = await orTimeout(promise, this.#statementTimeoutMs);
|
|
111
|
+
if (result === "timed-out") {
|
|
112
|
+
promise.catch(() => {});
|
|
113
|
+
throw new AbortError(`${name} did not complete within ${this.#statementTimeoutMs}ms`);
|
|
114
|
+
}
|
|
115
|
+
return result;
|
|
116
|
+
}
|
|
92
117
|
async assumeOwnership(purgeLock) {
|
|
93
118
|
const db = this.#db;
|
|
94
119
|
const owner = this.#taskID;
|
|
@@ -97,10 +122,10 @@ var Storer = class {
|
|
|
97
122
|
const addressWithProtocol = ownerProtocol === "ws" ? ownerAddress : `${ownerProtocol}://${ownerAddress}`;
|
|
98
123
|
this.#lc.info?.(`assuming ownership at ${addressWithProtocol}`);
|
|
99
124
|
const start = performance.now();
|
|
100
|
-
await db`UPDATE ${this.#cdc("replicationState")} SET ${db({
|
|
125
|
+
await this.#withTimeout("assume-ownership", db`UPDATE ${this.#cdc("replicationState")} SET ${db({
|
|
101
126
|
owner,
|
|
102
127
|
ownerAddress: addressWithProtocol
|
|
103
|
-
})}
|
|
128
|
+
})}`);
|
|
104
129
|
const elapsed = (performance.now() - start).toFixed(2);
|
|
105
130
|
this.#lc.info?.(`assumed ownership at ${addressWithProtocol} (${elapsed} ms)`);
|
|
106
131
|
if (purgeLock) purgeLock.release();
|
|
@@ -123,7 +148,7 @@ var Storer = class {
|
|
|
123
148
|
* snapshot that the SQLite change log continues folding onto.
|
|
124
149
|
*/
|
|
125
150
|
async getStartStreamInitializationParameters() {
|
|
126
|
-
const [[{ lastWatermark }], result, tableMetadata, backfilling] = await runTx(this.#db, (sql) => [
|
|
151
|
+
const [[{ lastWatermark }], result, tableMetadata, backfilling] = await this.#withTimeout("get-stream-params", runTx(this.#db, (sql) => [
|
|
127
152
|
sql`
|
|
128
153
|
SELECT "lastWatermark" FROM ${this.#cdc("replicationState")}`,
|
|
129
154
|
sql`
|
|
@@ -146,7 +171,7 @@ var Storer = class {
|
|
|
146
171
|
sql`
|
|
147
172
|
SELECT "schema", "table", "column", "backfill" FROM ${this.#cdc("backfilling")}
|
|
148
173
|
ORDER BY "schema" COLLATE "C", "table" COLLATE "C", "column" COLLATE "C"`
|
|
149
|
-
], { mode: READONLY });
|
|
174
|
+
], { mode: READONLY }));
|
|
150
175
|
return {
|
|
151
176
|
lastWatermark,
|
|
152
177
|
backfillRequests: parse(result, backfillRequestsSchema),
|
|
@@ -157,10 +182,50 @@ var Storer = class {
|
|
|
157
182
|
};
|
|
158
183
|
}
|
|
159
184
|
async getMinWatermarkForCatchup() {
|
|
160
|
-
const [{ minWatermark }] = await this.#db`
|
|
161
|
-
SELECT min(watermark) as "minWatermark" FROM ${this.#cdc("changeLog")}
|
|
185
|
+
const [{ minWatermark }] = await this.#withTimeout("get-min-watermark", this.#db`
|
|
186
|
+
SELECT min(watermark) as "minWatermark" FROM ${this.#cdc("changeLog")}`);
|
|
162
187
|
return minWatermark;
|
|
163
188
|
}
|
|
189
|
+
/**
|
|
190
|
+
* Returns the retained Postgres catchup bounds.
|
|
191
|
+
* `minWatermark` is the oldest transaction, or `null` for a new replica.
|
|
192
|
+
* `lastWatermark` is the durable head.
|
|
193
|
+
*/
|
|
194
|
+
async getCatchupBounds() {
|
|
195
|
+
const [bounds] = await this.#withTimeout("get-catchup-bounds", this.#db`
|
|
196
|
+
SELECT
|
|
197
|
+
(SELECT min(watermark) FROM ${this.#cdc("changeLog")}) as "minWatermark",
|
|
198
|
+
(SELECT "lastWatermark" FROM ${this.#cdc("replicationState")}) as "lastWatermark"`);
|
|
199
|
+
return bounds;
|
|
200
|
+
}
|
|
201
|
+
/**
|
|
202
|
+
* Lists at most `limit` committed transaction watermarks in
|
|
203
|
+
* `(afterWatermark, throughWatermark]`, in ascending order.
|
|
204
|
+
*/
|
|
205
|
+
async listCommitWatermarks(afterWatermark, throughWatermark, limit) {
|
|
206
|
+
return (await this.#db`
|
|
207
|
+
SELECT watermark FROM ${this.#cdc("changeLog")}
|
|
208
|
+
WHERE precommit IS NOT NULL
|
|
209
|
+
AND watermark > ${afterWatermark}
|
|
210
|
+
AND watermark <= ${throughWatermark}
|
|
211
|
+
ORDER BY watermark
|
|
212
|
+
LIMIT ${limit}`).map(({ watermark }) => watermark);
|
|
213
|
+
}
|
|
214
|
+
/**
|
|
215
|
+
* Reads `(afterWatermark, throughWatermark]` in stream order and bounded batches.
|
|
216
|
+
* This query matches the subscriber catchup query.
|
|
217
|
+
*
|
|
218
|
+
* Both queries fail when an escaped NUL reaches `change->'tag'`.
|
|
219
|
+
* Postgres cannot convert that value to text.
|
|
220
|
+
*/
|
|
221
|
+
readCatchupRange(afterWatermark, throughWatermark, batchRows = 2e3) {
|
|
222
|
+
assert(Number.isSafeInteger(batchRows) && batchRows > 0, "Postgres change log batch size must be a positive safe integer");
|
|
223
|
+
return this.#db`
|
|
224
|
+
SELECT watermark, change->'tag' as tag, change::text FROM ${this.#cdc("changeLog")}
|
|
225
|
+
WHERE watermark > ${afterWatermark}
|
|
226
|
+
AND watermark <= ${throughWatermark}
|
|
227
|
+
ORDER BY watermark, pos`.cursor(batchRows);
|
|
228
|
+
}
|
|
164
229
|
purgeRecordsBefore(watermark) {
|
|
165
230
|
return runTx(this.#db, async (sql) => {
|
|
166
231
|
await sql`
|
|
@@ -214,30 +279,22 @@ var Storer = class {
|
|
|
214
279
|
}]);
|
|
215
280
|
}
|
|
216
281
|
#readyForMore = null;
|
|
217
|
-
#backpressureTimeout;
|
|
218
282
|
readyForMore() {
|
|
219
283
|
if (!this.#running) return;
|
|
220
284
|
if (this.#readyForMore === null && this.#approximateQueuedBytes > this.#backPressureThresholdBytes) {
|
|
221
285
|
this.#lc.warn?.(`applying back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)\n\nTo inspect changeLog backlog in your change DB:\n SELECT\n (change->'relation'->>'schema') || '.' || (change->'relation'->>'name') AS table_name,\n change->>'tag' AS operation,\n COUNT(*) AS count\n FROM "<app_id>/cdc"."changeLog"\n GROUP BY 1, 2\n ORDER BY 3 DESC\n LIMIT 20;`);
|
|
222
286
|
this.#readyForMore = resolver();
|
|
223
|
-
this.#resetBackpressureTimeout(true);
|
|
224
287
|
}
|
|
225
288
|
return this.#readyForMore?.promise;
|
|
226
289
|
}
|
|
227
|
-
#resetBackpressureTimeout(reschedule) {
|
|
228
|
-
clearTimeout(this.#backpressureTimeout);
|
|
229
|
-
this.#backpressureTimeout = reschedule ? setTimeout(() => {
|
|
230
|
-
this.#readyForMore?.reject(new AbortError(`No statement processed within the last ${this.#statementTimeoutMs}ms. Considering the change-log to be wedged.`));
|
|
231
|
-
this.#readyForMore = null;
|
|
232
|
-
}, this.#statementTimeoutMs) : void 0;
|
|
233
|
-
}
|
|
234
290
|
#maybeReleaseBackPressure() {
|
|
235
|
-
if (this.#readyForMore !== null)
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
291
|
+
if (this.#readyForMore !== null) {
|
|
292
|
+
if (this.#approximateQueuedBytes < this.#backPressureThresholdBytes * .8) {
|
|
293
|
+
this.#lc.info?.(`releasing back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)`);
|
|
294
|
+
this.#readyForMore.resolve();
|
|
295
|
+
this.#readyForMore = null;
|
|
296
|
+
}
|
|
297
|
+
}
|
|
241
298
|
}
|
|
242
299
|
/**
|
|
243
300
|
* Flushes any buffered {@link PendingTransaction.batch} rows to the changeLog.
|
|
@@ -276,6 +333,7 @@ var Storer = class {
|
|
|
276
333
|
this.#running = true;
|
|
277
334
|
this.#stopped = stopped;
|
|
278
335
|
this.#lc.info?.("starting storer");
|
|
336
|
+
this.#progressMonitor.start();
|
|
279
337
|
let err;
|
|
280
338
|
try {
|
|
281
339
|
await this.#processQueue();
|
|
@@ -287,7 +345,6 @@ var Storer = class {
|
|
|
287
345
|
this.#readyForMore.resolve();
|
|
288
346
|
this.#readyForMore = null;
|
|
289
347
|
}
|
|
290
|
-
this.#resetBackpressureTimeout(false);
|
|
291
348
|
this.#cancelQueueEntries(this.#queue.drain().filter((entry) => entry !== void 0), err);
|
|
292
349
|
this.#running = false;
|
|
293
350
|
signalStopped();
|
|
@@ -313,9 +370,20 @@ var Storer = class {
|
|
|
313
370
|
async #processQueue() {
|
|
314
371
|
let tx = null;
|
|
315
372
|
let msg;
|
|
373
|
+
let lastTaskDone;
|
|
374
|
+
const nextMessage = async () => {
|
|
375
|
+
lastTaskDone?.();
|
|
376
|
+
lastTaskDone = void 0;
|
|
377
|
+
const entry = await this.#queue.dequeue();
|
|
378
|
+
if (entry !== "stop") lastTaskDone = this.#progressMonitor.trackTask({
|
|
379
|
+
task: "queue-entry",
|
|
380
|
+
type: entry[0]
|
|
381
|
+
});
|
|
382
|
+
return entry;
|
|
383
|
+
};
|
|
316
384
|
const catchupQueue = [];
|
|
317
385
|
try {
|
|
318
|
-
while ((msg = await
|
|
386
|
+
while ((msg = await nextMessage()) !== "stop") {
|
|
319
387
|
const [msgType] = msg;
|
|
320
388
|
switch (msgType) {
|
|
321
389
|
case "ready": {
|
|
@@ -345,10 +413,7 @@ var Storer = class {
|
|
|
345
413
|
const { promise, resolve, reject } = resolver();
|
|
346
414
|
promise.catch(() => {});
|
|
347
415
|
tx = {
|
|
348
|
-
pool: new TransactionPool(this.#lc.withContext("watermark", watermark), {
|
|
349
|
-
mode: READ_COMMITTED,
|
|
350
|
-
statementResponseTimeout: this.#statementTimeoutMs
|
|
351
|
-
}),
|
|
416
|
+
pool: new TransactionPool(this.#lc.withContext("watermark", watermark), { mode: READ_COMMITTED }),
|
|
352
417
|
preCommitWatermark: watermark,
|
|
353
418
|
pos: 0,
|
|
354
419
|
startingReplicationState: promise,
|
|
@@ -420,6 +485,10 @@ var Storer = class {
|
|
|
420
485
|
const reader = new TransactionPool(lc, { mode: READONLY });
|
|
421
486
|
reader.run(this.#db);
|
|
422
487
|
let lastWatermark;
|
|
488
|
+
const catchupSnapshotted = this.#progressMonitor.trackTask({
|
|
489
|
+
task: "capture-catchup-snapshot",
|
|
490
|
+
subscribers: subs.map(({ subscriber: s }) => s.id)
|
|
491
|
+
});
|
|
423
492
|
try {
|
|
424
493
|
[{lastWatermark}] = await reader.processReadTask((sql) => sql`
|
|
425
494
|
SELECT "lastWatermark" FROM ${this.#cdc("replicationState")}
|
|
@@ -427,10 +496,18 @@ var Storer = class {
|
|
|
427
496
|
} catch (e) {
|
|
428
497
|
subs.map(({ subscriber }) => subscriber.fail(e));
|
|
429
498
|
throw e;
|
|
499
|
+
} finally {
|
|
500
|
+
catchupSnapshotted();
|
|
430
501
|
}
|
|
431
502
|
Promise.all(subs.map((sub) => this.#catchup(lc.withContext("subscriber", sub.subscriber.id), sub, lastWatermark, reader))).finally(() => reader.setDone());
|
|
432
503
|
}
|
|
433
504
|
async #catchup(lc, { subscriber: sub, mode }, lastWatermark, reader) {
|
|
505
|
+
let entriesReceived = 0;
|
|
506
|
+
let catchupProgressed = this.#progressMonitor.trackTask({
|
|
507
|
+
task: "catchup",
|
|
508
|
+
subscriber: sub.id,
|
|
509
|
+
entriesReceived
|
|
510
|
+
});
|
|
434
511
|
try {
|
|
435
512
|
lc.info?.(`starting catchup`);
|
|
436
513
|
await reader.processReadTask(async (tx) => {
|
|
@@ -444,21 +521,32 @@ var Storer = class {
|
|
|
444
521
|
WHERE watermark >= ${sub.watermark}
|
|
445
522
|
AND watermark <= ${lastWatermark}
|
|
446
523
|
ORDER BY watermark, pos`.cursor(2e3)) {
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
lastBatchConsumed
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
524
|
+
catchupProgressed();
|
|
525
|
+
entriesReceived += entries.length;
|
|
526
|
+
try {
|
|
527
|
+
const start = performance.now();
|
|
528
|
+
await lastBatchConsumed;
|
|
529
|
+
const elapsed = performance.now() - start;
|
|
530
|
+
if (lastBatchConsumed) lc[elapsed > 100 ? "info" : "debug"]?.(`waited ${elapsed.toFixed(3)} ms for ${sub.id} to consume last batch of catchup entries`);
|
|
531
|
+
for (const entry of entries) if (entry.watermark === sub.watermark) watermarkFound = true;
|
|
532
|
+
else if (watermarkFound) {
|
|
533
|
+
lastBatchConsumed = sub.catchup(reconstructWatermarkedChange(entry));
|
|
534
|
+
count++;
|
|
535
|
+
} else if (mode === "backup") throw new AutoResetSignal(`backup replica at watermark ${sub.watermark} is behind change db: ${entry.watermark})`);
|
|
536
|
+
else {
|
|
537
|
+
lc.warn?.(`rejecting subscriber at watermark ${sub.watermark} (earliest watermark: ${entry.watermark})`);
|
|
538
|
+
sub.close(2, `earliest supported watermark is ${entry.watermark} (requested ${sub.watermark})`);
|
|
539
|
+
return;
|
|
540
|
+
}
|
|
541
|
+
} finally {
|
|
542
|
+
catchupProgressed = this.#progressMonitor.trackTask({
|
|
543
|
+
task: "catchup",
|
|
544
|
+
subscriber: sub.id,
|
|
545
|
+
entriesReceived
|
|
546
|
+
});
|
|
460
547
|
}
|
|
461
548
|
}
|
|
549
|
+
catchupProgressed();
|
|
462
550
|
if (watermarkFound) {
|
|
463
551
|
await lastBatchConsumed;
|
|
464
552
|
lc.info?.(`caught up ${sub.id} with ${count} changes (${Date.now() - start} ms)`);
|
|
@@ -472,6 +560,8 @@ var Storer = class {
|
|
|
472
560
|
this.#onFatal(err);
|
|
473
561
|
}
|
|
474
562
|
sub.fail(err);
|
|
563
|
+
} finally {
|
|
564
|
+
catchupProgressed();
|
|
475
565
|
}
|
|
476
566
|
}
|
|
477
567
|
/**
|
|
@@ -569,6 +659,7 @@ var Storer = class {
|
|
|
569
659
|
*/
|
|
570
660
|
async stop() {
|
|
571
661
|
if (this.#running) {
|
|
662
|
+
this.#progressMonitor.stop();
|
|
572
663
|
this.#lc.info?.(`draining ${this.#queue.size()} changeLog entries`);
|
|
573
664
|
this.abort();
|
|
574
665
|
this.#queue.enqueue("stop");
|
|
@@ -629,6 +720,65 @@ var PurgeLocker = class {
|
|
|
629
720
|
return new PurgeLock(this.#lc, tx, replicaVersion, watermark);
|
|
630
721
|
}
|
|
631
722
|
};
|
|
723
|
+
/**
|
|
724
|
+
* Periodic monitor for detecting the pathological condition in which a db
|
|
725
|
+
* query hangs forever because of a connection problem, such as a half-closed
|
|
726
|
+
* socket.
|
|
727
|
+
*
|
|
728
|
+
* Use this for db calls that recur or accumulate progress within a single
|
|
729
|
+
* logical operation on the main storer loop (e.g. the queue-processing loop,
|
|
730
|
+
* or a multi-batch catchup read), where "no progress for N ms" is the
|
|
731
|
+
* meaningful signal. For a one-off db call -- including ones made before
|
|
732
|
+
* {@link Storer.run} starts this monitor's polling -- use a plain timeout
|
|
733
|
+
* (see `Storer.#withTimeout`) instead.
|
|
734
|
+
*
|
|
735
|
+
* The failure handler should stop the server. The ProgressMonitor will
|
|
736
|
+
* assume this and stop itself after calling it.
|
|
737
|
+
*
|
|
738
|
+
* Internal to the Storer, but exported for testing.
|
|
739
|
+
*/
|
|
740
|
+
var ProgressMonitor = class {
|
|
741
|
+
#lc;
|
|
742
|
+
#failureThresholdMs;
|
|
743
|
+
#onFailure;
|
|
744
|
+
#taskUpdates = /* @__PURE__ */ new Map();
|
|
745
|
+
#timer;
|
|
746
|
+
constructor(lc, failureThresholdMs, onFailure) {
|
|
747
|
+
this.#lc = lc.withContext("component", "storer-progress-monitor");
|
|
748
|
+
this.#failureThresholdMs = failureThresholdMs;
|
|
749
|
+
this.#onFailure = onFailure;
|
|
750
|
+
}
|
|
751
|
+
start() {
|
|
752
|
+
clearInterval(this.#timer);
|
|
753
|
+
this.#timer = setInterval(() => this.checkTaskProgress(/* @__PURE__ */ new Date()), this.#failureThresholdMs).unref();
|
|
754
|
+
}
|
|
755
|
+
checkTaskProgress(now) {
|
|
756
|
+
for (const [task, lastUpdate] of this.#taskUpdates.entries()) if (now.getTime() - lastUpdate.getTime() >= this.#failureThresholdMs) {
|
|
757
|
+
this.#lc.error?.(`Last task update was more than ${this.#failureThresholdMs}ms ago`, {
|
|
758
|
+
task,
|
|
759
|
+
lastUpdate
|
|
760
|
+
});
|
|
761
|
+
this.#onFailure(/* @__PURE__ */ new Error(`Task failed to progress for over ${this.#failureThresholdMs}ms: ${BigIntJSON.stringify(task)}`));
|
|
762
|
+
this.stop();
|
|
763
|
+
return;
|
|
764
|
+
}
|
|
765
|
+
}
|
|
766
|
+
/**
|
|
767
|
+
* Starts a task to be tracked.
|
|
768
|
+
*
|
|
769
|
+
* @return a callback to invoke when the task is done.
|
|
770
|
+
*/
|
|
771
|
+
trackTask(task, now = /* @__PURE__ */ new Date()) {
|
|
772
|
+
this.#taskUpdates.set(task, now);
|
|
773
|
+
return () => {
|
|
774
|
+
this.#taskUpdates.delete(task);
|
|
775
|
+
};
|
|
776
|
+
}
|
|
777
|
+
stop() {
|
|
778
|
+
clearInterval(this.#timer);
|
|
779
|
+
this.#taskUpdates.clear();
|
|
780
|
+
}
|
|
781
|
+
};
|
|
632
782
|
//#endregion
|
|
633
783
|
export { PurgeLocker, Storer };
|
|
634
784
|
|