@rocicorp/zero 1.10.0-head-23da5f1c-20260822 → 1.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/out/shared/src/promise-race.d.ts +14 -0
  2. package/out/shared/src/promise-race.d.ts.map +1 -1
  3. package/out/shared/src/promise-race.js +32 -1
  4. package/out/shared/src/promise-race.js.map +1 -1
  5. package/out/zero/package.js +1 -1
  6. package/out/zero/package.js.map +1 -1
  7. package/out/zero-cache/src/config/normalize.d.ts +5 -0
  8. package/out/zero-cache/src/config/normalize.d.ts.map +1 -1
  9. package/out/zero-cache/src/config/normalize.js +10 -2
  10. package/out/zero-cache/src/config/normalize.js.map +1 -1
  11. package/out/zero-cache/src/config/zero-config.d.ts +3 -1
  12. package/out/zero-cache/src/config/zero-config.d.ts.map +1 -1
  13. package/out/zero-cache/src/config/zero-config.js +10 -12
  14. package/out/zero-cache/src/config/zero-config.js.map +1 -1
  15. package/out/zero-cache/src/db/migration-lite.d.ts.map +1 -1
  16. package/out/zero-cache/src/db/migration-lite.js +5 -3
  17. package/out/zero-cache/src/db/migration-lite.js.map +1 -1
  18. package/out/zero-cache/src/db/transaction-pool.d.ts +0 -13
  19. package/out/zero-cache/src/db/transaction-pool.d.ts.map +1 -1
  20. package/out/zero-cache/src/db/transaction-pool.js +34 -70
  21. package/out/zero-cache/src/db/transaction-pool.js.map +1 -1
  22. package/out/zero-cache/src/server/change-streamer.d.ts.map +1 -1
  23. package/out/zero-cache/src/server/change-streamer.js +1 -1
  24. package/out/zero-cache/src/server/change-streamer.js.map +1 -1
  25. package/out/zero-cache/src/server/main.d.ts.map +1 -1
  26. package/out/zero-cache/src/server/main.js +4 -4
  27. package/out/zero-cache/src/server/main.js.map +1 -1
  28. package/out/zero-cache/src/server/replicator.d.ts.map +1 -1
  29. package/out/zero-cache/src/server/replicator.js +2 -2
  30. package/out/zero-cache/src/server/replicator.js.map +1 -1
  31. package/out/zero-cache/src/services/analyze.d.ts.map +1 -1
  32. package/out/zero-cache/src/services/analyze.js +1 -0
  33. package/out/zero-cache/src/services/analyze.js.map +1 -1
  34. package/out/zero-cache/src/services/change-source/common/backfill-manager.d.ts +12 -2
  35. package/out/zero-cache/src/services/change-source/common/backfill-manager.d.ts.map +1 -1
  36. package/out/zero-cache/src/services/change-source/common/backfill-manager.js +14 -3
  37. package/out/zero-cache/src/services/change-source/common/backfill-manager.js.map +1 -1
  38. package/out/zero-cache/src/services/change-source/common/replica-schema.d.ts.map +1 -1
  39. package/out/zero-cache/src/services/change-source/common/replica-schema.js +10 -41
  40. package/out/zero-cache/src/services/change-source/common/replica-schema.js.map +1 -1
  41. package/out/zero-cache/src/services/change-source/pg/backfill-stream.d.ts +3 -2
  42. package/out/zero-cache/src/services/change-source/pg/backfill-stream.d.ts.map +1 -1
  43. package/out/zero-cache/src/services/change-source/pg/backfill-stream.js +41 -15
  44. package/out/zero-cache/src/services/change-source/pg/backfill-stream.js.map +1 -1
  45. package/out/zero-cache/src/services/change-source/pg/change-source.d.ts +23 -1
  46. package/out/zero-cache/src/services/change-source/pg/change-source.d.ts.map +1 -1
  47. package/out/zero-cache/src/services/change-source/pg/change-source.js +25 -16
  48. package/out/zero-cache/src/services/change-source/pg/change-source.js.map +1 -1
  49. package/out/zero-cache/src/services/change-streamer/broadcast.d.ts.map +1 -1
  50. package/out/zero-cache/src/services/change-streamer/broadcast.js +4 -1
  51. package/out/zero-cache/src/services/change-streamer/broadcast.js.map +1 -1
  52. package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts.map +1 -1
  53. package/out/zero-cache/src/services/change-streamer/change-streamer-service.js +16 -9
  54. package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map +1 -1
  55. package/out/zero-cache/src/services/change-streamer/forwarder.d.ts.map +1 -1
  56. package/out/zero-cache/src/services/change-streamer/forwarder.js +12 -3
  57. package/out/zero-cache/src/services/change-streamer/forwarder.js.map +1 -1
  58. package/out/zero-cache/src/services/change-streamer/litestream3-prometheus-poller.d.ts.map +1 -1
  59. package/out/zero-cache/src/services/change-streamer/litestream3-prometheus-poller.js +11 -5
  60. package/out/zero-cache/src/services/change-streamer/litestream3-prometheus-poller.js.map +1 -1
  61. package/out/zero-cache/src/services/change-streamer/storer.d.ts +40 -1
  62. package/out/zero-cache/src/services/change-streamer/storer.d.ts.map +1 -1
  63. package/out/zero-cache/src/services/change-streamer/storer.js +179 -51
  64. package/out/zero-cache/src/services/change-streamer/storer.js.map +1 -1
  65. package/out/zero-cache/src/services/change-streamer/subscriber.d.ts +8 -1
  66. package/out/zero-cache/src/services/change-streamer/subscriber.d.ts.map +1 -1
  67. package/out/zero-cache/src/services/change-streamer/subscriber.js +10 -1
  68. package/out/zero-cache/src/services/change-streamer/subscriber.js.map +1 -1
  69. package/out/zero-cache/src/services/litestream/commands.d.ts.map +1 -1
  70. package/out/zero-cache/src/services/litestream/commands.js +8 -4
  71. package/out/zero-cache/src/services/litestream/commands.js.map +1 -1
  72. package/out/zero-cache/src/services/replicator/change-log-db.js.map +1 -1
  73. package/out/zero-cache/src/services/replicator/change-processor.js +2 -5
  74. package/out/zero-cache/src/services/replicator/change-processor.js.map +1 -1
  75. package/out/zero-cache/src/services/replicator/incremental-sync.d.ts +1 -1
  76. package/out/zero-cache/src/services/replicator/incremental-sync.d.ts.map +1 -1
  77. package/out/zero-cache/src/services/replicator/incremental-sync.js +1 -6
  78. package/out/zero-cache/src/services/replicator/incremental-sync.js.map +1 -1
  79. package/out/zero-cache/src/services/replicator/replicator.d.ts +1 -1
  80. package/out/zero-cache/src/services/replicator/replicator.d.ts.map +1 -1
  81. package/out/zero-cache/src/services/replicator/replicator.js +2 -2
  82. package/out/zero-cache/src/services/replicator/replicator.js.map +1 -1
  83. package/out/zero-cache/src/types/pg.d.ts +15 -2
  84. package/out/zero-cache/src/types/pg.d.ts.map +1 -1
  85. package/out/zero-cache/src/types/pg.js +7 -4
  86. package/out/zero-cache/src/types/pg.js.map +1 -1
  87. package/out/zero-cache/src/types/streams.d.ts +4 -9
  88. package/out/zero-cache/src/types/streams.d.ts.map +1 -1
  89. package/out/zero-cache/src/types/streams.js.map +1 -1
  90. package/out/zero-cache/src/types/subscription.d.ts +2 -11
  91. package/out/zero-cache/src/types/subscription.d.ts.map +1 -1
  92. package/out/zero-cache/src/types/subscription.js +4 -25
  93. package/out/zero-cache/src/types/subscription.js.map +1 -1
  94. package/out/zero-client/src/client/inspector/lazy-inspector.d.ts.map +1 -1
  95. package/out/zero-client/src/client/inspector/lazy-inspector.js +17 -10
  96. package/out/zero-client/src/client/inspector/lazy-inspector.js.map +1 -1
  97. package/out/zero-client/src/client/mutation-tracker.d.ts.map +1 -1
  98. package/out/zero-client/src/client/mutation-tracker.js +1 -1
  99. package/out/zero-client/src/client/mutation-tracker.js.map +1 -1
  100. package/out/zero-client/src/client/version.js +1 -1
  101. package/out/zero-protocol/src/custom-queries.d.ts +1 -1
  102. package/out/zero-protocol/src/error.d.ts +4 -4
  103. package/out/zero-protocol/src/inspect-down.js +1 -1
  104. package/out/zero-protocol/src/mutate-server.d.ts +2 -2
  105. package/out/zero-protocol/src/push.d.ts +1 -1
  106. package/out/zero-protocol/src/query-server.d.ts +2 -2
  107. package/package.json +1 -1
@@ -2,6 +2,7 @@ import { assert, unreachable } from "../../../../shared/src/asserts.js";
2
2
  import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
3
3
  import { AbortError } from "../../../../shared/src/abort-error.js";
4
4
  import { parse, valita_exports } from "../../../../shared/src/valita.js";
5
+ import { BigIntJSON } from "../../../../shared/src/bigint-json.js";
5
6
  import { cdcSchema } from "../../types/shards.js";
6
7
  import { isDataChange, isSchemaChange } from "../change-source/protocol/current/data.js";
7
8
  import "../change-source/protocol/current/downstream.js";
@@ -13,7 +14,7 @@ import { READONLY, READ_COMMITTED } from "../../db/mode-enum.js";
13
14
  import { runTx } from "../../db/run-transaction.js";
14
15
  import { Queue } from "../../../../shared/src/queue.js";
15
16
  import { AutoResetSignal, markResetRequired } from "./schema/tables.js";
16
- import { TransactionPool } from "../../db/transaction-pool.js";
17
+ import { TransactionPool, sharedSnapshot } from "../../db/transaction-pool.js";
17
18
  import { orTimeout } from "../../types/timeout.js";
18
19
  import { extractChangeSubstring, reconstructWatermarkedChange, serializeChangeStreamData } from "./change-log-codec.js";
19
20
  import { resolver } from "@rocicorp/resolver";
@@ -58,7 +59,10 @@ var Storer = class {
58
59
  #taskID;
59
60
  #discoveryAddress;
60
61
  #discoveryProtocol;
62
+ #makeConnectionPool;
61
63
  #db;
64
+ #inserter;
65
+ #purger;
62
66
  #replicaVersion;
63
67
  #onCommitted;
64
68
  #onFatal;
@@ -67,21 +71,26 @@ var Storer = class {
67
71
  #statementTimeoutMs;
68
72
  #changeLogBatchSize;
69
73
  #drainTimeoutMs;
74
+ #progressMonitor;
70
75
  #approximateQueuedBytes = 0;
71
76
  #running = false;
72
- constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol, db, replicaVersion, onCommitted, onFatal, { backPressureLimitHeapProportion, statementTimeoutMs, changeLogBatchSize, drainTimeoutMs = 3e4 }) {
77
+ constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol, dbProvider, replicaVersion, onCommitted, onFatal, { backPressureLimitHeapProportion, statementTimeoutMs, changeLogBatchSize, drainTimeoutMs = 3e4 }) {
73
78
  this.#lc = lc.withContext("component", "change-log");
74
79
  this.#shard = shard;
75
80
  this.#taskID = taskID;
76
81
  this.#discoveryAddress = discoveryAddress;
77
82
  this.#discoveryProtocol = discoveryProtocol;
78
- this.#db = db;
83
+ this.#makeConnectionPool = dbProvider;
84
+ this.#db = dbProvider("change-stream-init", 3);
85
+ this.#inserter = dbProvider("change-log-inserter", 1);
86
+ this.#purger = dbProvider("change-log-purger", 1);
79
87
  this.#replicaVersion = replicaVersion;
80
88
  this.#onCommitted = onCommitted;
81
89
  this.#onFatal = onFatal;
82
90
  this.#statementTimeoutMs = statementTimeoutMs;
83
91
  this.#changeLogBatchSize = Math.max(1, changeLogBatchSize);
84
92
  this.#drainTimeoutMs = drainTimeoutMs;
93
+ this.#progressMonitor = new ProgressMonitor(lc, statementTimeoutMs, onFatal);
85
94
  const heapStats = getHeapStatistics();
86
95
  this.#backPressureThresholdBytes = (heapStats.heap_size_limit - heapStats.used_heap_size) * backPressureLimitHeapProportion;
87
96
  this.#lc.info?.(`Using up to ${(this.#backPressureThresholdBytes / 1024 ** 2).toFixed(2)} MB of --max-old-space-size (~${(heapStats.heap_size_limit / 1024 ** 2).toFixed(2)} MB) to absorb upstream spikes`, { heapStats });
@@ -89,6 +98,28 @@ var Storer = class {
89
98
  #cdc(table) {
90
99
  return this.#db(`${cdcSchema(this.#shard)}.${table}`);
91
100
  }
101
+ /**
102
+ * Bounds a one-off db call (i.e. not part of the main storer loop or the
103
+ * background catchup read, which are tracked by the ProgressMonitor
104
+ * instead) with a plain timeout. This covers calls like
105
+ * {@link assumeOwnership} and {@link getStartStreamInitializationParameters}
106
+ * that are made by the caller *before* {@link run()} -- and thus before the
107
+ * ProgressMonitor's polling starts -- as well as ones made well after,
108
+ * where a continuously-polling watchdog would be overkill for a single
109
+ * bounded db round trip.
110
+ *
111
+ * Does not cancel `promise` on timeout (there is no cancellation mechanism
112
+ * for a postgres.js query); it just stops waiting for it, and swallows a
113
+ * later rejection so it doesn't surface as an unhandled rejection.
114
+ */
115
+ async #withTimeout(name, promise) {
116
+ const result = await orTimeout(promise, this.#statementTimeoutMs);
117
+ if (result === "timed-out") {
118
+ promise.catch(() => {});
119
+ throw new AbortError(`${name} did not complete within ${this.#statementTimeoutMs}ms`);
120
+ }
121
+ return result;
122
+ }
92
123
  async assumeOwnership(purgeLock) {
93
124
  const db = this.#db;
94
125
  const owner = this.#taskID;
@@ -97,10 +128,10 @@ var Storer = class {
97
128
  const addressWithProtocol = ownerProtocol === "ws" ? ownerAddress : `${ownerProtocol}://${ownerAddress}`;
98
129
  this.#lc.info?.(`assuming ownership at ${addressWithProtocol}`);
99
130
  const start = performance.now();
100
- await db`UPDATE ${this.#cdc("replicationState")} SET ${db({
131
+ await this.#withTimeout("assume-ownership", db`UPDATE ${this.#cdc("replicationState")} SET ${db({
101
132
  owner,
102
133
  ownerAddress: addressWithProtocol
103
- })}`;
134
+ })}`);
104
135
  const elapsed = (performance.now() - start).toFixed(2);
105
136
  this.#lc.info?.(`assumed ownership at ${addressWithProtocol} (${elapsed} ms)`);
106
137
  if (purgeLock) purgeLock.release();
@@ -123,7 +154,7 @@ var Storer = class {
123
154
  * snapshot that the SQLite change log continues folding onto.
124
155
  */
125
156
  async getStartStreamInitializationParameters() {
126
- const [[{ lastWatermark }], result, tableMetadata, backfilling] = await runTx(this.#db, (sql) => [
157
+ const [[{ lastWatermark }], result, tableMetadata, backfilling] = await this.#withTimeout("get-stream-params", runTx(this.#db, (sql) => [
127
158
  sql`
128
159
  SELECT "lastWatermark" FROM ${this.#cdc("replicationState")}`,
129
160
  sql`
@@ -146,7 +177,7 @@ var Storer = class {
146
177
  sql`
147
178
  SELECT "schema", "table", "column", "backfill" FROM ${this.#cdc("backfilling")}
148
179
  ORDER BY "schema" COLLATE "C", "table" COLLATE "C", "column" COLLATE "C"`
149
- ], { mode: READONLY });
180
+ ], { mode: READONLY }));
150
181
  return {
151
182
  lastWatermark,
152
183
  backfillRequests: parse(result, backfillRequestsSchema),
@@ -157,8 +188,8 @@ var Storer = class {
157
188
  };
158
189
  }
159
190
  async getMinWatermarkForCatchup() {
160
- const [{ minWatermark }] = await this.#db`
161
- SELECT min(watermark) as "minWatermark" FROM ${this.#cdc("changeLog")}`;
191
+ const [{ minWatermark }] = await this.#withTimeout("get-min-watermark", this.#db`
192
+ SELECT min(watermark) as "minWatermark" FROM ${this.#cdc("changeLog")}`);
162
193
  return minWatermark;
163
194
  }
164
195
  /**
@@ -167,10 +198,10 @@ var Storer = class {
167
198
  * `lastWatermark` is the durable head.
168
199
  */
169
200
  async getCatchupBounds() {
170
- const [bounds] = await this.#db`
201
+ const [bounds] = await this.#withTimeout("get-catchup-bounds", this.#db`
171
202
  SELECT
172
203
  (SELECT min(watermark) FROM ${this.#cdc("changeLog")}) as "minWatermark",
173
- (SELECT "lastWatermark" FROM ${this.#cdc("replicationState")}) as "lastWatermark"`;
204
+ (SELECT "lastWatermark" FROM ${this.#cdc("replicationState")}) as "lastWatermark"`);
174
205
  return bounds;
175
206
  }
176
207
  /**
@@ -202,7 +233,7 @@ var Storer = class {
202
233
  ORDER BY watermark, pos`.cursor(batchRows);
203
234
  }
204
235
  purgeRecordsBefore(watermark) {
205
- return runTx(this.#db, async (sql) => {
236
+ return runTx(this.#purger, async (sql) => {
206
237
  await sql`
207
238
  SELECT watermark FROM ${this.#cdc("changeLog")}
208
239
  ORDER BY watermark, pos LIMIT 1
@@ -254,30 +285,22 @@ var Storer = class {
254
285
  }]);
255
286
  }
256
287
  #readyForMore = null;
257
- #backpressureTimeout;
258
288
  readyForMore() {
259
289
  if (!this.#running) return;
260
290
  if (this.#readyForMore === null && this.#approximateQueuedBytes > this.#backPressureThresholdBytes) {
261
291
  this.#lc.warn?.(`applying back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)\n\nTo inspect changeLog backlog in your change DB:\n SELECT\n (change->'relation'->>'schema') || '.' || (change->'relation'->>'name') AS table_name,\n change->>'tag' AS operation,\n COUNT(*) AS count\n FROM "<app_id>/cdc"."changeLog"\n GROUP BY 1, 2\n ORDER BY 3 DESC\n LIMIT 20;`);
262
292
  this.#readyForMore = resolver();
263
- this.#resetBackpressureTimeout(true);
264
293
  }
265
294
  return this.#readyForMore?.promise;
266
295
  }
267
- #resetBackpressureTimeout(reschedule) {
268
- clearTimeout(this.#backpressureTimeout);
269
- this.#backpressureTimeout = reschedule ? setTimeout(() => {
270
- this.#readyForMore?.reject(new AbortError(`No statement processed within the last ${this.#statementTimeoutMs}ms. Considering the change-log to be wedged.`));
271
- this.#readyForMore = null;
272
- }, this.#statementTimeoutMs) : void 0;
273
- }
274
296
  #maybeReleaseBackPressure() {
275
- if (this.#readyForMore !== null) if (this.#approximateQueuedBytes < this.#backPressureThresholdBytes * .8) {
276
- this.#lc.info?.(`releasing back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)`);
277
- this.#readyForMore.resolve();
278
- this.#readyForMore = null;
279
- this.#resetBackpressureTimeout(false);
280
- } else this.#resetBackpressureTimeout(true);
297
+ if (this.#readyForMore !== null) {
298
+ if (this.#approximateQueuedBytes < this.#backPressureThresholdBytes * .8) {
299
+ this.#lc.info?.(`releasing back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)`);
300
+ this.#readyForMore.resolve();
301
+ this.#readyForMore = null;
302
+ }
303
+ }
281
304
  }
282
305
  /**
283
306
  * Flushes any buffered {@link PendingTransaction.batch} rows to the changeLog.
@@ -316,6 +339,7 @@ var Storer = class {
316
339
  this.#running = true;
317
340
  this.#stopped = stopped;
318
341
  this.#lc.info?.("starting storer");
342
+ this.#progressMonitor.start();
319
343
  let err;
320
344
  try {
321
345
  await this.#processQueue();
@@ -327,7 +351,6 @@ var Storer = class {
327
351
  this.#readyForMore.resolve();
328
352
  this.#readyForMore = null;
329
353
  }
330
- this.#resetBackpressureTimeout(false);
331
354
  this.#cancelQueueEntries(this.#queue.drain().filter((entry) => entry !== void 0), err);
332
355
  this.#running = false;
333
356
  signalStopped();
@@ -353,9 +376,20 @@ var Storer = class {
353
376
  async #processQueue() {
354
377
  let tx = null;
355
378
  let msg;
379
+ let lastTaskDone;
380
+ const nextMessage = async () => {
381
+ lastTaskDone?.();
382
+ lastTaskDone = void 0;
383
+ const entry = await this.#queue.dequeue();
384
+ if (entry !== "stop") lastTaskDone = this.#progressMonitor.trackTask({
385
+ task: "queue-entry",
386
+ type: entry[0]
387
+ });
388
+ return entry;
389
+ };
356
390
  const catchupQueue = [];
357
391
  try {
358
- while ((msg = await this.#queue.dequeue()) !== "stop") {
392
+ while ((msg = await nextMessage()) !== "stop") {
359
393
  const [msgType] = msg;
360
394
  switch (msgType) {
361
395
  case "ready": {
@@ -385,10 +419,7 @@ var Storer = class {
385
419
  const { promise, resolve, reject } = resolver();
386
420
  promise.catch(() => {});
387
421
  tx = {
388
- pool: new TransactionPool(this.#lc.withContext("watermark", watermark), {
389
- mode: READ_COMMITTED,
390
- statementResponseTimeout: this.#statementTimeoutMs
391
- }),
422
+ pool: new TransactionPool(this.#lc.withContext("watermark", watermark), { mode: READ_COMMITTED }),
392
423
  preCommitWatermark: watermark,
393
424
  pos: 0,
394
425
  startingReplicationState: promise,
@@ -396,7 +427,7 @@ var Storer = class {
396
427
  batch: [],
397
428
  lastFlush: void 0
398
429
  };
399
- tx.pool.run(this.#db);
430
+ tx.pool.run(this.#inserter);
400
431
  tx.pool.process((tx) => {
401
432
  tx`
402
433
  SELECT "owner" FROM ${this.#cdc("replicationState")} FOR UPDATE`.then(([result]) => resolve(result), reject);
@@ -455,22 +486,46 @@ var Storer = class {
455
486
  }
456
487
  }
457
488
  async #startCatchup(subs) {
458
- if (subs.length === 0) return;
489
+ const numCatchups = subs.length;
490
+ if (numCatchups === 0) return;
459
491
  const lc = this.#lc.withContext("pool", "catchup");
460
- const reader = new TransactionPool(lc, { mode: READONLY });
461
- reader.run(this.#db);
492
+ const { init, cleanup, snapshotID } = sharedSnapshot();
493
+ const reader = new TransactionPool(lc, {
494
+ mode: READONLY,
495
+ init,
496
+ cleanup,
497
+ initialWorkers: subs.length
498
+ });
499
+ const catchupConns = this.#makeConnectionPool("subscriber-catchup", numCatchups);
500
+ reader.run(catchupConns);
462
501
  let lastWatermark;
502
+ const catchupSnapshotted = this.#progressMonitor.trackTask({
503
+ task: "capture-catchup-snapshot",
504
+ subscribers: subs.map(({ subscriber: s }) => s.id)
505
+ });
463
506
  try {
464
507
  [{lastWatermark}] = await reader.processReadTask((sql) => sql`
465
508
  SELECT "lastWatermark" FROM ${this.#cdc("replicationState")}
466
509
  `);
510
+ lc.info?.(`snapshotted db at ${lastWatermark} (${await snapshotID}) to catchup ${numCatchups} subscriber(s)`);
467
511
  } catch (e) {
468
512
  subs.map(({ subscriber }) => subscriber.fail(e));
469
513
  throw e;
514
+ } finally {
515
+ catchupSnapshotted();
470
516
  }
471
- Promise.all(subs.map((sub) => this.#catchup(lc.withContext("subscriber", sub.subscriber.id), sub, lastWatermark, reader))).finally(() => reader.setDone());
517
+ Promise.allSettled(subs.map((sub) => this.#catchup(lc.withContext("subscriber", sub.subscriber.id), sub, lastWatermark, reader))).finally(() => {
518
+ reader.setDone();
519
+ catchupConns.end().catch(() => {});
520
+ });
472
521
  }
473
522
  async #catchup(lc, { subscriber: sub, mode }, lastWatermark, reader) {
523
+ let entriesReceived = 0;
524
+ let catchupProgressed = this.#progressMonitor.trackTask({
525
+ task: "catchup",
526
+ subscriber: sub.id,
527
+ entriesReceived
528
+ });
474
529
  try {
475
530
  lc.info?.(`starting catchup`);
476
531
  await reader.processReadTask(async (tx) => {
@@ -484,21 +539,32 @@ var Storer = class {
484
539
  WHERE watermark >= ${sub.watermark}
485
540
  AND watermark <= ${lastWatermark}
486
541
  ORDER BY watermark, pos`.cursor(2e3)) {
487
- const start = performance.now();
488
- await lastBatchConsumed;
489
- const elapsed = performance.now() - start;
490
- if (lastBatchConsumed) lc[elapsed > 100 ? "info" : "debug"]?.(`waited ${elapsed.toFixed(3)} ms for ${sub.id} to consume last batch of catchup entries`);
491
- for (const entry of entries) if (entry.watermark === sub.watermark) watermarkFound = true;
492
- else if (watermarkFound) {
493
- lastBatchConsumed = sub.catchup(reconstructWatermarkedChange(entry));
494
- count++;
495
- } else if (mode === "backup") throw new AutoResetSignal(`backup replica at watermark ${sub.watermark} is behind change db: ${entry.watermark})`);
496
- else {
497
- lc.warn?.(`rejecting subscriber at watermark ${sub.watermark} (earliest watermark: ${entry.watermark})`);
498
- sub.close(2, `earliest supported watermark is ${entry.watermark} (requested ${sub.watermark})`);
499
- return;
542
+ catchupProgressed();
543
+ entriesReceived += entries.length;
544
+ try {
545
+ const start = performance.now();
546
+ await lastBatchConsumed;
547
+ const elapsed = performance.now() - start;
548
+ if (lastBatchConsumed) lc[elapsed > 100 ? "info" : "debug"]?.(`waited ${elapsed.toFixed(3)} ms for ${sub.id} to consume last batch of catchup entries`);
549
+ for (const entry of entries) if (entry.watermark === sub.watermark) watermarkFound = true;
550
+ else if (watermarkFound) {
551
+ lastBatchConsumed = sub.catchup(reconstructWatermarkedChange(entry));
552
+ count++;
553
+ } else if (mode === "backup") throw new AutoResetSignal(`backup replica at watermark ${sub.watermark} is behind change db: ${entry.watermark})`);
554
+ else {
555
+ lc.warn?.(`rejecting subscriber at watermark ${sub.watermark} (earliest watermark: ${entry.watermark})`);
556
+ sub.close(2, `earliest supported watermark is ${entry.watermark} (requested ${sub.watermark})`);
557
+ return;
558
+ }
559
+ } finally {
560
+ catchupProgressed = this.#progressMonitor.trackTask({
561
+ task: "catchup",
562
+ subscriber: sub.id,
563
+ entriesReceived
564
+ });
500
565
  }
501
566
  }
567
+ catchupProgressed();
502
568
  if (watermarkFound) {
503
569
  await lastBatchConsumed;
504
570
  lc.info?.(`caught up ${sub.id} with ${count} changes (${Date.now() - start} ms)`);
@@ -512,6 +578,8 @@ var Storer = class {
512
578
  this.#onFatal(err);
513
579
  }
514
580
  sub.fail(err);
581
+ } finally {
582
+ catchupProgressed();
515
583
  }
516
584
  }
517
585
  /**
@@ -609,6 +677,7 @@ var Storer = class {
609
677
  */
610
678
  async stop() {
611
679
  if (this.#running) {
680
+ this.#progressMonitor.stop();
612
681
  this.#lc.info?.(`draining ${this.#queue.size()} changeLog entries`);
613
682
  this.abort();
614
683
  this.#queue.enqueue("stop");
@@ -669,6 +738,65 @@ var PurgeLocker = class {
669
738
  return new PurgeLock(this.#lc, tx, replicaVersion, watermark);
670
739
  }
671
740
  };
741
+ /**
742
+ * Periodic monitor for detecting the pathological condition in which a db
743
+ * query hangs forever because of a connection problem, such as a half-closed
744
+ * socket.
745
+ *
746
+ * Use this for db calls that recur or accumulate progress within a single
747
+ * logical operation on the main storer loop (e.g. the queue-processing loop,
748
+ * or a multi-batch catchup read), where "no progress for N ms" is the
749
+ * meaningful signal. For a one-off db call -- including ones made before
750
+ * {@link Storer.run} starts this monitor's polling -- use a plain timeout
751
+ * (see `Storer.#withTimeout`) instead.
752
+ *
753
+ * The failure handler should stop the server. The ProgressMonitor will
754
+ * assume this and stop itself after calling it.
755
+ *
756
+ * Internal to the Storer, but exported for testing.
757
+ */
758
+ var ProgressMonitor = class {
759
+ #lc;
760
+ #failureThresholdMs;
761
+ #onFailure;
762
+ #taskUpdates = /* @__PURE__ */ new Map();
763
+ #timer;
764
+ constructor(lc, failureThresholdMs, onFailure) {
765
+ this.#lc = lc.withContext("component", "storer-progress-monitor");
766
+ this.#failureThresholdMs = failureThresholdMs;
767
+ this.#onFailure = onFailure;
768
+ }
769
+ start() {
770
+ clearInterval(this.#timer);
771
+ this.#timer = setInterval(() => this.checkTaskProgress(/* @__PURE__ */ new Date()), this.#failureThresholdMs).unref();
772
+ }
773
+ checkTaskProgress(now) {
774
+ for (const [task, lastUpdate] of this.#taskUpdates.entries()) if (now.getTime() - lastUpdate.getTime() >= this.#failureThresholdMs) {
775
+ this.#lc.error?.(`Last task update was more than ${this.#failureThresholdMs}ms ago`, {
776
+ task,
777
+ lastUpdate
778
+ });
779
+ this.#onFailure(/* @__PURE__ */ new Error(`Task failed to progress for over ${this.#failureThresholdMs}ms: ${BigIntJSON.stringify(task)}`));
780
+ this.stop();
781
+ return;
782
+ }
783
+ }
784
+ /**
785
+ * Starts a task to be tracked.
786
+ *
787
+ * @return a callback to invoke when the task is done.
788
+ */
789
+ trackTask(task, now = /* @__PURE__ */ new Date()) {
790
+ this.#taskUpdates.set(task, now);
791
+ return () => {
792
+ this.#taskUpdates.delete(task);
793
+ };
794
+ }
795
+ stop() {
796
+ clearInterval(this.#timer);
797
+ this.#taskUpdates.clear();
798
+ }
799
+ };
672
800
  //#endregion
673
801
  export { PurgeLocker, Storer };
674
802