@rocicorp/zero 1.10.0-canary.1 → 1.10.0-canary.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (215) hide show
  1. package/out/shared/src/promise-race.d.ts +14 -0
  2. package/out/shared/src/promise-race.d.ts.map +1 -1
  3. package/out/shared/src/promise-race.js +32 -1
  4. package/out/shared/src/promise-race.js.map +1 -1
  5. package/out/zero/package.js +1 -1
  6. package/out/zero/package.js.map +1 -1
  7. package/out/zero-cache/src/auth/write-authorizer.d.ts.map +1 -1
  8. package/out/zero-cache/src/auth/write-authorizer.js +8 -1
  9. package/out/zero-cache/src/auth/write-authorizer.js.map +1 -1
  10. package/out/zero-cache/src/config/normalize.d.ts +5 -0
  11. package/out/zero-cache/src/config/normalize.d.ts.map +1 -1
  12. package/out/zero-cache/src/config/normalize.js +16 -3
  13. package/out/zero-cache/src/config/normalize.js.map +1 -1
  14. package/out/zero-cache/src/config/zero-config.d.ts +14 -4
  15. package/out/zero-cache/src/config/zero-config.d.ts.map +1 -1
  16. package/out/zero-cache/src/config/zero-config.js +31 -13
  17. package/out/zero-cache/src/config/zero-config.js.map +1 -1
  18. package/out/zero-cache/src/db/migration-lite.d.ts.map +1 -1
  19. package/out/zero-cache/src/db/migration-lite.js +5 -3
  20. package/out/zero-cache/src/db/migration-lite.js.map +1 -1
  21. package/out/zero-cache/src/db/transaction-pool.d.ts +5 -15
  22. package/out/zero-cache/src/db/transaction-pool.d.ts.map +1 -1
  23. package/out/zero-cache/src/db/transaction-pool.js +8 -72
  24. package/out/zero-cache/src/db/transaction-pool.js.map +1 -1
  25. package/out/zero-cache/src/server/change-streamer.d.ts.map +1 -1
  26. package/out/zero-cache/src/server/change-streamer.js +24 -13
  27. package/out/zero-cache/src/server/change-streamer.js.map +1 -1
  28. package/out/zero-cache/src/server/main.d.ts.map +1 -1
  29. package/out/zero-cache/src/server/main.js +4 -4
  30. package/out/zero-cache/src/server/main.js.map +1 -1
  31. package/out/zero-cache/src/server/replicator.d.ts.map +1 -1
  32. package/out/zero-cache/src/server/replicator.js +1 -1
  33. package/out/zero-cache/src/server/replicator.js.map +1 -1
  34. package/out/zero-cache/src/server/runner/run-worker.js +1 -1
  35. package/out/zero-cache/src/server/worker-urls.d.ts +0 -1
  36. package/out/zero-cache/src/server/worker-urls.d.ts.map +1 -1
  37. package/out/zero-cache/src/server/worker-urls.js +1 -2
  38. package/out/zero-cache/src/server/worker-urls.js.map +1 -1
  39. package/out/zero-cache/src/services/change-source/common/replica-schema.d.ts.map +1 -1
  40. package/out/zero-cache/src/services/change-source/common/replica-schema.js +8 -2
  41. package/out/zero-cache/src/services/change-source/common/replica-schema.js.map +1 -1
  42. package/out/zero-cache/src/services/change-source/pg/backfill-stream.d.ts.map +1 -1
  43. package/out/zero-cache/src/services/change-source/pg/backfill-stream.js +18 -4
  44. package/out/zero-cache/src/services/change-source/pg/backfill-stream.js.map +1 -1
  45. package/out/zero-cache/src/services/change-source/pg/change-source.d.ts.map +1 -1
  46. package/out/zero-cache/src/services/change-source/pg/change-source.js +12 -10
  47. package/out/zero-cache/src/services/change-source/pg/change-source.js.map +1 -1
  48. package/out/zero-cache/src/services/change-source/pg/decommission.d.ts.map +1 -1
  49. package/out/zero-cache/src/services/change-source/pg/decommission.js +20 -11
  50. package/out/zero-cache/src/services/change-source/pg/decommission.js.map +1 -1
  51. package/out/zero-cache/src/services/change-source/pg/initial-sync.d.ts.map +1 -1
  52. package/out/zero-cache/src/services/change-source/pg/initial-sync.js +51 -74
  53. package/out/zero-cache/src/services/change-source/pg/initial-sync.js.map +1 -1
  54. package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.d.ts +20 -0
  55. package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.d.ts.map +1 -1
  56. package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.js +43 -3
  57. package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.js.map +1 -1
  58. package/out/zero-cache/src/services/change-source/pg/replication-slots.d.ts +24 -1
  59. package/out/zero-cache/src/services/change-source/pg/replication-slots.d.ts.map +1 -1
  60. package/out/zero-cache/src/services/change-source/pg/replication-slots.js +35 -5
  61. package/out/zero-cache/src/services/change-source/pg/replication-slots.js.map +1 -1
  62. package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.d.ts +1 -4
  63. package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.d.ts.map +1 -1
  64. package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.js +11 -18
  65. package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.js.map +1 -1
  66. package/out/zero-cache/src/services/change-streamer/backup-monitor.d.ts.map +1 -1
  67. package/out/zero-cache/src/services/change-streamer/backup-monitor.js +8 -3
  68. package/out/zero-cache/src/services/change-streamer/backup-monitor.js.map +1 -1
  69. package/out/zero-cache/src/services/change-streamer/change-log-compare-digest.d.ts +34 -0
  70. package/out/zero-cache/src/services/change-streamer/change-log-compare-digest.d.ts.map +1 -0
  71. package/out/zero-cache/src/services/change-streamer/change-log-compare-digest.js +67 -0
  72. package/out/zero-cache/src/services/change-streamer/change-log-compare-digest.js.map +1 -0
  73. package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts +2 -4
  74. package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts.map +1 -1
  75. package/out/zero-cache/src/services/change-streamer/change-streamer-http.js +8 -2
  76. package/out/zero-cache/src/services/change-streamer/change-streamer-http.js.map +1 -1
  77. package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts +34 -13
  78. package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts.map +1 -1
  79. package/out/zero-cache/src/services/change-streamer/change-streamer-service.js +152 -29
  80. package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map +1 -1
  81. package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts +3 -9
  82. package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts.map +1 -1
  83. package/out/zero-cache/src/services/change-streamer/change-streamer.js.map +1 -1
  84. package/out/zero-cache/src/services/change-streamer/forwarder.js +1 -1
  85. package/out/zero-cache/src/services/change-streamer/forwarder.js.map +1 -1
  86. package/out/zero-cache/src/services/change-streamer/litestream3-prometheus-poller.d.ts.map +1 -1
  87. package/out/zero-cache/src/services/change-streamer/litestream3-prometheus-poller.js +11 -5
  88. package/out/zero-cache/src/services/change-streamer/litestream3-prometheus-poller.js.map +1 -1
  89. package/out/zero-cache/src/services/change-streamer/shard-sampling.d.ts +11 -0
  90. package/out/zero-cache/src/services/change-streamer/shard-sampling.d.ts.map +1 -0
  91. package/out/zero-cache/src/services/change-streamer/shard-sampling.js +19 -0
  92. package/out/zero-cache/src/services/change-streamer/shard-sampling.js.map +1 -0
  93. package/out/zero-cache/src/services/change-streamer/snapshot-reservations.d.ts +11 -1
  94. package/out/zero-cache/src/services/change-streamer/snapshot-reservations.d.ts.map +1 -1
  95. package/out/zero-cache/src/services/change-streamer/snapshot-reservations.js +38 -5
  96. package/out/zero-cache/src/services/change-streamer/snapshot-reservations.js.map +1 -1
  97. package/out/zero-cache/src/services/change-streamer/sqlite-change-log-catchup.d.ts +25 -1
  98. package/out/zero-cache/src/services/change-streamer/sqlite-change-log-catchup.d.ts.map +1 -1
  99. package/out/zero-cache/src/services/change-streamer/sqlite-change-log-catchup.js +95 -12
  100. package/out/zero-cache/src/services/change-streamer/sqlite-change-log-catchup.js.map +1 -1
  101. package/out/zero-cache/src/services/change-streamer/sqlite-change-log-comparator.d.ts +83 -0
  102. package/out/zero-cache/src/services/change-streamer/sqlite-change-log-comparator.d.ts.map +1 -0
  103. package/out/zero-cache/src/services/change-streamer/sqlite-change-log-comparator.js +417 -0
  104. package/out/zero-cache/src/services/change-streamer/sqlite-change-log-comparator.js.map +1 -0
  105. package/out/zero-cache/src/services/change-streamer/sqlite-change-log-read-router.d.ts +73 -0
  106. package/out/zero-cache/src/services/change-streamer/sqlite-change-log-read-router.d.ts.map +1 -0
  107. package/out/zero-cache/src/services/change-streamer/sqlite-change-log-read-router.js +157 -0
  108. package/out/zero-cache/src/services/change-streamer/sqlite-change-log-read-router.js.map +1 -0
  109. package/out/zero-cache/src/services/change-streamer/sqlite-change-log-reader.d.ts +1 -1
  110. package/out/zero-cache/src/services/change-streamer/sqlite-change-log-reader.d.ts.map +1 -1
  111. package/out/zero-cache/src/services/change-streamer/sqlite-change-log-reader.js +8 -3
  112. package/out/zero-cache/src/services/change-streamer/sqlite-change-log-reader.js.map +1 -1
  113. package/out/zero-cache/src/services/change-streamer/storer.d.ts +57 -0
  114. package/out/zero-cache/src/services/change-streamer/storer.d.ts.map +1 -1
  115. package/out/zero-cache/src/services/change-streamer/storer.js +190 -40
  116. package/out/zero-cache/src/services/change-streamer/storer.js.map +1 -1
  117. package/out/zero-cache/src/services/change-streamer/vfs-watermark-poller.d.ts +29 -0
  118. package/out/zero-cache/src/services/change-streamer/vfs-watermark-poller.d.ts.map +1 -0
  119. package/out/zero-cache/src/services/change-streamer/vfs-watermark-poller.js +154 -0
  120. package/out/zero-cache/src/services/change-streamer/vfs-watermark-poller.js.map +1 -0
  121. package/out/zero-cache/src/services/http-service.d.ts +1 -0
  122. package/out/zero-cache/src/services/http-service.d.ts.map +1 -1
  123. package/out/zero-cache/src/services/http-service.js +7 -3
  124. package/out/zero-cache/src/services/http-service.js.map +1 -1
  125. package/out/zero-cache/src/services/litestream/commands.d.ts.map +1 -1
  126. package/out/zero-cache/src/services/litestream/commands.js +8 -4
  127. package/out/zero-cache/src/services/litestream/commands.js.map +1 -1
  128. package/out/zero-cache/src/services/litestream/metrics.d.ts +1 -0
  129. package/out/zero-cache/src/services/litestream/metrics.d.ts.map +1 -1
  130. package/out/zero-cache/src/services/litestream/metrics.js +4 -1
  131. package/out/zero-cache/src/services/litestream/metrics.js.map +1 -1
  132. package/out/zero-cache/src/services/mutagen/mutagen.d.ts.map +1 -1
  133. package/out/zero-cache/src/services/mutagen/mutagen.js +1 -1
  134. package/out/zero-cache/src/services/mutagen/mutagen.js.map +1 -1
  135. package/out/zero-cache/src/services/replicator/change-log-db.d.ts +10 -0
  136. package/out/zero-cache/src/services/replicator/change-log-db.d.ts.map +1 -1
  137. package/out/zero-cache/src/services/replicator/change-log-db.js +15 -1
  138. package/out/zero-cache/src/services/replicator/change-log-db.js.map +1 -1
  139. package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.d.ts +3 -2
  140. package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.d.ts.map +1 -1
  141. package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.js +8 -6
  142. package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.js.map +1 -1
  143. package/out/zero-cache/src/services/view-syncer/client-handler.d.ts +3 -2
  144. package/out/zero-cache/src/services/view-syncer/client-handler.d.ts.map +1 -1
  145. package/out/zero-cache/src/services/view-syncer/client-handler.js +25 -17
  146. package/out/zero-cache/src/services/view-syncer/client-handler.js.map +1 -1
  147. package/out/zero-cache/src/services/view-syncer/cvr-store.d.ts.map +1 -1
  148. package/out/zero-cache/src/services/view-syncer/cvr-store.js +14 -1
  149. package/out/zero-cache/src/services/view-syncer/cvr-store.js.map +1 -1
  150. package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts +3 -3
  151. package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts.map +1 -1
  152. package/out/zero-cache/src/services/view-syncer/view-syncer.js +6 -2
  153. package/out/zero-cache/src/services/view-syncer/view-syncer.js.map +1 -1
  154. package/out/zero-cache/src/types/downstream.d.ts +11 -0
  155. package/out/zero-cache/src/types/downstream.d.ts.map +1 -0
  156. package/out/zero-cache/src/types/pg.d.ts.map +1 -1
  157. package/out/zero-cache/src/types/pg.js +14 -4
  158. package/out/zero-cache/src/types/pg.js.map +1 -1
  159. package/out/zero-cache/src/types/poke-chunk.d.ts +18 -0
  160. package/out/zero-cache/src/types/poke-chunk.d.ts.map +1 -0
  161. package/out/zero-cache/src/types/poke-chunk.js +57 -0
  162. package/out/zero-cache/src/types/poke-chunk.js.map +1 -0
  163. package/out/zero-cache/src/types/processes.js +1 -4
  164. package/out/zero-cache/src/types/processes.js.map +1 -1
  165. package/out/zero-cache/src/types/streams.d.ts +4 -9
  166. package/out/zero-cache/src/types/streams.d.ts.map +1 -1
  167. package/out/zero-cache/src/types/streams.js.map +1 -1
  168. package/out/zero-cache/src/types/subscription.d.ts +2 -11
  169. package/out/zero-cache/src/types/subscription.d.ts.map +1 -1
  170. package/out/zero-cache/src/types/subscription.js +4 -25
  171. package/out/zero-cache/src/types/subscription.js.map +1 -1
  172. package/out/zero-cache/src/workers/connection.d.ts +22 -4
  173. package/out/zero-cache/src/workers/connection.d.ts.map +1 -1
  174. package/out/zero-cache/src/workers/connection.js +112 -14
  175. package/out/zero-cache/src/workers/connection.js.map +1 -1
  176. package/out/zero-client/src/client/version.js +1 -1
  177. package/out/zero-client/src/client/zero-poke-handler.d.ts +7 -2
  178. package/out/zero-client/src/client/zero-poke-handler.d.ts.map +1 -1
  179. package/out/zero-client/src/client/zero-poke-handler.js +52 -2
  180. package/out/zero-client/src/client/zero-poke-handler.js.map +1 -1
  181. package/out/zero-client/src/client/zero.d.ts.map +1 -1
  182. package/out/zero-client/src/client/zero.js +38 -6
  183. package/out/zero-client/src/client/zero.js.map +1 -1
  184. package/out/zero-protocol/src/down.d.ts +1 -1
  185. package/out/zero-protocol/src/poke.d.ts +89 -7
  186. package/out/zero-protocol/src/poke.d.ts.map +1 -1
  187. package/out/zero-protocol/src/poke.js +16 -8
  188. package/out/zero-protocol/src/poke.js.map +1 -1
  189. package/out/zero-protocol/src/protocol-version.d.ts +12 -7
  190. package/out/zero-protocol/src/protocol-version.d.ts.map +1 -1
  191. package/out/zero-protocol/src/protocol-version.js.map +1 -1
  192. package/out/zql/src/ivm/memory-source.d.ts +10 -2
  193. package/out/zql/src/ivm/memory-source.d.ts.map +1 -1
  194. package/out/zql/src/ivm/memory-source.js +14 -6
  195. package/out/zql/src/ivm/memory-source.js.map +1 -1
  196. package/out/zql/src/ivm/stream.d.ts +0 -1
  197. package/out/zql/src/ivm/stream.d.ts.map +1 -1
  198. package/out/zql/src/ivm/stream.js +1 -7
  199. package/out/zql/src/ivm/stream.js.map +1 -1
  200. package/out/zql/src/ivm/union-fan-in.d.ts +1 -1
  201. package/out/zql/src/ivm/union-fan-in.d.ts.map +1 -1
  202. package/out/zql/src/ivm/union-fan-in.js +11 -2
  203. package/out/zql/src/ivm/union-fan-in.js.map +1 -1
  204. package/out/zqlite/src/db.js +2 -2
  205. package/out/zqlite/src/sqlite-cost-model.js +5 -5
  206. package/out/zqlite/src/table-source.d.ts.map +1 -1
  207. package/out/zqlite/src/table-source.js +4 -4
  208. package/out/zqlite/src/table-source.js.map +1 -1
  209. package/package.json +1 -1
  210. package/out/zero-cache/src/server/backup-watermark-poller.js +0 -31
  211. package/out/zero-cache/src/server/backup-watermark-poller.js.map +0 -1
  212. package/out/zero-cache/src/services/litestream/vfs-watermark-poller.d.ts +0 -57
  213. package/out/zero-cache/src/services/litestream/vfs-watermark-poller.d.ts.map +0 -1
  214. package/out/zero-cache/src/services/litestream/vfs-watermark-poller.js +0 -137
  215. package/out/zero-cache/src/services/litestream/vfs-watermark-poller.js.map +0 -1
@@ -2,6 +2,7 @@ import { assert, unreachable } from "../../../../shared/src/asserts.js";
2
2
  import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
3
3
  import { AbortError } from "../../../../shared/src/abort-error.js";
4
4
  import { parse, valita_exports } from "../../../../shared/src/valita.js";
5
+ import { BigIntJSON } from "../../../../shared/src/bigint-json.js";
5
6
  import { cdcSchema } from "../../types/shards.js";
6
7
  import { isDataChange, isSchemaChange } from "../change-source/protocol/current/data.js";
7
8
  import "../change-source/protocol/current/downstream.js";
@@ -67,6 +68,7 @@ var Storer = class {
67
68
  #statementTimeoutMs;
68
69
  #changeLogBatchSize;
69
70
  #drainTimeoutMs;
71
+ #progressMonitor;
70
72
  #approximateQueuedBytes = 0;
71
73
  #running = false;
72
74
  constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol, db, replicaVersion, onCommitted, onFatal, { backPressureLimitHeapProportion, statementTimeoutMs, changeLogBatchSize, drainTimeoutMs = 3e4 }) {
@@ -82,6 +84,7 @@ var Storer = class {
82
84
  this.#statementTimeoutMs = statementTimeoutMs;
83
85
  this.#changeLogBatchSize = Math.max(1, changeLogBatchSize);
84
86
  this.#drainTimeoutMs = drainTimeoutMs;
87
+ this.#progressMonitor = new ProgressMonitor(lc, statementTimeoutMs, onFatal);
85
88
  const heapStats = getHeapStatistics();
86
89
  this.#backPressureThresholdBytes = (heapStats.heap_size_limit - heapStats.used_heap_size) * backPressureLimitHeapProportion;
87
90
  this.#lc.info?.(`Using up to ${(this.#backPressureThresholdBytes / 1024 ** 2).toFixed(2)} MB of --max-old-space-size (~${(heapStats.heap_size_limit / 1024 ** 2).toFixed(2)} MB) to absorb upstream spikes`, { heapStats });
@@ -89,6 +92,28 @@ var Storer = class {
89
92
  #cdc(table) {
90
93
  return this.#db(`${cdcSchema(this.#shard)}.${table}`);
91
94
  }
95
+ /**
96
+ * Bounds a one-off db call (i.e. not part of the main storer loop or the
97
+ * background catchup read, which are tracked by the ProgressMonitor
98
+ * instead) with a plain timeout. This covers calls like
99
+ * {@link assumeOwnership} and {@link getStartStreamInitializationParameters}
100
+ * that are made by the caller *before* {@link run()} -- and thus before the
101
+ * ProgressMonitor's polling starts -- as well as ones made well after,
102
+ * where a continuously-polling watchdog would be overkill for a single
103
+ * bounded db round trip.
104
+ *
105
+ * Does not cancel `promise` on timeout (there is no cancellation mechanism
106
+ * for a postgres.js query); it just stops waiting for it, and swallows a
107
+ * later rejection so it doesn't surface as an unhandled rejection.
108
+ */
109
+ async #withTimeout(name, promise) {
110
+ const result = await orTimeout(promise, this.#statementTimeoutMs);
111
+ if (result === "timed-out") {
112
+ promise.catch(() => {});
113
+ throw new AbortError(`${name} did not complete within ${this.#statementTimeoutMs}ms`);
114
+ }
115
+ return result;
116
+ }
92
117
  async assumeOwnership(purgeLock) {
93
118
  const db = this.#db;
94
119
  const owner = this.#taskID;
@@ -97,10 +122,10 @@ var Storer = class {
97
122
  const addressWithProtocol = ownerProtocol === "ws" ? ownerAddress : `${ownerProtocol}://${ownerAddress}`;
98
123
  this.#lc.info?.(`assuming ownership at ${addressWithProtocol}`);
99
124
  const start = performance.now();
100
- await db`UPDATE ${this.#cdc("replicationState")} SET ${db({
125
+ await this.#withTimeout("assume-ownership", db`UPDATE ${this.#cdc("replicationState")} SET ${db({
101
126
  owner,
102
127
  ownerAddress: addressWithProtocol
103
- })}`;
128
+ })}`);
104
129
  const elapsed = (performance.now() - start).toFixed(2);
105
130
  this.#lc.info?.(`assumed ownership at ${addressWithProtocol} (${elapsed} ms)`);
106
131
  if (purgeLock) purgeLock.release();
@@ -123,7 +148,7 @@ var Storer = class {
123
148
  * snapshot that the SQLite change log continues folding onto.
124
149
  */
125
150
  async getStartStreamInitializationParameters() {
126
- const [[{ lastWatermark }], result, tableMetadata, backfilling] = await runTx(this.#db, (sql) => [
151
+ const [[{ lastWatermark }], result, tableMetadata, backfilling] = await this.#withTimeout("get-stream-params", runTx(this.#db, (sql) => [
127
152
  sql`
128
153
  SELECT "lastWatermark" FROM ${this.#cdc("replicationState")}`,
129
154
  sql`
@@ -146,7 +171,7 @@ var Storer = class {
146
171
  sql`
147
172
  SELECT "schema", "table", "column", "backfill" FROM ${this.#cdc("backfilling")}
148
173
  ORDER BY "schema" COLLATE "C", "table" COLLATE "C", "column" COLLATE "C"`
149
- ], { mode: READONLY });
174
+ ], { mode: READONLY }));
150
175
  return {
151
176
  lastWatermark,
152
177
  backfillRequests: parse(result, backfillRequestsSchema),
@@ -157,10 +182,50 @@ var Storer = class {
157
182
  };
158
183
  }
159
184
  async getMinWatermarkForCatchup() {
160
- const [{ minWatermark }] = await this.#db`
161
- SELECT min(watermark) as "minWatermark" FROM ${this.#cdc("changeLog")}`;
185
+ const [{ minWatermark }] = await this.#withTimeout("get-min-watermark", this.#db`
186
+ SELECT min(watermark) as "minWatermark" FROM ${this.#cdc("changeLog")}`);
162
187
  return minWatermark;
163
188
  }
189
+ /**
190
+ * Returns the retained Postgres catchup bounds.
191
+ * `minWatermark` is the oldest transaction, or `null` for a new replica.
192
+ * `lastWatermark` is the durable head.
193
+ */
194
+ async getCatchupBounds() {
195
+ const [bounds] = await this.#withTimeout("get-catchup-bounds", this.#db`
196
+ SELECT
197
+ (SELECT min(watermark) FROM ${this.#cdc("changeLog")}) as "minWatermark",
198
+ (SELECT "lastWatermark" FROM ${this.#cdc("replicationState")}) as "lastWatermark"`);
199
+ return bounds;
200
+ }
201
+ /**
202
+ * Lists at most `limit` committed transaction watermarks in
203
+ * `(afterWatermark, throughWatermark]`, in ascending order.
204
+ */
205
+ async listCommitWatermarks(afterWatermark, throughWatermark, limit) {
206
+ return (await this.#db`
207
+ SELECT watermark FROM ${this.#cdc("changeLog")}
208
+ WHERE precommit IS NOT NULL
209
+ AND watermark > ${afterWatermark}
210
+ AND watermark <= ${throughWatermark}
211
+ ORDER BY watermark
212
+ LIMIT ${limit}`).map(({ watermark }) => watermark);
213
+ }
214
+ /**
215
+ * Reads `(afterWatermark, throughWatermark]` in stream order and bounded batches.
216
+ * This query matches the subscriber catchup query.
217
+ *
218
+ * Both queries fail when an escaped NUL reaches `change->'tag'`.
219
+ * Postgres cannot convert that value to text.
220
+ */
221
+ readCatchupRange(afterWatermark, throughWatermark, batchRows = 2e3) {
222
+ assert(Number.isSafeInteger(batchRows) && batchRows > 0, "Postgres change log batch size must be a positive safe integer");
223
+ return this.#db`
224
+ SELECT watermark, change->'tag' as tag, change::text FROM ${this.#cdc("changeLog")}
225
+ WHERE watermark > ${afterWatermark}
226
+ AND watermark <= ${throughWatermark}
227
+ ORDER BY watermark, pos`.cursor(batchRows);
228
+ }
164
229
  purgeRecordsBefore(watermark) {
165
230
  return runTx(this.#db, async (sql) => {
166
231
  await sql`
@@ -214,30 +279,22 @@ var Storer = class {
214
279
  }]);
215
280
  }
216
281
  #readyForMore = null;
217
- #backpressureTimeout;
218
282
  readyForMore() {
219
283
  if (!this.#running) return;
220
284
  if (this.#readyForMore === null && this.#approximateQueuedBytes > this.#backPressureThresholdBytes) {
221
285
  this.#lc.warn?.(`applying back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)\n\nTo inspect changeLog backlog in your change DB:\n SELECT\n (change->'relation'->>'schema') || '.' || (change->'relation'->>'name') AS table_name,\n change->>'tag' AS operation,\n COUNT(*) AS count\n FROM "<app_id>/cdc"."changeLog"\n GROUP BY 1, 2\n ORDER BY 3 DESC\n LIMIT 20;`);
222
286
  this.#readyForMore = resolver();
223
- this.#resetBackpressureTimeout(true);
224
287
  }
225
288
  return this.#readyForMore?.promise;
226
289
  }
227
- #resetBackpressureTimeout(reschedule) {
228
- clearTimeout(this.#backpressureTimeout);
229
- this.#backpressureTimeout = reschedule ? setTimeout(() => {
230
- this.#readyForMore?.reject(new AbortError(`No statement processed within the last ${this.#statementTimeoutMs}ms. Considering the change-log to be wedged.`));
231
- this.#readyForMore = null;
232
- }, this.#statementTimeoutMs) : void 0;
233
- }
234
290
  #maybeReleaseBackPressure() {
235
- if (this.#readyForMore !== null) if (this.#approximateQueuedBytes < this.#backPressureThresholdBytes * .8) {
236
- this.#lc.info?.(`releasing back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)`);
237
- this.#readyForMore.resolve();
238
- this.#readyForMore = null;
239
- this.#resetBackpressureTimeout(false);
240
- } else this.#resetBackpressureTimeout(true);
291
+ if (this.#readyForMore !== null) {
292
+ if (this.#approximateQueuedBytes < this.#backPressureThresholdBytes * .8) {
293
+ this.#lc.info?.(`releasing back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)`);
294
+ this.#readyForMore.resolve();
295
+ this.#readyForMore = null;
296
+ }
297
+ }
241
298
  }
242
299
  /**
243
300
  * Flushes any buffered {@link PendingTransaction.batch} rows to the changeLog.
@@ -276,6 +333,7 @@ var Storer = class {
276
333
  this.#running = true;
277
334
  this.#stopped = stopped;
278
335
  this.#lc.info?.("starting storer");
336
+ this.#progressMonitor.start();
279
337
  let err;
280
338
  try {
281
339
  await this.#processQueue();
@@ -287,7 +345,6 @@ var Storer = class {
287
345
  this.#readyForMore.resolve();
288
346
  this.#readyForMore = null;
289
347
  }
290
- this.#resetBackpressureTimeout(false);
291
348
  this.#cancelQueueEntries(this.#queue.drain().filter((entry) => entry !== void 0), err);
292
349
  this.#running = false;
293
350
  signalStopped();
@@ -313,9 +370,20 @@ var Storer = class {
313
370
  async #processQueue() {
314
371
  let tx = null;
315
372
  let msg;
373
+ let lastTaskDone;
374
+ const nextMessage = async () => {
375
+ lastTaskDone?.();
376
+ lastTaskDone = void 0;
377
+ const entry = await this.#queue.dequeue();
378
+ if (entry !== "stop") lastTaskDone = this.#progressMonitor.trackTask({
379
+ task: "queue-entry",
380
+ type: entry[0]
381
+ });
382
+ return entry;
383
+ };
316
384
  const catchupQueue = [];
317
385
  try {
318
- while ((msg = await this.#queue.dequeue()) !== "stop") {
386
+ while ((msg = await nextMessage()) !== "stop") {
319
387
  const [msgType] = msg;
320
388
  switch (msgType) {
321
389
  case "ready": {
@@ -345,10 +413,7 @@ var Storer = class {
345
413
  const { promise, resolve, reject } = resolver();
346
414
  promise.catch(() => {});
347
415
  tx = {
348
- pool: new TransactionPool(this.#lc.withContext("watermark", watermark), {
349
- mode: READ_COMMITTED,
350
- statementResponseTimeout: this.#statementTimeoutMs
351
- }),
416
+ pool: new TransactionPool(this.#lc.withContext("watermark", watermark), { mode: READ_COMMITTED }),
352
417
  preCommitWatermark: watermark,
353
418
  pos: 0,
354
419
  startingReplicationState: promise,
@@ -420,6 +485,10 @@ var Storer = class {
420
485
  const reader = new TransactionPool(lc, { mode: READONLY });
421
486
  reader.run(this.#db);
422
487
  let lastWatermark;
488
+ const catchupSnapshotted = this.#progressMonitor.trackTask({
489
+ task: "capture-catchup-snapshot",
490
+ subscribers: subs.map(({ subscriber: s }) => s.id)
491
+ });
423
492
  try {
424
493
  [{lastWatermark}] = await reader.processReadTask((sql) => sql`
425
494
  SELECT "lastWatermark" FROM ${this.#cdc("replicationState")}
@@ -427,10 +496,18 @@ var Storer = class {
427
496
  } catch (e) {
428
497
  subs.map(({ subscriber }) => subscriber.fail(e));
429
498
  throw e;
499
+ } finally {
500
+ catchupSnapshotted();
430
501
  }
431
502
  Promise.all(subs.map((sub) => this.#catchup(lc.withContext("subscriber", sub.subscriber.id), sub, lastWatermark, reader))).finally(() => reader.setDone());
432
503
  }
433
504
  async #catchup(lc, { subscriber: sub, mode }, lastWatermark, reader) {
505
+ let entriesReceived = 0;
506
+ let catchupProgressed = this.#progressMonitor.trackTask({
507
+ task: "catchup",
508
+ subscriber: sub.id,
509
+ entriesReceived
510
+ });
434
511
  try {
435
512
  lc.info?.(`starting catchup`);
436
513
  await reader.processReadTask(async (tx) => {
@@ -444,21 +521,32 @@ var Storer = class {
444
521
  WHERE watermark >= ${sub.watermark}
445
522
  AND watermark <= ${lastWatermark}
446
523
  ORDER BY watermark, pos`.cursor(2e3)) {
447
- const start = performance.now();
448
- await lastBatchConsumed;
449
- const elapsed = performance.now() - start;
450
- if (lastBatchConsumed) lc[elapsed > 100 ? "info" : "debug"]?.(`waited ${elapsed.toFixed(3)} ms for ${sub.id} to consume last batch of catchup entries`);
451
- for (const entry of entries) if (entry.watermark === sub.watermark) watermarkFound = true;
452
- else if (watermarkFound) {
453
- lastBatchConsumed = sub.catchup(reconstructWatermarkedChange(entry));
454
- count++;
455
- } else if (mode === "backup") throw new AutoResetSignal(`backup replica at watermark ${sub.watermark} is behind change db: ${entry.watermark})`);
456
- else {
457
- lc.warn?.(`rejecting subscriber at watermark ${sub.watermark} (earliest watermark: ${entry.watermark})`);
458
- sub.close(2, `earliest supported watermark is ${entry.watermark} (requested ${sub.watermark})`);
459
- return;
524
+ catchupProgressed();
525
+ entriesReceived += entries.length;
526
+ try {
527
+ const start = performance.now();
528
+ await lastBatchConsumed;
529
+ const elapsed = performance.now() - start;
530
+ if (lastBatchConsumed) lc[elapsed > 100 ? "info" : "debug"]?.(`waited ${elapsed.toFixed(3)} ms for ${sub.id} to consume last batch of catchup entries`);
531
+ for (const entry of entries) if (entry.watermark === sub.watermark) watermarkFound = true;
532
+ else if (watermarkFound) {
533
+ lastBatchConsumed = sub.catchup(reconstructWatermarkedChange(entry));
534
+ count++;
535
+ } else if (mode === "backup") throw new AutoResetSignal(`backup replica at watermark ${sub.watermark} is behind change db: ${entry.watermark})`);
536
+ else {
537
+ lc.warn?.(`rejecting subscriber at watermark ${sub.watermark} (earliest watermark: ${entry.watermark})`);
538
+ sub.close(2, `earliest supported watermark is ${entry.watermark} (requested ${sub.watermark})`);
539
+ return;
540
+ }
541
+ } finally {
542
+ catchupProgressed = this.#progressMonitor.trackTask({
543
+ task: "catchup",
544
+ subscriber: sub.id,
545
+ entriesReceived
546
+ });
460
547
  }
461
548
  }
549
+ catchupProgressed();
462
550
  if (watermarkFound) {
463
551
  await lastBatchConsumed;
464
552
  lc.info?.(`caught up ${sub.id} with ${count} changes (${Date.now() - start} ms)`);
@@ -472,6 +560,8 @@ var Storer = class {
472
560
  this.#onFatal(err);
473
561
  }
474
562
  sub.fail(err);
563
+ } finally {
564
+ catchupProgressed();
475
565
  }
476
566
  }
477
567
  /**
@@ -569,6 +659,7 @@ var Storer = class {
569
659
  */
570
660
  async stop() {
571
661
  if (this.#running) {
662
+ this.#progressMonitor.stop();
572
663
  this.#lc.info?.(`draining ${this.#queue.size()} changeLog entries`);
573
664
  this.abort();
574
665
  this.#queue.enqueue("stop");
@@ -629,6 +720,65 @@ var PurgeLocker = class {
629
720
  return new PurgeLock(this.#lc, tx, replicaVersion, watermark);
630
721
  }
631
722
  };
723
+ /**
724
+ * Periodic monitor for detecting the pathological condition in which a db
725
+ * query hangs forever because of a connection problem, such as a half-closed
726
+ * socket.
727
+ *
728
+ * Use this for db calls that recur or accumulate progress within a single
729
+ * logical operation on the main storer loop (e.g. the queue-processing loop,
730
+ * or a multi-batch catchup read), where "no progress for N ms" is the
731
+ * meaningful signal. For a one-off db call -- including ones made before
732
+ * {@link Storer.run} starts this monitor's polling -- use a plain timeout
733
+ * (see `Storer.#withTimeout`) instead.
734
+ *
735
+ * The failure handler should stop the server. The ProgressMonitor will
736
+ * assume this and stop itself after calling it.
737
+ *
738
+ * Internal to the Storer, but exported for testing.
739
+ */
740
+ var ProgressMonitor = class {
741
+ #lc;
742
+ #failureThresholdMs;
743
+ #onFailure;
744
+ #taskUpdates = /* @__PURE__ */ new Map();
745
+ #timer;
746
+ constructor(lc, failureThresholdMs, onFailure) {
747
+ this.#lc = lc.withContext("component", "storer-progress-monitor");
748
+ this.#failureThresholdMs = failureThresholdMs;
749
+ this.#onFailure = onFailure;
750
+ }
751
+ start() {
752
+ clearInterval(this.#timer);
753
+ this.#timer = setInterval(() => this.checkTaskProgress(/* @__PURE__ */ new Date()), this.#failureThresholdMs).unref();
754
+ }
755
+ checkTaskProgress(now) {
756
+ for (const [task, lastUpdate] of this.#taskUpdates.entries()) if (now.getTime() - lastUpdate.getTime() >= this.#failureThresholdMs) {
757
+ this.#lc.error?.(`Last task update was more than ${this.#failureThresholdMs}ms ago`, {
758
+ task,
759
+ lastUpdate
760
+ });
761
+ this.#onFailure(/* @__PURE__ */ new Error(`Task failed to progress for over ${this.#failureThresholdMs}ms: ${BigIntJSON.stringify(task)}`));
762
+ this.stop();
763
+ return;
764
+ }
765
+ }
766
+ /**
767
+ * Starts a task to be tracked.
768
+ *
769
+ * @return a callback to invoke when the task is done.
770
+ */
771
+ trackTask(task, now = /* @__PURE__ */ new Date()) {
772
+ this.#taskUpdates.set(task, now);
773
+ return () => {
774
+ this.#taskUpdates.delete(task);
775
+ };
776
+ }
777
+ stop() {
778
+ clearInterval(this.#timer);
779
+ this.#taskUpdates.clear();
780
+ }
781
+ };
632
782
  //#endregion
633
783
  export { PurgeLocker, Storer };
634
784