@rocicorp/zero 1.11.0-canary.11 → 1.11.0-canary.12

package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts

@@ -1,11 +1,14 @@
 import type { Enum } from '../../../../shared/src/enum.ts';
 import * as v from '../../../../shared/src/valita.ts';
 import type { PreSerialized, Sized, Source } from '../../types/streams.ts';
+import type { Subscription } from '../../types/subscription.ts';
 import { type ChangeStreamData } from '../change-source/protocol/current/downstream.ts';
 import type { ReplicatorMode } from '../replicator/replicator.ts';
 import type { Service } from '../service.ts';
+import type { PreSerializedBatch } from './broadcast.ts';
 import * as ErrorType from './error-type-enum.ts';
-import type { SnapshotMessage } from './snapshot.ts';
+import type { SnapshotMessage } from './snapshot-message.ts';
+import type { SubscribeDownstream } from './subscribe.ts';
 type ErrorType = Enum<typeof ErrorType>;
 export type ChangeTag = ChangeStreamData[1]['tag'];
 /**
@@ -62,9 +65,9 @@ export interface ChangeStreamer {
      * date. Each result contains parsed, validated data and its approximate
      * serialized transport size.
      */
-    subscribe(ctx: SubscriberContext): Promise<Source<SizedDownstream>>;
+    subscribe(ctx: SubscriberContext): Promise<Source<Sized<SubscribeDownstream>>>;
 }
-export declare const PROTOCOL_VERSION = 6;
+export declare const PROTOCOL_VERSION = 7;
 export type SubscriberContext = {
     /**
      * The supported change-streamer protocol version.
@@ -96,19 +99,6 @@ export type SubscriberContext = {
      * Only changes after the watermark will be streamed.
      */
     watermark: string;
-    /**
-     * Whether this is the first subscription request made by the task,
-     * i.e. indicating that the watermark comes from a restored replica
-     * backup. The ChangeStreamer uses this to determine which changes
-     * are safe to purge from the Storer.
-     */
-    initial: boolean;
-    /**
-     * Legacy topology hint retained on the wire. It no longer affects local
-     * routing: the change-streamer writes the SQLite log itself, so no
-     * subscriber's ACK advances its head or releases its catchup barrier.
-     */
-    logsChangeStream: boolean;
     /**
      * Whether the subscriber supports batched WebSocket frames.
      */
@@ -418,8 +408,11 @@ export interface ChangeStreamerService extends Omit<ChangeStreamer, 'subscribe'>
     /**
      * The server-side interface overrides `subscribe()` to return a stream
      * of already-stringified {@link Downstream} payloads or pre-serialized Buffers.
+     *
+     * If a `downstream` subscription is supplied it is used directly as the
+     * returned Source. Otherwise, a new one is created.
      */
-    subscribe(ctx: SubscriberContext): Promise<Source<string | PreSerialized>>;
+    subscribe(ctx: SubscriberContext, downstream?: Subscription<string | PreSerializedBatch>): Promise<Source<string | PreSerialized>>;
     /**
      * Starts a snapshot reservation to preserve change-log entries while
      * a soon-to-be-subscriber downloads the current replica backup. Once

package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,gCAAgC,CAAC;AACzD,OAAO,KAAK,CAAC,MAAM,kCAAkC,CAAC;AACtD,OAAO,KAAK,EAAC,aAAa,EAAE,KAAK,EAAE,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACzE,OAAO,EAEL,KAAK,gBAAgB,EACtB,MAAM,iDAAiD,CAAC;AACzD,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAEhE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,KAAK,SAAS,MAAM,sBAAsB,CAAC;AAClD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,eAAe,CAAC;AAEnD,KAAK,SAAS,GAAG,IAAI,CAAC,OAAO,SAAS,CAAC,CAAC;AAExC,MAAM,MAAM,SAAS,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;AAEnD;;;;;;;GAOG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,MAAM;IACjB,GAAG,EAAE,SAAS;IACd,IAAI,EAAE,MAAM;CACb,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,MAAM,WAAW,cAAc;IAC7B;;;;;OAKG;IACH,SAAS,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC;CACrE;AAgCD,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAElC,MAAM,MAAM,iBAAiB,GAAG;IAC9B;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;;;OAIG;IACH,IAAI,EAAE,cAAc,CAAC;IAErB;;;;OAIG;IACH,cAAc,EAAE,MAAM,CAAC;IAEvB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;;;;OAKG;IACH,OAAO,EAAE,OAAO,CAAC;IAEjB;;;;OAIG;IACH,gBAAgB,EAAE,OAAO,CAAC;IAE1B;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CACjC,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,YAAY;;;;;;;;;;aASvB,CAAC;AAEH,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AAElD,eAAO,MAAM,mBAAmB;;;;;;;;;;eAA+C,CAAC;AAEhF;;;;;GAKG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE,QAAA,MAAM,uBAAuB;;;aAG3B,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,QAAA,MAAM,WAAW;;;eAAyD,CAAC;AAE3E,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAI5B,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC;AAEhD,wBAAgB,uBAAuB,CAAC,GAAG,EAAE,SAAS,yDAWrD;AAED;;;;;;;;;;GAUG;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,0EAA0E;AAC1E,MAAM,MAAM,eAAe,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC;AAEhD,MAAM,WAAW,qBACf,SAAQ,IAAI,CAAC,cAAc,EAAE,WAAW,CAAC,EAAE,OAAO;IAClD;;;OAGG;IACH,SAAS,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC;IAE3E;;;;;;;;OAQG;IACH,wBAAwB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC;IAE3E;;;;;;;;;OASG;IACH,oBAAoB,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/C"}
+{"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,gCAAgC,CAAC;AACzD,OAAO,KAAK,CAAC,MAAM,kCAAkC,CAAC;AACtD,OAAO,KAAK,EAAC,aAAa,EAAE,KAAK,EAAE,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACzE,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAC9D,OAAO,EAEL,KAAK,gBAAgB,EACtB,MAAM,iDAAiD,CAAC;AACzD,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAEhE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,gBAAgB,CAAC;AACvD,OAAO,KAAK,SAAS,MAAM,sBAAsB,CAAC;AAClD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,uBAAuB,CAAC;AAC3D,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,gBAAgB,CAAC;AAExD,KAAK,SAAS,GAAG,IAAI,CAAC,OAAO,SAAS,CAAC,CAAC;AAExC,MAAM,MAAM,SAAS,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;AAEnD;;;;;;;GAOG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,MAAM;IACjB,GAAG,EAAE,SAAS;IACd,IAAI,EAAE,MAAM;CACb,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,MAAM,WAAW,cAAc;IAC7B;;;;;OAKG;IACH,SAAS,CACP,GAAG,EAAE,iBAAiB,GACrB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC;CAChD;AA2CD,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAElC,MAAM,MAAM,iBAAiB,GAAG;IAC9B;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;;;OAIG;IACH,IAAI,EAAE,cAAc,CAAC;IAErB;;;;OAIG;IACH,cAAc,EAAE,MAAM,CAAC;IAEvB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CACjC,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,YAAY;;;;;;;;;;aASvB,CAAC;AAEH,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AAElD,eAAO,MAAM,mBAAmB;;;;;;;;;;eAA+C,CAAC;AAEhF;;;;;GAKG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE,QAAA,MAAM,uBAAuB;;;aAG3B,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,QAAA,MAAM,WAAW;;;eAAyD,CAAC;AAE3E,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAI5B,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC;AAEhD,wBAAgB,uBAAuB,CAAC,GAAG,EAAE,SAAS,yDAWrD;AAED;;;;;;;;;;GAUG;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,0EAA0E;AAC1E,MAAM,MAAM,eAAe,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC;AAEhD,MAAM,WAAW,qBACf,SAAQ,IAAI,CAAC,cAAc,EAAE,WAAW,CAAC,EAAE,OAAO;IAClD;;;;;;OAMG;IACH,SAAS,CACP,GAAG,EAAE,iBAAiB,EACtB,UAAU,CAAC,EAAE,YAAY,CAAC,MAAM,GAAG,kBAAkB,CAAC,GACrD,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC;IAE3C;;;;;;;;OAQG;IACH,wBAAwB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC;IAE3E;;;;;;;;;OASG;IACH,oBAAoB,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/C"}

package/out/zero-cache/src/services/change-streamer/change-streamer.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"sourcesContent":["import type {Enum} from '../../../../shared/src/enum.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {PreSerialized, Sized, Source} from '../../types/streams.ts';\nimport {\n  changeStreamDataSchema,\n  type ChangeStreamData,\n} from '../change-source/protocol/current/downstream.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport {changeSourceTimingsSchema} from '../replicator/reporter/report-schema.ts';\nimport type {Service} from '../service.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport type {SnapshotMessage} from './snapshot.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nexport type ChangeTag = ChangeStreamData[1]['tag'];\n\n/**\n * Internally all downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering while replaying stored changes and\n * filtering changes already seen by a subscriber.\n *\n * Only commit watermarks are exposed to subscribers. The JSON string is the\n * canonical serialization shared by storage and live forwarding.\n */\nexport type WatermarkedChange = [\n  watermark: string,\n  tag: ChangeTag,\n  json: string,\n];\n\n/**\n * The ChangeStreamer is the component between replicators (\"subscribers\")\n * and a canonical upstream source of changes (e.g. a Postgres logical\n * replication slot). It facilitates multiple subscribers without incurring\n * the associated upstream expense (e.g. PG replication slots are resource\n * intensive) with a \"forward-store-ack\" procedure.\n *\n * * Changes from the upstream source are immediately **forwarded** to\n *   connected subscribers to minimize latency.\n *\n * * They are then **stored** in a separate DB to facilitate catchup\n *   of connecting subscribers that are behind.\n *\n * * **Acknowledgements** are sent upstream after they are successfully\n *   stored.\n *\n * Unlike Postgres replication slots, in which the progress of a static\n * subscriber is tracked in the replication slot, the ChangeStreamer\n * supports a dynamic set of subscribers (i.e.. zero-caches) that can\n * can continually change.\n *\n * However, it is not the case that the ChangeStreamer needs to support\n * arbitrarily old subscribers. Because the replica is continually\n * backed up to a global location and used to initialize new subscriber\n * tasks, an initial subscription request from a subscriber constitutes\n * a signal for how \"behind\" a new subscriber task can be. This is\n * reflected in the {@link SubscriberContext}, which indicates whether\n * the watermark corresponds to an \"initial\" watermark derived from the\n * replica at task startup.\n *\n * The ChangeStreamer uses a combination of this signal with ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\n */\nexport interface ChangeStreamer {\n  /**\n   * Subscribes to changes based on the supplied subscriber `ctx`,\n   * which indicates the watermark at which the subscriber is up to\n   * date. Each result contains parsed, validated data and its approximate\n   * serialized transport size.\n   */\n  subscribe(ctx: SubscriberContext): Promise<Source<SizedDownstream>>;\n}\n\n// v1: v0.18\n//   - Client-side support for JSON_FORMAT. Introduced in 0.18.\n// v2: v0.19\n//   - Adds the \"status\" message which is initially used to signal that the\n//     subscription is valid (i.e. starting at the requested watermark).\n// v3: v0.20\n//   - Adds the \"taskID\" to the subscription context, and support for\n//     the backup monitor-mediated \"/snapshot\" request.\n// v4: v0.25\n//   - Adds the \"replicaVersion\" and \"minWatermark\" fields to the \"/snapshot\"\n//     status request so that a subscriber can verify whether its replica,\n//     whether it be restored or existing in a permanent volume, is compatible\n//     with the change-streamer.\n// v5: v0.26\n//   - Moves relation.keyColumns and relation.replicaIdentity to\n//     relation.rowKey: { columns, type }.\n//   - Adds `metadata` to `create-table` message\n//   - Adds `tableMetadata` to `add-column` message\n//   - Adds `table-update-metadata` message\n// v6: v0.26\n//   - Adds support for `backfill` messages\n// v6: v1.0.1  (backwards compatible, no version change)\n//   - Adds lag reporting to status messages\n// v6: (backwards compatible, no version change)\n//   - Adds the optional `commitTimeMs` field to `commit` messages, carrying\n//     the upstream commit timestamp for end-to-end serving lag measurement.\n//     The stream is parsed in 'passthrough' mode, so an older peer ignores the\n//     field, and a newer peer treats its absence (including in changes\n//     replayed from the Change DB) as \"no commit time reported\".\n\nexport const PROTOCOL_VERSION = 6;\n\nexport type SubscriberContext = {\n  /**\n   * The supported change-streamer protocol version.\n   */\n  protocolVersion: number;\n\n  /**\n   * Task ID. This is used to link the request with a preceding snapshot\n   * reservation.\n   */\n  taskID: string;\n\n  /**\n   * Subscriber id. This is only used for debugging.\n   */\n  id: string;\n\n  /**\n   * The ReplicatorMode of the subscriber. 'backup' indicates that the\n   * subscriber is local to the `change-streamer` in the `replication-manager`,\n   * while 'serving' indicates that user-facing requests depend on the subscriber.\n   */\n  mode: ReplicatorMode;\n\n  /**\n   * The ChangeStreamer will return an Error if the subscriber is\n   * on a different replica version (i.e. the initial snapshot associated\n   * with the replication slot).\n   */\n  replicaVersion: string;\n\n  /**\n   * The watermark up to which the subscriber is up to date.\n   * Only changes after the watermark will be streamed.\n   */\n  watermark: string;\n\n  /**\n   * Whether this is the first subscription request made by the task,\n   * i.e. indicating that the watermark comes from a restored replica\n   * backup. The ChangeStreamer uses this to determine which changes\n   * are safe to purge from the Storer.\n   */\n  initial: boolean;\n\n  /**\n   * Legacy topology hint retained on the wire. It no longer affects local\n   * routing: the change-streamer writes the SQLite log itself, so no\n   * subscriber's ACK advances its head or releases its catchup barrier.\n   */\n  logsChangeStream: boolean;\n\n  /**\n   * Whether the subscriber supports batched WebSocket frames.\n   */\n  wsBatched?: boolean | undefined;\n};\n\n/**\n * The StatusMessage payload for now is empty, but can be extended to\n * include meta-level information in the future.\n */\nexport const statusSchema = v.object({\n  tag: v.literal('status'),\n\n  lagReport: v\n    .object({\n      lastTimings: changeSourceTimingsSchema.optional(),\n      nextSendTimeMs: v.number(),\n    })\n    .optional(),\n});\n\nexport type Status = v.Infer<typeof statusSchema>;\n\nexport const statusMessageSchema = v.tuple([v.literal('status'), statusSchema]);\n\n/**\n * A StatusMessage will be immediately sent on a (v2+) subscription to\n * indicate that the subscription is valid (i.e. starting at the requested\n * watermark). Invalid subscriptions will instead result in a\n * SubscriptionError as the first message.\n */\nexport type StatusMessage = v.Infer<typeof statusMessageSchema>;\n\nconst subscriptionErrorSchema = v.object({\n  type: v.number(), // ErrorType\n  message: v.string().optional(),\n});\n\nexport type SubscriptionError = v.Infer<typeof subscriptionErrorSchema>;\n\nconst errorSchema = v.tuple([v.literal('error'), subscriptionErrorSchema]);\n\nexport const downstreamSchema = v.union(\n  statusMessageSchema,\n  changeStreamDataSchema,\n  errorSchema,\n);\n\nexport type Error = v.Infer<typeof errorSchema>;\n\nexport function errorTypeToReadableName(val: ErrorType) {\n  switch (val) {\n    case ErrorType.WrongReplicaVersion:\n      return 'WrongReplicaVersion';\n    case ErrorType.WatermarkTooOld:\n      return 'WatermarkTooOld';\n    case ErrorType.Unknown:\n      return 'Unknown';\n    default:\n      return 'Unknown';\n  }\n}\n\n/**\n * A stream of transactions, each starting with a {@link Begin} message,\n * containing one or more {@link Data} messages, and ending with a\n * {@link Commit} or {@link Rollback} message. The 'commit' tuple\n * includes a `watermark` that should be stored with the committed\n * data and used for resuming a subscription (e.g. in the\n * {@link SubscriberContext}).\n *\n * A {@link SubscriptionError} indicates an unrecoverable error that requires\n * manual intervention (e.g. configuration / operational error).\n */\nexport type Downstream = v.Infer<typeof downstreamSchema>;\n\n/** A downstream message and its approximate serialized transport size. */\nexport type SizedDownstream = Sized<Downstream>;\n\nexport interface ChangeStreamerService\n  extends Omit<ChangeStreamer, 'subscribe'>, Service {\n  /**\n   * The server-side interface overrides `subscribe()` to return a stream\n   * of already-stringified {@link Downstream} payloads or pre-serialized Buffers.\n   */\n  subscribe(ctx: SubscriberContext): Promise<Source<string | PreSerialized>>;\n\n  /**\n   * Starts a snapshot reservation to preserve change-log entries while\n   * a soon-to-be-subscriber downloads the current replica backup. Once\n   * the change-streamer knows that a backup is available\n   * (via {@link trackBackupWatermark}), it sends a confirmation\n   * SnapshotMessage and reserves the corresponding entries in the change-log\n   * until the client with that `taskID` subscribes, or if the snapshot\n   * connection terminates.\n   */\n  startSnapshotReservation(taskID: string): Promise<Source<SnapshotMessage>>;\n\n  /**\n   * Informs the change-streamer of the watermark up to which the replica has\n   * been backed up. This serves two purposes:\n   * - It serves as the maximum watermark up to which the change-log can be\n   *   purged. Note that the change-streamer also takes into account the\n   *   positions of its subscribers and snapshot reservations when purging\n   *   changes in the change-log.\n   * - In RMv2, this ACK's the upstream change-source to allow its buffer of\n   *   changes (e.g. the replication_slot) to advance.\n   */\n  trackBackupWatermark(watermark: string): void;\n}\n"],"mappings":";;;;;;;;AAyKA,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CAEvB,WAAW,eACR,OAAO;EACN,aAAa,0BAA0B,SAAS;EAChD,gBAAgB,eAAE,OAAO;CAC3B,CAAC,EACA,SAAS;AACd,CAAC;AAID,IAAa,sBAAsB,eAAE,MAAM,CAAC,eAAE,QAAQ,QAAQ,GAAG,YAAY,CAAC;AAU9E,IAAM,0BAA0B,eAAE,OAAO;CACvC,MAAM,eAAE,OAAO;CACf,SAAS,eAAE,OAAO,EAAE,SAAS;AAC/B,CAAC;AAID,IAAM,cAAc,eAAE,MAAM,CAAC,eAAE,QAAQ,OAAO,GAAG,uBAAuB,CAAC;AAEzE,IAAa,mBAAmB,eAAE,MAChC,qBACA,wBACA,WACF;AAIA,SAAgB,wBAAwB,KAAgB;CACtD,QAAQ,KAAR;EACE,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,SACE,OAAO;CACX;AACF"}
+{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"sourcesContent":["import type {Enum} from '../../../../shared/src/enum.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {PreSerialized, Sized, Source} from '../../types/streams.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport {\n  changeStreamDataSchema,\n  type ChangeStreamData,\n} from '../change-source/protocol/current/downstream.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport {changeSourceTimingsSchema} from '../replicator/reporter/report-schema.ts';\nimport type {Service} from '../service.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport type {SnapshotMessage} from './snapshot-message.ts';\nimport type {SubscribeDownstream} from './subscribe.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nexport type ChangeTag = ChangeStreamData[1]['tag'];\n\n/**\n * Internally all downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering while replaying stored changes and\n * filtering changes already seen by a subscriber.\n *\n * Only commit watermarks are exposed to subscribers. The JSON string is the\n * canonical serialization shared by storage and live forwarding.\n */\nexport type WatermarkedChange = [\n  watermark: string,\n  tag: ChangeTag,\n  json: string,\n];\n\n/**\n * The ChangeStreamer is the component between replicators (\"subscribers\")\n * and a canonical upstream source of changes (e.g. a Postgres logical\n * replication slot). It facilitates multiple subscribers without incurring\n * the associated upstream expense (e.g. PG replication slots are resource\n * intensive) with a \"forward-store-ack\" procedure.\n *\n * * Changes from the upstream source are immediately **forwarded** to\n *   connected subscribers to minimize latency.\n *\n * * They are then **stored** in a separate DB to facilitate catchup\n *   of connecting subscribers that are behind.\n *\n * * **Acknowledgements** are sent upstream after they are successfully\n *   stored.\n *\n * Unlike Postgres replication slots, in which the progress of a static\n * subscriber is tracked in the replication slot, the ChangeStreamer\n * supports a dynamic set of subscribers (i.e.. zero-caches) that can\n * can continually change.\n *\n * However, it is not the case that the ChangeStreamer needs to support\n * arbitrarily old subscribers. Because the replica is continually\n * backed up to a global location and used to initialize new subscriber\n * tasks, an initial subscription request from a subscriber constitutes\n * a signal for how \"behind\" a new subscriber task can be. This is\n * reflected in the {@link SubscriberContext}, which indicates whether\n * the watermark corresponds to an \"initial\" watermark derived from the\n * replica at task startup.\n *\n * The ChangeStreamer uses a combination of this signal with ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\n */\nexport interface ChangeStreamer {\n  /**\n   * Subscribes to changes based on the supplied subscriber `ctx`,\n   * which indicates the watermark at which the subscriber is up to\n   * date. Each result contains parsed, validated data and its approximate\n   * serialized transport size.\n   */\n  subscribe(\n    ctx: SubscriberContext,\n  ): Promise<Source<Sized<SubscribeDownstream>>>;\n}\n\n// v1: v0.18\n//   - Client-side support for JSON_FORMAT. Introduced in 0.18.\n// v2: v0.19\n//   - Adds the \"status\" message which is initially used to signal that the\n//     subscription is valid (i.e. starting at the requested watermark).\n// v3: v0.20\n//   - Adds the \"taskID\" to the subscription context, and support for\n//     the backup monitor-mediated \"/snapshot\" request.\n// v4: v0.25\n//   - Adds the \"replicaVersion\" and \"minWatermark\" fields to the \"/snapshot\"\n//     status request so that a subscriber can verify whether its replica,\n//     whether it be restored or existing in a permanent volume, is compatible\n//     with the change-streamer.\n// v5: v0.26\n//   - Moves relation.keyColumns and relation.replicaIdentity to\n//     relation.rowKey: { columns, type }.\n//   - Adds `metadata` to `create-table` message\n//   - Adds `tableMetadata` to `add-column` message\n//   - Adds `table-update-metadata` message\n// v6: v0.26\n//   - Adds support for `backfill` messages\n// v6: v1.0.1  (backwards compatible, no version change)\n//   - Adds lag reporting to status messages\n// v6: (backwards compatible, no version change)\n//   - Adds the optional `commitTimeMs` field to `commit` messages, carrying\n//     the upstream commit timestamp for end-to-end serving lag measurement.\n//     The stream is parsed in 'passthrough' mode, so an older peer ignores the\n//     field, and a newer peer treats its absence (including in changes\n//     replayed from the Change DB) as \"no commit time reported\".\n// v7: v1.12\n//   - Merges the separate `/snapshot` and `/changes` connections into a single\n//     bidirectional `/subscribe` connection (see subscribe.ts). The view-syncer\n//     drives the connection with upstream `reserve-snapshot` / `start-subscription`\n//     application messages; the change-streamer replies with a `reserved`\n//     message and then the change stream on the same socket. This guarantees the\n//     replication-manager that reserves the change log is the one that serves the\n//     subscription. The legacy `/snapshot` + `/changes` endpoints remain for\n//     v4-v6 subscribers during rollout.\n//   - Drops the legacy \"initial\" and \"logsChangeStream\" query parameters from the\n//     subscriber context\n\nexport const PROTOCOL_VERSION = 7;\n\nexport type SubscriberContext = {\n  /**\n   * The supported change-streamer protocol version.\n   */\n  protocolVersion: number;\n\n  /**\n   * Task ID. This is used to link the request with a preceding snapshot\n   * reservation.\n   */\n  taskID: string;\n\n  /**\n   * Subscriber id. This is only used for debugging.\n   */\n  id: string;\n\n  /**\n   * The ReplicatorMode of the subscriber. 'backup' indicates that the\n   * subscriber is local to the `change-streamer` in the `replication-manager`,\n   * while 'serving' indicates that user-facing requests depend on the subscriber.\n   */\n  mode: ReplicatorMode;\n\n  /**\n   * The ChangeStreamer will return an Error if the subscriber is\n   * on a different replica version (i.e. the initial snapshot associated\n   * with the replication slot).\n   */\n  replicaVersion: string;\n\n  /**\n   * The watermark up to which the subscriber is up to date.\n   * Only changes after the watermark will be streamed.\n   */\n  watermark: string;\n\n  /**\n   * Whether the subscriber supports batched WebSocket frames.\n   */\n  wsBatched?: boolean | undefined;\n};\n\n/**\n * The StatusMessage payload for now is empty, but can be extended to\n * include meta-level information in the future.\n */\nexport const statusSchema = v.object({\n  tag: v.literal('status'),\n\n  lagReport: v\n    .object({\n      lastTimings: changeSourceTimingsSchema.optional(),\n      nextSendTimeMs: v.number(),\n    })\n    .optional(),\n});\n\nexport type Status = v.Infer<typeof statusSchema>;\n\nexport const statusMessageSchema = v.tuple([v.literal('status'), statusSchema]);\n\n/**\n * A StatusMessage will be immediately sent on a (v2+) subscription to\n * indicate that the subscription is valid (i.e. starting at the requested\n * watermark). Invalid subscriptions will instead result in a\n * SubscriptionError as the first message.\n */\nexport type StatusMessage = v.Infer<typeof statusMessageSchema>;\n\nconst subscriptionErrorSchema = v.object({\n  type: v.number(), // ErrorType\n  message: v.string().optional(),\n});\n\nexport type SubscriptionError = v.Infer<typeof subscriptionErrorSchema>;\n\nconst errorSchema = v.tuple([v.literal('error'), subscriptionErrorSchema]);\n\nexport const downstreamSchema = v.union(\n  statusMessageSchema,\n  changeStreamDataSchema,\n  errorSchema,\n);\n\nexport type Error = v.Infer<typeof errorSchema>;\n\nexport function errorTypeToReadableName(val: ErrorType) {\n  switch (val) {\n    case ErrorType.WrongReplicaVersion:\n      return 'WrongReplicaVersion';\n    case ErrorType.WatermarkTooOld:\n      return 'WatermarkTooOld';\n    case ErrorType.Unknown:\n      return 'Unknown';\n    default:\n      return 'Unknown';\n  }\n}\n\n/**\n * A stream of transactions, each starting with a {@link Begin} message,\n * containing one or more {@link Data} messages, and ending with a\n * {@link Commit} or {@link Rollback} message. The 'commit' tuple\n * includes a `watermark` that should be stored with the committed\n * data and used for resuming a subscription (e.g. in the\n * {@link SubscriberContext}).\n *\n * A {@link SubscriptionError} indicates an unrecoverable error that requires\n * manual intervention (e.g. configuration / operational error).\n */\nexport type Downstream = v.Infer<typeof downstreamSchema>;\n\n/** A downstream message and its approximate serialized transport size. */\nexport type SizedDownstream = Sized<Downstream>;\n\nexport interface ChangeStreamerService\n  extends Omit<ChangeStreamer, 'subscribe'>, Service {\n  /**\n   * The server-side interface overrides `subscribe()` to return a stream\n   * of already-stringified {@link Downstream} payloads or pre-serialized Buffers.\n   *\n   * If a `downstream` subscription is supplied it is used directly as the\n   * returned Source. Otherwise, a new one is created.\n   */\n  subscribe(\n    ctx: SubscriberContext,\n    downstream?: Subscription<string | PreSerializedBatch>,\n  ): Promise<Source<string | PreSerialized>>;\n\n  /**\n   * Starts a snapshot reservation to preserve change-log entries while\n   * a soon-to-be-subscriber downloads the current replica backup. Once\n   * the change-streamer knows that a backup is available\n   * (via {@link trackBackupWatermark}), it sends a confirmation\n   * SnapshotMessage and reserves the corresponding entries in the change-log\n   * until the client with that `taskID` subscribes, or if the snapshot\n   * connection terminates.\n   */\n  startSnapshotReservation(taskID: string): Promise<Source<SnapshotMessage>>;\n\n  /**\n   * Informs the change-streamer of the watermark up to which the replica has\n   * been backed up. This serves two purposes:\n   * - It serves as the maximum watermark up to which the change-log can be\n   *   purged. Note that the change-streamer also takes into account the\n   *   positions of its subscribers and snapshot reservations when purging\n   *   changes in the change-log.\n   * - In RMv2, this ACK's the upstream change-source to allow its buffer of\n   *   changes (e.g. the replication_slot) to advance.\n   */\n  trackBackupWatermark(watermark: string): void;\n}\n"],"mappings":";;;;;;;;AA0KA,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CAEvB,WAAW,eACR,OAAO;EACN,aAAa,0BAA0B,SAAS;EAChD,gBAAgB,eAAE,OAAO;CAC3B,CAAC,EACA,SAAS;AACd,CAAC;AAID,IAAa,sBAAsB,eAAE,MAAM,CAAC,eAAE,QAAQ,QAAQ,GAAG,YAAY,CAAC;AAU9E,IAAM,0BAA0B,eAAE,OAAO;CACvC,MAAM,eAAE,OAAO;CACf,SAAS,eAAE,OAAO,EAAE,SAAS;AAC/B,CAAC;AAID,IAAM,cAAc,eAAE,MAAM,CAAC,eAAE,QAAQ,OAAO,GAAG,uBAAuB,CAAC;AAEzE,IAAa,mBAAmB,eAAE,MAChC,qBACA,wBACA,WACF;AAIA,SAAgB,wBAAwB,KAAgB;CACtD,QAAQ,KAAR;EACE,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,SACE,OAAO;CACX;AACF"}

package/out/zero-cache/src/services/change-streamer/schema/tables.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"tables.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-streamer/schema/tables.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAErC,OAAO,EAAC,UAAU,EAAC,MAAM,0CAA0C,CAAC;AAGpE,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AACjE,OAAO,KAAK,EACV,UAAU,EACV,MAAM,EACN,aAAa,EACd,MAAM,8CAA8C,CAAC;AACtD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8CAA8C,CAAC;AAOpF,eAAO,MAAM,SAAS,QAAQ,CAAC;AAM/B,MAAM,MAAM,cAAc,GAAG;IAG3B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAmBF;;;GAGG;AACH,MAAM,MAAM,gBAAgB,GAAG;IAC7B,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B,CAAC;AAEF,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,OAAO,UASzD;AAED,wBAAsB,6BAA6B,CACjD,KAAK,EAAE,OAAO,EACd,GAAG,EAAE,UAAU,GACd,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAIxB;AAED;;;;;GAKG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;CACjC,CAAC;AAaF,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,OAAO,UAiBlD;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,aAAa,CAAC;CACzB,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,UAAU,CAAC;CACtB,CAAC;AAYF,UAAU,SAAS;IACjB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,CAAC,cAAc,EAC3B,KAAK,EAAE,OAAO,iBAIf;AAED,wBAAsB,iBAAiB,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,iBAKtE;AAED,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,iBAAiB,EAAE,IAAI,CACrB,iBAAiB,EACjB,cAAc,GAAG,gBAAgB,GAAG,WAAW,CAChD,EACD,KAAK,EAAE,OAAO,EACd,SAAS,EAAE,OAAO,EAClB,SAAS,CAAC,EAAE,SAAS,EACrB,YAAY,GAAE,OAAO,UAAuB,EAC5C,kBAAkB,UAAO,iBAgI1B;AAMD,eAAO,MAAM,wBAAwB,yBAAyB,CAAC;AAE/D,qBAAa,eAAgB,SAAQ,UAAU;IAC7C,QAAQ,CAAC,IAAI,qBAAqB;CACnC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,4BAA4B,CAChD,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,iBA4Cf"}
+{"version":3,"file":"tables.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-streamer/schema/tables.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAErC,OAAO,EAAC,UAAU,EAAC,MAAM,0CAA0C,CAAC;AAGpE,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AACjE,OAAO,KAAK,EACV,UAAU,EACV,MAAM,EACN,aAAa,EACd,MAAM,8CAA8C,CAAC;AACtD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8CAA8C,CAAC;AAOpF,eAAO,MAAM,SAAS,QAAQ,CAAC;AAM/B,MAAM,MAAM,cAAc,GAAG;IAG3B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAmBF;;;GAGG;AACH,MAAM,MAAM,gBAAgB,GAAG;IAC7B,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B,CAAC;AAEF,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,OAAO,UASzD;AAED,wBAAsB,6BAA6B,CACjD,KAAK,EAAE,OAAO,EACd,GAAG,EAAE,UAAU,GACd,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAIxB;AAED;;;;;GAKG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;CACjC,CAAC;AAaF,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,OAAO,UAiBlD;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,aAAa,CAAC;CACzB,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,UAAU,CAAC;CACtB,CAAC;AAYF,UAAU,SAAS;IACjB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,CAAC,cAAc,EAC3B,KAAK,EAAE,OAAO,iBAIf;AAED,wBAAsB,iBAAiB,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,iBAKtE;AAED,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,iBAAiB,EAAE,IAAI,CACrB,iBAAiB,EACjB,cAAc,GAAG,gBAAgB,GAAG,WAAW,CAChD,EACD,KAAK,EAAE,OAAO,EACd,SAAS,EAAE,OAAO,EAClB,SAAS,CAAC,EAAE,SAAS,EACrB,YAAY,GAAE,OAAO,UAAuB,EAC5C,kBAAkB,UAAO,iBA8I1B;AAMD,eAAO,MAAM,wBAAwB,yBAAyB,CAAC;AAE/D,qBAAa,eAAgB,SAAQ,UAAU;IAC7C,QAAQ,CAAC,IAAI,qBAAqB;CACnC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,4BAA4B,CAChD,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,iBA4Cf"}

package/out/zero-cache/src/services/change-streamer/schema/tables.js

@@ -119,13 +119,24 @@ async function ensureReplicationConfig(lc, db, subscriptionState, shard, autoRes
 			stmts.push(sql`INSERT INTO ${sql(schema)}."replicationConfig" ${sql(replicaConfig)}`, sql`INSERT INTO ${sql(schema)}."replicationState"  ${sql(replicationState)} 
               ON CONFLICT (lock) DO UPDATE SET ${sql(replicationState)}`, ...initialTx.map((change) => sql`INSERT INTO ${sql(schema)}."changeLog" ${sql(change)}`));
 			if (needsTruncate) {
-				const timer = setTimeoutFn(async () => {
-					lc.info?.("ensureReplicationConfig blocked, terminating lock holders");
-					await terminateChangeDBLockHolders(lc, db, shard);
-				}, LOCK_HOLDER_TERMINATE_TIMEOUT_MS);
+				let done = false;
+				let timer;
+				const scheduleTerminate = () => {
+					timer = setTimeoutFn(async () => {
+						lc.info?.("ensureReplicationConfig blocked, terminating lock holders");
+						try {
+							await terminateChangeDBLockHolders(lc, db, shard);
+						} catch (e) {
+							lc.warn?.("error terminating lock holders", e);
+						}
+						if (!done) scheduleTerminate();
+					}, LOCK_HOLDER_TERMINATE_TIMEOUT_MS);
+				};
+				scheduleTerminate();
 				try {
 					return await Promise.all(stmts);
 				} finally {
+					done = true;
 					clearTimeout(timer);
 				}
 			}

package/out/zero-cache/src/services/change-streamer/schema/tables.js.map

@@ -1 +1 @@
-{"version":3,"file":"tables.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-streamer/schema/tables.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {ident} from 'pg-format';\nimport type postgres from 'postgres';\nimport {type PendingQuery, type Row} from 'postgres';\nimport {AbortError} from '../../../../../shared/src/abort-error.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport {type PostgresDB} from '../../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../../types/shards.ts';\nimport type {\n  BackfillID,\n  Change,\n  TableMetadata,\n} from '../../change-source/protocol/current/data.ts';\nimport type {SubscriptionState} from '../../replicator/schema/replication-state.ts';\n\n// For readability in the sql statements.\nfunction schema(shard: ShardID) {\n  return ident(cdcSchema(shard));\n}\n\nexport const PG_SCHEMA = 'cdc';\n\nfunction createSchema(shard: ShardID) {\n  return /*sql*/ `CREATE SCHEMA IF NOT EXISTS ${schema(shard)};`;\n}\n\nexport type ChangeLogEntry = {\n  // A strictly monotonically increasing, lexicographically sortable\n  // value that uniquely identifies a position in the change stream.\n  watermark: string;\n  change: Change;\n};\n\ntype FullChangeLogEntry = ChangeLogEntry & {pos: number};\n\nfunction createChangeLogTable(shard: ShardID) {\n  // Note: The \"change\" column used to be JSONB, but that was problematic in that\n  // it does not handle the NULL unicode character.\n  // https://vladimir.varank.in/notes/2021/01/you-dont-insert-unicode-null-character-as-postgres-jsonb/\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"changeLog\" (\n    watermark  TEXT,\n    pos        INT8,\n    change     JSON NOT NULL,\n    precommit  TEXT,  -- Only exists on commit entries. Purely for debugging.\n    PRIMARY KEY (watermark, pos)\n  );\n`;\n}\n\n/**\n * Tracks the watermark from which to resume the change stream and the\n * current owner (task ID) acting as the single writer to the changeLog.\n */\nexport type ReplicationState = {\n  lastWatermark: string;\n  owner: string | null;\n  ownerAddress: string | null;\n};\n\nexport function createReplicationStateTable(shard: ShardID) {\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"replicationState\" (\n    \"lastWatermark\" TEXT NOT NULL,\n    \"owner\" TEXT,\n    \"ownerAddress\" TEXT,\n    \"lock\" INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n  );\n`;\n}\n\nexport async function discoverChangeStreamerAddress(\n  shard: ShardID,\n  sql: PostgresDB,\n): Promise<string | null> {\n  const result = await sql<{ownerAddress: string | null}[]> /*sql*/ `\n    SELECT \"ownerAddress\" FROM ${sql(cdcSchema(shard))}.\"replicationState\"`;\n  return result[0].ownerAddress;\n}\n\n/**\n * This mirrors the analogously named table in the SQLite replica\n * (`services/replicator/schema/replication-state.ts`), and is used\n * to detect when the replica has been reset and is no longer compatible\n * with the current ChangeLog.\n */\nexport type ReplicationConfig = {\n  replicaVersion: string;\n  publications: readonly string[];\n};\n\nfunction createReplicationConfigTable(shard: ShardID) {\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"replicationConfig\" (\n    \"replicaVersion\" TEXT NOT NULL,\n    \"publications\" TEXT[] NOT NULL,\n    \"resetRequired\" BOOL,\n    \"lock\" INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n  );\n`;\n}\n\nexport function createBackfillTables(shard: ShardID) {\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"tableMetadata\" (\n    \"schema\" TEXT NOT NULL,\n    \"table\" TEXT NOT NULL,\n    \"metadata\" JSONB NOT NULL,\n    PRIMARY KEY(\"schema\", \"table\")\n  );\n\n  CREATE TABLE ${schema(shard)}.\"backfilling\" (\n    \"schema\" TEXT NOT NULL,\n    \"table\" TEXT NOT NULL,\n    \"column\" TEXT NOT NULL,\n    \"backfill\" JSONB NOT NULL,\n    PRIMARY KEY(\"schema\", \"table\", \"column\")\n  );\n  `;\n}\n\nexport type TableMetadataRow = {\n  schema: string;\n  table: string;\n  metadata: TableMetadata;\n};\n\nexport type BackfillingColumn = {\n  schema: string;\n  table: string;\n  column: string;\n  backfill: BackfillID;\n};\n\nfunction createTables(shard: ShardID) {\n  return (\n    createSchema(shard) +\n    createChangeLogTable(shard) +\n    createReplicationStateTable(shard) +\n    createReplicationConfigTable(shard) +\n    createBackfillTables(shard)\n  );\n}\n\ninterface PurgeLock {\n  release(): Promise<void>;\n}\n\nexport async function setupCDCTables(\n  lc: LogContext,\n  db: postgres.TransactionSql,\n  shard: ShardID,\n) {\n  lc.info?.(`Setting up CDC tables`);\n  await db.unsafe(createTables(shard));\n}\n\nexport async function markResetRequired(sql: PostgresDB, shard: ShardID) {\n  const schema = cdcSchema(shard);\n  await sql`\n  UPDATE ${sql(schema)}.\"replicationConfig\"\n    SET \"resetRequired\" = true`;\n}\n\nexport async function ensureReplicationConfig(\n  lc: LogContext,\n  db: PostgresDB,\n  subscriptionState: Pick<\n    SubscriptionState,\n    'publications' | 'replicaVersion' | 'watermark'\n  >,\n  shard: ShardID,\n  autoReset: boolean,\n  purgeLock?: PurgeLock,\n  setTimeoutFn: typeof setTimeout = setTimeout,\n  pgChangeLogEnabled = true,\n) {\n  const {publications, replicaVersion, watermark} = subscriptionState;\n  const replicaConfig = {publications, replicaVersion};\n  const replicationState: ReplicationState = {\n    lastWatermark: replicaVersion,\n    owner: null,\n    ownerAddress: null,\n  };\n  const schema = cdcSchema(shard);\n\n  await runTx(db, async sql => {\n    const stmts: PendingQuery<Row[]>[] = [];\n    let needsTruncate = false;\n    const results = await sql<\n      {\n        replicaVersion: string;\n        publications: string[];\n        resetRequired: boolean | null;\n      }[]\n    > /*sql*/ `\n    SELECT \"replicaVersion\", \"publications\", \"resetRequired\" \n      FROM ${sql(schema)}.\"replicationConfig\"`;\n\n    if (results.length) {\n      const {replicaVersion, publications} = results[0];\n      if (\n        replicaVersion !== replicaConfig.replicaVersion ||\n        !equals(new Set(publications), new Set(replicaConfig.publications))\n      ) {\n        if (replicaConfig.replicaVersion !== watermark) {\n          throw new AutoResetSignal(\n            `Cannot reset change db@${replicaVersion} to ` +\n              `service replica@${replicaConfig.replicaVersion} ` +\n              `from watermark ${watermark}`,\n          );\n        }\n        lc.info?.(\n          `Data in cdc tables @${replicaVersion} is incompatible ` +\n            `with replica @${replicaConfig.replicaVersion}. Clearing tables.`,\n        );\n        // Release any purge lock held by the caller; the changeLog is\n        // incompatible with the replica and needs to be truncated.\n        void purgeLock?.release();\n\n        // Note: The order of TRUNCATE matters. The replicationState table is\n        //       truncated before the changeLog table, in order to acquire\n        //       (exclusive) locks on the tables in the same order that the\n        //       storer.ts acquires locks when writing the changeLog, namely:\n        //\n        // 1. SELECT ... FROM replicationState FOR UPDATE;\n        // 2. INSERT INTO changeLog ...;\n        //    ...\n        // n. UPDATE replicationState ...;\n        needsTruncate = true;\n        stmts.push(\n          sql`TRUNCATE TABLE ${sql(schema)}.\"replicationState\"`,\n          ...(pgChangeLogEnabled\n            ? [sql`TRUNCATE TABLE ${sql(schema)}.\"changeLog\"`]\n            : []),\n          sql`TRUNCATE TABLE ${sql(schema)}.\"replicationConfig\"`,\n          sql`TRUNCATE TABLE ${sql(schema)}.\"tableMetadata\"`,\n          sql`TRUNCATE TABLE ${sql(schema)}.\"backfilling\"`,\n        );\n      }\n    }\n    // Initialize (or re-initialize TRUNCATED) tables\n    if (results.length === 0 || needsTruncate) {\n      // When enabled, the PG storer uses the earliest changeLog entry as the\n      // safe watermark from which subscribers can be resumed. These initial\n      // entries ensure that subscribers can start from a freshly synced\n      // replica, even if new changes have been replicated and not purged.\n      //\n      // TODO: Replace this with an explicit `firstWatermark` column in the\n      //       change db.\n      const watermark = replicaConfig.replicaVersion;\n      const initialTx: FullChangeLogEntry[] = pgChangeLogEnabled\n        ? [\n            {watermark, pos: 0, change: {tag: 'begin'}},\n            {watermark, pos: 1, change: {tag: 'commit'}},\n          ]\n        : [];\n\n      stmts.push(\n        sql`INSERT INTO ${sql(schema)}.\"replicationConfig\" ${sql(replicaConfig)}`,\n        sql`INSERT INTO ${sql(schema)}.\"replicationState\"  ${sql(replicationState)} \n              ON CONFLICT (lock) DO UPDATE SET ${sql(replicationState)}`,\n        ...initialTx.map(\n          change => sql`INSERT INTO ${sql(schema)}.\"changeLog\" ${sql(change)}`,\n        ),\n      );\n\n      if (needsTruncate) {\n        // The TRUNCATE statements require ACCESS EXCLUSIVE locks, which may\n        // be blocked by old storer catchup reads. Race against a timeout\n        // that terminates the blocking backends if the TRUNCATE takes too\n        // long.\n        const timer = setTimeoutFn(async () => {\n          lc.info?.(\n            'ensureReplicationConfig blocked, terminating lock holders',\n          );\n          await terminateChangeDBLockHolders(lc, db, shard);\n        }, LOCK_HOLDER_TERMINATE_TIMEOUT_MS);\n        try {\n          return await Promise.all(stmts);\n        } finally {\n          clearTimeout(timer);\n        }\n      }\n      return Promise.all(stmts);\n    }\n\n    const {resetRequired} = results[0];\n    if (resetRequired) {\n      if (autoReset) {\n        throw new AutoResetSignal('reset required by replication stream');\n      }\n      lc.error?.(\n        '\\n\\n\\n' +\n          'Reset required but --auto-reset is not enabled.\\n' +\n          'This can happen for upstream databases that do not support event triggers.\\n' +\n          'To correct this, see https://zero.rocicorp.dev/docs/connecting-to-postgres#schema-changes' +\n          '\\n\\n\\n',\n      );\n    }\n\n    return [];\n  });\n}\n\n// The time to wait for a TRUNCATE in ensureReplicationConfig before\n// terminating blocking backends via terminateChangeDBLockHolders.\nconst LOCK_HOLDER_TERMINATE_TIMEOUT_MS = 5_000;\n\nexport const CHANGE_STREAMER_APP_NAME = 'zero-change-streamer';\n\nexport class AutoResetSignal extends AbortError {\n  readonly name = 'AutoResetSignal';\n}\n\n/**\n * Terminates zero-cache backends that are blocking the current backend\n * from acquiring locks on CDC tables (e.g., during TRUNCATE).\n *\n * This is used during change-DB takeover when the new replication-manager's\n * `ensureReplicationConfig` needs to TRUNCATE tables, but the old\n * replication-manager's storer is still reading from them (e.g., large\n * catchup cursors).\n *\n * The function:\n * 1. Finds backends waiting for a lock on a TRUNCATE in {schema}\n * 2. Uses `pg_blocking_pids()` to identify which backends are blocking them\n * 3. Terminates blocking backends that have `application_name = 'zero-change-streamer'`\n *\n * Must be called on a **separate connection** from the one that is blocked,\n * since the blocked connection is inside a pending transaction.\n */\nexport async function terminateChangeDBLockHolders(\n  lc: LogContext,\n  db: PostgresDB,\n  shard: ShardID,\n) {\n  const schema = cdcSchema(shard);\n\n  // Step 1: Find backends that are blocked waiting for a lock,\n  // whose query involves a TRUNCATE on this shard's CDC schema.\n  const blocked = await db<{pid: number}[]>`\n    SELECT pid FROM pg_stat_activity\n      WHERE wait_event_type = 'Lock'\n        AND application_name = ${CHANGE_STREAMER_APP_NAME}\n        AND query LIKE ${'%TRUNCATE%' + schema + '%'}`;\n\n  if (blocked.length === 0) {\n    lc.info?.('no blocked TRUNCATE backends found');\n    return;\n  }\n\n  const blockedPids = blocked.map(r => r.pid);\n  lc.info?.(`found blocked TRUNCATE backends: ${JSON.stringify(blockedPids)}`);\n\n  // Step 2: For each blocked backend, find and terminate its blockers\n  // that are zero-change-streamer connections.\n  const terminated = await db<\n    {pid: number; applicationName: string; query: string; terminated: boolean}[]\n  >`\n    SELECT pid, application_name as \"applicationName\", query,\n           pg_terminate_backend(pid) as terminated\n      FROM pg_stat_activity\n      WHERE pid = ANY(\n        SELECT unnest(pg_blocking_pids(blocked.pid))\n          FROM unnest(${blockedPids}::int[]) AS blocked(pid)\n      )\n      AND application_name = ${CHANGE_STREAMER_APP_NAME}\n      AND pid != ALL(${blockedPids}::int[])`;\n\n  if (terminated.length === 0) {\n    lc.info?.(`no ${CHANGE_STREAMER_APP_NAME} blockers found to terminate`);\n  } else {\n    for (const {pid, applicationName, query, terminated: ok} of terminated) {\n      lc.info?.(\n        `terminated blocking backend pid=${pid} app=${applicationName} ok=${ok} query=${query.slice(0, 200)}`,\n      );\n    }\n  }\n}\n"],"mappings":";;;;;;;;AAiBA,SAAS,OAAO,OAAgB;CAC9B,OAAO,MAAM,UAAU,KAAK,CAAC;AAC/B;AAIA,SAAS,aAAa,OAAgB;CACpC,OAAe,+BAA+B,OAAO,KAAK,EAAE;AAC9D;AAWA,SAAS,qBAAqB,OAAgB;CAI5C,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;;AAQ/B;AAYA,SAAgB,4BAA4B,OAAgB;CAC1D,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;AAO/B;AAEA,eAAsB,8BACpB,OACA,KACwB;CAGxB,QAAO,MAFc,GAA6C;iCACnC,IAAI,UAAU,KAAK,CAAC,EAAE,sBACvC,GAAG;AACnB;AAaA,SAAS,6BAA6B,OAAgB;CACpD,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;AAO/B;AAEA,SAAgB,qBAAqB,OAAgB;CACnD,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;iBAOd,OAAO,KAAK,EAAE;;;;;;;;AAQ/B;AAeA,SAAS,aAAa,OAAgB;CACpC,OACE,aAAa,KAAK,IAClB,qBAAqB,KAAK,IAC1B,4BAA4B,KAAK,IACjC,6BAA6B,KAAK,IAClC,qBAAqB,KAAK;AAE9B;AAMA,eAAsB,eACpB,IACA,IACA,OACA;CACA,GAAG,OAAO,uBAAuB;CACjC,MAAM,GAAG,OAAO,aAAa,KAAK,CAAC;AACrC;AAEA,eAAsB,kBAAkB,KAAiB,OAAgB;CAEvE,MAAM,GAAG;WACA,IAFM,UAAU,KAEZ,CAAM,EAAE;;AAEvB;AAEA,eAAsB,wBACpB,IACA,IACA,mBAIA,OACA,WACA,WACA,eAAkC,YAClC,qBAAqB,MACrB;CACA,MAAM,EAAC,cAAc,gBAAgB,cAAa;CAClD,MAAM,gBAAgB;EAAC;EAAc;CAAc;CACnD,MAAM,mBAAqC;EACzC,eAAe;EACf,OAAO;EACP,cAAc;CAChB;CACA,MAAM,SAAS,UAAU,KAAK;CAE9B,MAAM,MAAM,IAAI,OAAM,QAAO;EAC3B,MAAM,QAA+B,CAAC;EACtC,IAAI,gBAAgB;EACpB,MAAM,UAAU,MAAM,GAMZ;;aAED,IAAI,MAAM,EAAE;EAErB,IAAI,QAAQ,QAAQ;GAClB,MAAM,EAAC,gBAAgB,iBAAgB,QAAQ;GAC/C,IACE,mBAAmB,cAAc,kBACjC,CAAC,OAAO,IAAI,IAAI,YAAY,GAAG,IAAI,IAAI,cAAc,YAAY,CAAC,GAClE;IACA,IAAI,cAAc,mBAAmB,WACnC,MAAM,IAAI,gBACR,0BAA0B,eAAe,sBACpB,cAAc,eAAe,kBAC9B,WACtB;IAEF,GAAG,OACD,uBAAuB,eAAe,iCACnB,cAAc,eAAe,mBAClD;IAGA,WAAgB,QAAQ;IAWxB,gBAAgB;IAChB,MAAM,KACJ,GAAG,kBAAkB,IAAI,MAAM,EAAE,sBACjC,GAAI,qBACA,CAAC,GAAG,kBAAkB,IAAI,MAAM,EAAE,aAAa,IAC/C,CAAC,GACL,GAAG,kBAAkB,IAAI,MAAM,EAAE,uBACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,mBACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,eACnC;GACF;EACF;EAEA,IAAI,QAAQ,WAAW,KAAK,eAAe;GAQzC,MAAM,YAAY,cAAc;GAChC,MAAM,YAAkC,qBACpC,CACE;IAAC;IAAW,KAAK;IAAG,QAAQ,EAAC,KAAK,QAAO;GAAC,GAC1C;IAAC;IAAW,KAAK;IAAG,QAAQ,EAAC,KAAK,SAAQ;GAAC,CAC7C,IACA,CAAC;GAEL,MAAM,KACJ,GAAG,eAAe,IAAI,MAAM,EAAE,uBAAuB,IAAI,aAAa,KACtE,GAAG,eAAe,IAAI,MAAM,EAAE,uBAAuB,IAAI,gBAAgB,EAAE;iDAClC,IAAI,gBAAgB,KAC7D,GAAG,UAAU,KACX,WAAU,GAAG,eAAe,IAAI,MAAM,EAAE,eAAe,IAAI,MAAM,GACnE,CACF;GAEA,IAAI,eAAe;IAKjB,MAAM,QAAQ,aAAa,YAAY;KACrC,GAAG,OACD,2DACF;KACA,MAAM,6BAA6B,IAAI,IAAI,KAAK;IAClD,GAAG,gCAAgC;IACnC,IAAI;KACF,OAAO,MAAM,QAAQ,IAAI,KAAK;IAChC,UAAU;KACR,aAAa,KAAK;IACpB;GACF;GACA,OAAO,QAAQ,IAAI,KAAK;EAC1B;EAEA,MAAM,EAAC,kBAAiB,QAAQ;EAChC,IAAI,eAAe;GACjB,IAAI,WACF,MAAM,IAAI,gBAAgB,sCAAsC;GAElE,GAAG,QACD,oOAKF;EACF;EAEA,OAAO,CAAC;CACV,CAAC;AACH;AAIA,IAAM,mCAAmC;AAEzC,IAAa,2BAA2B;AAExC,IAAa,kBAAb,cAAqC,WAAW;CAC9C,OAAgB;AAClB;;;;;;;;;;;;;;;;;;AAmBA,eAAsB,6BACpB,IACA,IACA,OACA;CAKA,MAAM,UAAU,MAAM,EAAmB;;;iCAGV,yBAAyB;yBACjC,eARR,UAAU,KAQa,IAAS;CAE/C,IAAI,QAAQ,WAAW,GAAG;EACxB,GAAG,OAAO,oCAAoC;EAC9C;CACF;CAEA,MAAM,cAAc,QAAQ,KAAI,MAAK,EAAE,GAAG;CAC1C,GAAG,OAAO,oCAAoC,KAAK,UAAU,WAAW,GAAG;CAI3E,MAAM,aAAa,MAAM,EAExB;;;;;;wBAMqB,YAAY;;+BAEL,yBAAyB;uBACjC,YAAY;CAEjC,IAAI,WAAW,WAAW,GACxB,GAAG,OAAO,MAAM,yBAAyB,6BAA6B;MAEtE,KAAK,MAAM,EAAC,KAAK,iBAAiB,OAAO,YAAY,QAAO,YAC1D,GAAG,OACD,mCAAmC,IAAI,OAAO,gBAAgB,MAAM,GAAG,SAAS,MAAM,MAAM,GAAG,GAAG,GACpG;AAGN"}
+{"version":3,"file":"tables.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-streamer/schema/tables.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {ident} from 'pg-format';\nimport type postgres from 'postgres';\nimport {type PendingQuery, type Row} from 'postgres';\nimport {AbortError} from '../../../../../shared/src/abort-error.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport {type PostgresDB} from '../../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../../types/shards.ts';\nimport type {\n  BackfillID,\n  Change,\n  TableMetadata,\n} from '../../change-source/protocol/current/data.ts';\nimport type {SubscriptionState} from '../../replicator/schema/replication-state.ts';\n\n// For readability in the sql statements.\nfunction schema(shard: ShardID) {\n  return ident(cdcSchema(shard));\n}\n\nexport const PG_SCHEMA = 'cdc';\n\nfunction createSchema(shard: ShardID) {\n  return /*sql*/ `CREATE SCHEMA IF NOT EXISTS ${schema(shard)};`;\n}\n\nexport type ChangeLogEntry = {\n  // A strictly monotonically increasing, lexicographically sortable\n  // value that uniquely identifies a position in the change stream.\n  watermark: string;\n  change: Change;\n};\n\ntype FullChangeLogEntry = ChangeLogEntry & {pos: number};\n\nfunction createChangeLogTable(shard: ShardID) {\n  // Note: The \"change\" column used to be JSONB, but that was problematic in that\n  // it does not handle the NULL unicode character.\n  // https://vladimir.varank.in/notes/2021/01/you-dont-insert-unicode-null-character-as-postgres-jsonb/\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"changeLog\" (\n    watermark  TEXT,\n    pos        INT8,\n    change     JSON NOT NULL,\n    precommit  TEXT,  -- Only exists on commit entries. Purely for debugging.\n    PRIMARY KEY (watermark, pos)\n  );\n`;\n}\n\n/**\n * Tracks the watermark from which to resume the change stream and the\n * current owner (task ID) acting as the single writer to the changeLog.\n */\nexport type ReplicationState = {\n  lastWatermark: string;\n  owner: string | null;\n  ownerAddress: string | null;\n};\n\nexport function createReplicationStateTable(shard: ShardID) {\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"replicationState\" (\n    \"lastWatermark\" TEXT NOT NULL,\n    \"owner\" TEXT,\n    \"ownerAddress\" TEXT,\n    \"lock\" INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n  );\n`;\n}\n\nexport async function discoverChangeStreamerAddress(\n  shard: ShardID,\n  sql: PostgresDB,\n): Promise<string | null> {\n  const result = await sql<{ownerAddress: string | null}[]> /*sql*/ `\n    SELECT \"ownerAddress\" FROM ${sql(cdcSchema(shard))}.\"replicationState\"`;\n  return result[0].ownerAddress;\n}\n\n/**\n * This mirrors the analogously named table in the SQLite replica\n * (`services/replicator/schema/replication-state.ts`), and is used\n * to detect when the replica has been reset and is no longer compatible\n * with the current ChangeLog.\n */\nexport type ReplicationConfig = {\n  replicaVersion: string;\n  publications: readonly string[];\n};\n\nfunction createReplicationConfigTable(shard: ShardID) {\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"replicationConfig\" (\n    \"replicaVersion\" TEXT NOT NULL,\n    \"publications\" TEXT[] NOT NULL,\n    \"resetRequired\" BOOL,\n    \"lock\" INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n  );\n`;\n}\n\nexport function createBackfillTables(shard: ShardID) {\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"tableMetadata\" (\n    \"schema\" TEXT NOT NULL,\n    \"table\" TEXT NOT NULL,\n    \"metadata\" JSONB NOT NULL,\n    PRIMARY KEY(\"schema\", \"table\")\n  );\n\n  CREATE TABLE ${schema(shard)}.\"backfilling\" (\n    \"schema\" TEXT NOT NULL,\n    \"table\" TEXT NOT NULL,\n    \"column\" TEXT NOT NULL,\n    \"backfill\" JSONB NOT NULL,\n    PRIMARY KEY(\"schema\", \"table\", \"column\")\n  );\n  `;\n}\n\nexport type TableMetadataRow = {\n  schema: string;\n  table: string;\n  metadata: TableMetadata;\n};\n\nexport type BackfillingColumn = {\n  schema: string;\n  table: string;\n  column: string;\n  backfill: BackfillID;\n};\n\nfunction createTables(shard: ShardID) {\n  return (\n    createSchema(shard) +\n    createChangeLogTable(shard) +\n    createReplicationStateTable(shard) +\n    createReplicationConfigTable(shard) +\n    createBackfillTables(shard)\n  );\n}\n\ninterface PurgeLock {\n  release(): Promise<void>;\n}\n\nexport async function setupCDCTables(\n  lc: LogContext,\n  db: postgres.TransactionSql,\n  shard: ShardID,\n) {\n  lc.info?.(`Setting up CDC tables`);\n  await db.unsafe(createTables(shard));\n}\n\nexport async function markResetRequired(sql: PostgresDB, shard: ShardID) {\n  const schema = cdcSchema(shard);\n  await sql`\n  UPDATE ${sql(schema)}.\"replicationConfig\"\n    SET \"resetRequired\" = true`;\n}\n\nexport async function ensureReplicationConfig(\n  lc: LogContext,\n  db: PostgresDB,\n  subscriptionState: Pick<\n    SubscriptionState,\n    'publications' | 'replicaVersion' | 'watermark'\n  >,\n  shard: ShardID,\n  autoReset: boolean,\n  purgeLock?: PurgeLock,\n  setTimeoutFn: typeof setTimeout = setTimeout,\n  pgChangeLogEnabled = true,\n) {\n  const {publications, replicaVersion, watermark} = subscriptionState;\n  const replicaConfig = {publications, replicaVersion};\n  const replicationState: ReplicationState = {\n    lastWatermark: replicaVersion,\n    owner: null,\n    ownerAddress: null,\n  };\n  const schema = cdcSchema(shard);\n\n  await runTx(db, async sql => {\n    const stmts: PendingQuery<Row[]>[] = [];\n    let needsTruncate = false;\n    const results = await sql<\n      {\n        replicaVersion: string;\n        publications: string[];\n        resetRequired: boolean | null;\n      }[]\n    > /*sql*/ `\n    SELECT \"replicaVersion\", \"publications\", \"resetRequired\" \n      FROM ${sql(schema)}.\"replicationConfig\"`;\n\n    if (results.length) {\n      const {replicaVersion, publications} = results[0];\n      if (\n        replicaVersion !== replicaConfig.replicaVersion ||\n        !equals(new Set(publications), new Set(replicaConfig.publications))\n      ) {\n        if (replicaConfig.replicaVersion !== watermark) {\n          throw new AutoResetSignal(\n            `Cannot reset change db@${replicaVersion} to ` +\n              `service replica@${replicaConfig.replicaVersion} ` +\n              `from watermark ${watermark}`,\n          );\n        }\n        lc.info?.(\n          `Data in cdc tables @${replicaVersion} is incompatible ` +\n            `with replica @${replicaConfig.replicaVersion}. Clearing tables.`,\n        );\n        // Release any purge lock held by the caller; the changeLog is\n        // incompatible with the replica and needs to be truncated.\n        void purgeLock?.release();\n\n        // Note: The order of TRUNCATE matters. The replicationState table is\n        //       truncated before the changeLog table, in order to acquire\n        //       (exclusive) locks on the tables in the same order that the\n        //       storer.ts acquires locks when writing the changeLog, namely:\n        //\n        // 1. SELECT ... FROM replicationState FOR UPDATE;\n        // 2. INSERT INTO changeLog ...;\n        //    ...\n        // n. UPDATE replicationState ...;\n        needsTruncate = true;\n        stmts.push(\n          sql`TRUNCATE TABLE ${sql(schema)}.\"replicationState\"`,\n          ...(pgChangeLogEnabled\n            ? [sql`TRUNCATE TABLE ${sql(schema)}.\"changeLog\"`]\n            : []),\n          sql`TRUNCATE TABLE ${sql(schema)}.\"replicationConfig\"`,\n          sql`TRUNCATE TABLE ${sql(schema)}.\"tableMetadata\"`,\n          sql`TRUNCATE TABLE ${sql(schema)}.\"backfilling\"`,\n        );\n      }\n    }\n    // Initialize (or re-initialize TRUNCATED) tables\n    if (results.length === 0 || needsTruncate) {\n      // When enabled, the PG storer uses the earliest changeLog entry as the\n      // safe watermark from which subscribers can be resumed. These initial\n      // entries ensure that subscribers can start from a freshly synced\n      // replica, even if new changes have been replicated and not purged.\n      //\n      // TODO: Replace this with an explicit `firstWatermark` column in the\n      //       change db.\n      const watermark = replicaConfig.replicaVersion;\n      const initialTx: FullChangeLogEntry[] = pgChangeLogEnabled\n        ? [\n            {watermark, pos: 0, change: {tag: 'begin'}},\n            {watermark, pos: 1, change: {tag: 'commit'}},\n          ]\n        : [];\n\n      stmts.push(\n        sql`INSERT INTO ${sql(schema)}.\"replicationConfig\" ${sql(replicaConfig)}`,\n        sql`INSERT INTO ${sql(schema)}.\"replicationState\"  ${sql(replicationState)} \n              ON CONFLICT (lock) DO UPDATE SET ${sql(replicationState)}`,\n        ...initialTx.map(\n          change => sql`INSERT INTO ${sql(schema)}.\"changeLog\" ${sql(change)}`,\n        ),\n      );\n\n      if (needsTruncate) {\n        // The TRUNCATE statements require ACCESS EXCLUSIVE locks, which may\n        // be blocked by old storer catchup reads. Race against a timeout\n        // that terminates the blocking backends if the TRUNCATE takes too\n        // long. The check is repeated until the statements complete, since\n        // the TRUNCATE may not yet be waiting on a lock when the timer fires.\n        let done = false;\n        let timer: ReturnType<typeof setTimeout> | undefined;\n        const scheduleTerminate = () => {\n          timer = setTimeoutFn(async () => {\n            lc.info?.(\n              'ensureReplicationConfig blocked, terminating lock holders',\n            );\n            try {\n              await terminateChangeDBLockHolders(lc, db, shard);\n            } catch (e) {\n              lc.warn?.('error terminating lock holders', e);\n            }\n            if (!done) {\n              scheduleTerminate();\n            }\n          }, LOCK_HOLDER_TERMINATE_TIMEOUT_MS);\n        };\n        scheduleTerminate();\n        try {\n          return await Promise.all(stmts);\n        } finally {\n          done = true;\n          clearTimeout(timer);\n        }\n      }\n      return Promise.all(stmts);\n    }\n\n    const {resetRequired} = results[0];\n    if (resetRequired) {\n      if (autoReset) {\n        throw new AutoResetSignal('reset required by replication stream');\n      }\n      lc.error?.(\n        '\\n\\n\\n' +\n          'Reset required but --auto-reset is not enabled.\\n' +\n          'This can happen for upstream databases that do not support event triggers.\\n' +\n          'To correct this, see https://zero.rocicorp.dev/docs/connecting-to-postgres#schema-changes' +\n          '\\n\\n\\n',\n      );\n    }\n\n    return [];\n  });\n}\n\n// The time to wait for a TRUNCATE in ensureReplicationConfig before\n// terminating blocking backends via terminateChangeDBLockHolders.\nconst LOCK_HOLDER_TERMINATE_TIMEOUT_MS = 5_000;\n\nexport const CHANGE_STREAMER_APP_NAME = 'zero-change-streamer';\n\nexport class AutoResetSignal extends AbortError {\n  readonly name = 'AutoResetSignal';\n}\n\n/**\n * Terminates zero-cache backends that are blocking the current backend\n * from acquiring locks on CDC tables (e.g., during TRUNCATE).\n *\n * This is used during change-DB takeover when the new replication-manager's\n * `ensureReplicationConfig` needs to TRUNCATE tables, but the old\n * replication-manager's storer is still reading from them (e.g., large\n * catchup cursors).\n *\n * The function:\n * 1. Finds backends waiting for a lock on a TRUNCATE in {schema}\n * 2. Uses `pg_blocking_pids()` to identify which backends are blocking them\n * 3. Terminates blocking backends that have `application_name = 'zero-change-streamer'`\n *\n * Must be called on a **separate connection** from the one that is blocked,\n * since the blocked connection is inside a pending transaction.\n */\nexport async function terminateChangeDBLockHolders(\n  lc: LogContext,\n  db: PostgresDB,\n  shard: ShardID,\n) {\n  const schema = cdcSchema(shard);\n\n  // Step 1: Find backends that are blocked waiting for a lock,\n  // whose query involves a TRUNCATE on this shard's CDC schema.\n  const blocked = await db<{pid: number}[]>`\n    SELECT pid FROM pg_stat_activity\n      WHERE wait_event_type = 'Lock'\n        AND application_name = ${CHANGE_STREAMER_APP_NAME}\n        AND query LIKE ${'%TRUNCATE%' + schema + '%'}`;\n\n  if (blocked.length === 0) {\n    lc.info?.('no blocked TRUNCATE backends found');\n    return;\n  }\n\n  const blockedPids = blocked.map(r => r.pid);\n  lc.info?.(`found blocked TRUNCATE backends: ${JSON.stringify(blockedPids)}`);\n\n  // Step 2: For each blocked backend, find and terminate its blockers\n  // that are zero-change-streamer connections.\n  const terminated = await db<\n    {pid: number; applicationName: string; query: string; terminated: boolean}[]\n  >`\n    SELECT pid, application_name as \"applicationName\", query,\n           pg_terminate_backend(pid) as terminated\n      FROM pg_stat_activity\n      WHERE pid = ANY(\n        SELECT unnest(pg_blocking_pids(blocked.pid))\n          FROM unnest(${blockedPids}::int[]) AS blocked(pid)\n      )\n      AND application_name = ${CHANGE_STREAMER_APP_NAME}\n      AND pid != ALL(${blockedPids}::int[])`;\n\n  if (terminated.length === 0) {\n    lc.info?.(`no ${CHANGE_STREAMER_APP_NAME} blockers found to terminate`);\n  } else {\n    for (const {pid, applicationName, query, terminated: ok} of terminated) {\n      lc.info?.(\n        `terminated blocking backend pid=${pid} app=${applicationName} ok=${ok} query=${query.slice(0, 200)}`,\n      );\n    }\n  }\n}\n"],"mappings":";;;;;;;;AAiBA,SAAS,OAAO,OAAgB;CAC9B,OAAO,MAAM,UAAU,KAAK,CAAC;AAC/B;AAIA,SAAS,aAAa,OAAgB;CACpC,OAAe,+BAA+B,OAAO,KAAK,EAAE;AAC9D;AAWA,SAAS,qBAAqB,OAAgB;CAI5C,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;;AAQ/B;AAYA,SAAgB,4BAA4B,OAAgB;CAC1D,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;AAO/B;AAEA,eAAsB,8BACpB,OACA,KACwB;CAGxB,QAAO,MAFc,GAA6C;iCACnC,IAAI,UAAU,KAAK,CAAC,EAAE,sBACvC,GAAG;AACnB;AAaA,SAAS,6BAA6B,OAAgB;CACpD,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;AAO/B;AAEA,SAAgB,qBAAqB,OAAgB;CACnD,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;iBAOd,OAAO,KAAK,EAAE;;;;;;;;AAQ/B;AAeA,SAAS,aAAa,OAAgB;CACpC,OACE,aAAa,KAAK,IAClB,qBAAqB,KAAK,IAC1B,4BAA4B,KAAK,IACjC,6BAA6B,KAAK,IAClC,qBAAqB,KAAK;AAE9B;AAMA,eAAsB,eACpB,IACA,IACA,OACA;CACA,GAAG,OAAO,uBAAuB;CACjC,MAAM,GAAG,OAAO,aAAa,KAAK,CAAC;AACrC;AAEA,eAAsB,kBAAkB,KAAiB,OAAgB;CAEvE,MAAM,GAAG;WACA,IAFM,UAAU,KAEZ,CAAM,EAAE;;AAEvB;AAEA,eAAsB,wBACpB,IACA,IACA,mBAIA,OACA,WACA,WACA,eAAkC,YAClC,qBAAqB,MACrB;CACA,MAAM,EAAC,cAAc,gBAAgB,cAAa;CAClD,MAAM,gBAAgB;EAAC;EAAc;CAAc;CACnD,MAAM,mBAAqC;EACzC,eAAe;EACf,OAAO;EACP,cAAc;CAChB;CACA,MAAM,SAAS,UAAU,KAAK;CAE9B,MAAM,MAAM,IAAI,OAAM,QAAO;EAC3B,MAAM,QAA+B,CAAC;EACtC,IAAI,gBAAgB;EACpB,MAAM,UAAU,MAAM,GAMZ;;aAED,IAAI,MAAM,EAAE;EAErB,IAAI,QAAQ,QAAQ;GAClB,MAAM,EAAC,gBAAgB,iBAAgB,QAAQ;GAC/C,IACE,mBAAmB,cAAc,kBACjC,CAAC,OAAO,IAAI,IAAI,YAAY,GAAG,IAAI,IAAI,cAAc,YAAY,CAAC,GAClE;IACA,IAAI,cAAc,mBAAmB,WACnC,MAAM,IAAI,gBACR,0BAA0B,eAAe,sBACpB,cAAc,eAAe,kBAC9B,WACtB;IAEF,GAAG,OACD,uBAAuB,eAAe,iCACnB,cAAc,eAAe,mBAClD;IAGA,WAAgB,QAAQ;IAWxB,gBAAgB;IAChB,MAAM,KACJ,GAAG,kBAAkB,IAAI,MAAM,EAAE,sBACjC,GAAI,qBACA,CAAC,GAAG,kBAAkB,IAAI,MAAM,EAAE,aAAa,IAC/C,CAAC,GACL,GAAG,kBAAkB,IAAI,MAAM,EAAE,uBACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,mBACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,eACnC;GACF;EACF;EAEA,IAAI,QAAQ,WAAW,KAAK,eAAe;GAQzC,MAAM,YAAY,cAAc;GAChC,MAAM,YAAkC,qBACpC,CACE;IAAC;IAAW,KAAK;IAAG,QAAQ,EAAC,KAAK,QAAO;GAAC,GAC1C;IAAC;IAAW,KAAK;IAAG,QAAQ,EAAC,KAAK,SAAQ;GAAC,CAC7C,IACA,CAAC;GAEL,MAAM,KACJ,GAAG,eAAe,IAAI,MAAM,EAAE,uBAAuB,IAAI,aAAa,KACtE,GAAG,eAAe,IAAI,MAAM,EAAE,uBAAuB,IAAI,gBAAgB,EAAE;iDAClC,IAAI,gBAAgB,KAC7D,GAAG,UAAU,KACX,WAAU,GAAG,eAAe,IAAI,MAAM,EAAE,eAAe,IAAI,MAAM,GACnE,CACF;GAEA,IAAI,eAAe;IAMjB,IAAI,OAAO;IACX,IAAI;IACJ,MAAM,0BAA0B;KAC9B,QAAQ,aAAa,YAAY;MAC/B,GAAG,OACD,2DACF;MACA,IAAI;OACF,MAAM,6BAA6B,IAAI,IAAI,KAAK;MAClD,SAAS,GAAG;OACV,GAAG,OAAO,kCAAkC,CAAC;MAC/C;MACA,IAAI,CAAC,MACH,kBAAkB;KAEtB,GAAG,gCAAgC;IACrC;IACA,kBAAkB;IAClB,IAAI;KACF,OAAO,MAAM,QAAQ,IAAI,KAAK;IAChC,UAAU;KACR,OAAO;KACP,aAAa,KAAK;IACpB;GACF;GACA,OAAO,QAAQ,IAAI,KAAK;EAC1B;EAEA,MAAM,EAAC,kBAAiB,QAAQ;EAChC,IAAI,eAAe;GACjB,IAAI,WACF,MAAM,IAAI,gBAAgB,sCAAsC;GAElE,GAAG,QACD,oOAKF;EACF;EAEA,OAAO,CAAC;CACV,CAAC;AACH;AAIA,IAAM,mCAAmC;AAEzC,IAAa,2BAA2B;AAExC,IAAa,kBAAb,cAAqC,WAAW;CAC9C,OAAgB;AAClB;;;;;;;;;;;;;;;;;;AAmBA,eAAsB,6BACpB,IACA,IACA,OACA;CAKA,MAAM,UAAU,MAAM,EAAmB;;;iCAGV,yBAAyB;yBACjC,eARR,UAAU,KAQa,IAAS;CAE/C,IAAI,QAAQ,WAAW,GAAG;EACxB,GAAG,OAAO,oCAAoC;EAC9C;CACF;CAEA,MAAM,cAAc,QAAQ,KAAI,MAAK,EAAE,GAAG;CAC1C,GAAG,OAAO,oCAAoC,KAAK,UAAU,WAAW,GAAG;CAI3E,MAAM,aAAa,MAAM,EAExB;;;;;;wBAMqB,YAAY;;+BAEL,yBAAyB;uBACjC,YAAY;CAEjC,IAAI,WAAW,WAAW,GACxB,GAAG,OAAO,MAAM,yBAAyB,6BAA6B;MAEtE,KAAK,MAAM,EAAC,KAAK,iBAAiB,OAAO,YAAY,QAAO,YAC1D,GAAG,OACD,mCAAmC,IAAI,OAAO,gBAAgB,MAAM,GAAG,SAAS,MAAM,MAAM,GAAG,GAAG,GACpG;AAGN"}

package/out/zero-cache/src/services/change-streamer/snapshot-message.d.ts

@@ -0,0 +1,92 @@
+/**
+ * Pure schema definitions for the `/snapshot` reservation status message.
+ *
+ * These live in their own module (depending only on valita) so that both
+ * {@link snapshot.ts} (which also pulls in the change-streamer HTTP client) and
+ * {@link subscribe.ts} can import them without forming an import cycle.
+ */
+import * as v from '../../../../shared/src/valita.ts';
+export declare const statusSchema: v.ObjectType<{
+    tag: v.Type<"status">;
+    /**
+     * The location from which litestream should perform the restore.
+     */
+    backupURL: v.Type<string>;
+    /**
+     * The `replicaVersion` of the backup. If a subscriber's restored or
+     * existing replica is of a different version, it should delete it and
+     * retry the restore from litestream (i.e. equivalent to a
+     * `WrongReplicaVersion` response from a `/changes` subscription).
+     */
+    replicaVersion: v.Type<string>;
+    /**
+     * The earliest watermark from which catchup is possible. If the
+     * subscriber's replica is older that this watermark, it should delete it
+     * and (retry the) restore from litestream (i.e. equivalent to a
+     * `WatermarkTooOld` response from a `/changes` subscription).
+     */
+    minWatermark: v.Type<string>;
+    /**
+     * The size in bytes of the replication-manager's replica when the snapshot
+     * was reserved, used as an estimate of the size of the restored replica
+     * when reporting restore progress. Absent from older replication-managers.
+     */
+    replicaSize: v.Optional<number>;
+}, undefined>;
+export type SnapshotStatus = v.Infer<typeof statusSchema>;
+export declare const statusMessageSchema: v.TupleType<[v.Type<"status">, v.ObjectType<{
+    tag: v.Type<"status">;
+    /**
+     * The location from which litestream should perform the restore.
+     */
+    backupURL: v.Type<string>;
+    /**
+     * The `replicaVersion` of the backup. If a subscriber's restored or
+     * existing replica is of a different version, it should delete it and
+     * retry the restore from litestream (i.e. equivalent to a
+     * `WrongReplicaVersion` response from a `/changes` subscription).
+     */
+    replicaVersion: v.Type<string>;
+    /**
+     * The earliest watermark from which catchup is possible. If the
+     * subscriber's replica is older that this watermark, it should delete it
+     * and (retry the) restore from litestream (i.e. equivalent to a
+     * `WatermarkTooOld` response from a `/changes` subscription).
+     */
+    minWatermark: v.Type<string>;
+    /**
+     * The size in bytes of the replication-manager's replica when the snapshot
+     * was reserved, used as an estimate of the size of the restored replica
+     * when reporting restore progress. Absent from older replication-managers.
+     */
+    replicaSize: v.Optional<number>;
+}, undefined>]>;
+export declare const snapshotMessageSchema: v.UnionType<[v.TupleType<[v.Type<"status">, v.ObjectType<{
+    tag: v.Type<"status">;
+    /**
+     * The location from which litestream should perform the restore.
+     */
+    backupURL: v.Type<string>;
+    /**
+     * The `replicaVersion` of the backup. If a subscriber's restored or
+     * existing replica is of a different version, it should delete it and
+     * retry the restore from litestream (i.e. equivalent to a
+     * `WrongReplicaVersion` response from a `/changes` subscription).
+     */
+    replicaVersion: v.Type<string>;
+    /**
+     * The earliest watermark from which catchup is possible. If the
+     * subscriber's replica is older that this watermark, it should delete it
+     * and (retry the) restore from litestream (i.e. equivalent to a
+     * `WatermarkTooOld` response from a `/changes` subscription).
+     */
+    minWatermark: v.Type<string>;
+    /**
+     * The size in bytes of the replication-manager's replica when the snapshot
+     * was reserved, used as an estimate of the size of the restored replica
+     * when reporting restore progress. Absent from older replication-managers.
+     */
+    replicaSize: v.Optional<number>;
+}, undefined>]>]>;
+export type SnapshotMessage = v.Infer<typeof statusMessageSchema>;
+//# sourceMappingURL=snapshot-message.d.ts.map

package/out/zero-cache/src/services/change-streamer/snapshot-message.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"snapshot-message.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot-message.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,CAAC,MAAM,kCAAkC,CAAC;AAEtD,eAAO,MAAM,YAAY;;IAGvB;;OAEG;;IAGH;;;;;OAKG;;IAGH;;;;;OAKG;;IAGH;;;;OAIG;;aAEH,CAAC;AAEH,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AAE1D,eAAO,MAAM,mBAAmB;;IA/B9B;;OAEG;;IAGH;;;;;OAKG;;IAGH;;;;;OAKG;;IAGH;;;;OAIG;;eAM0E,CAAC;AAEhF,eAAO,MAAM,qBAAqB;;IAjChC;;OAEG;;IAGH;;;;;OAKG;;IAGH;;;;;OAKG;;IAGH;;;;OAIG;;iBAQ4D,CAAC;AAElE,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC"}

package/out/zero-cache/src/services/change-streamer/snapshot-message.js

@@ -0,0 +1,42 @@
+import { valita_exports } from "../../../../shared/src/valita.js";
+//#region ../zero-cache/src/services/change-streamer/snapshot-message.ts
+/**
+* Pure schema definitions for the `/snapshot` reservation status message.
+*
+* These live in their own module (depending only on valita) so that both
+* {@link snapshot.ts} (which also pulls in the change-streamer HTTP client) and
+* {@link subscribe.ts} can import them without forming an import cycle.
+*/
+var statusSchema = valita_exports.object({
+	tag: valita_exports.literal("status"),
+	/**
+	* The location from which litestream should perform the restore.
+	*/
+	backupURL: valita_exports.string(),
+	/**
+	* The `replicaVersion` of the backup. If a subscriber's restored or
+	* existing replica is of a different version, it should delete it and
+	* retry the restore from litestream (i.e. equivalent to a
+	* `WrongReplicaVersion` response from a `/changes` subscription).
+	*/
+	replicaVersion: valita_exports.string(),
+	/**
+	* The earliest watermark from which catchup is possible. If the
+	* subscriber's replica is older that this watermark, it should delete it
+	* and (retry the) restore from litestream (i.e. equivalent to a
+	* `WatermarkTooOld` response from a `/changes` subscription).
+	*/
+	minWatermark: valita_exports.string(),
+	/**
+	* The size in bytes of the replication-manager's replica when the snapshot
+	* was reserved, used as an estimate of the size of the restored replica
+	* when reporting restore progress. Absent from older replication-managers.
+	*/
+	replicaSize: valita_exports.number().optional()
+});
+var statusMessageSchema = valita_exports.tuple([valita_exports.literal("status"), statusSchema]);
+valita_exports.union(statusMessageSchema);
+//#endregion
+export { statusSchema };
+
+//# sourceMappingURL=snapshot-message.js.map

package/out/zero-cache/src/services/change-streamer/snapshot-message.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"snapshot-message.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot-message.ts"],"sourcesContent":["/**\n * Pure schema definitions for the `/snapshot` reservation status message.\n *\n * These live in their own module (depending only on valita) so that both\n * {@link snapshot.ts} (which also pulls in the change-streamer HTTP client) and\n * {@link subscribe.ts} can import them without forming an import cycle.\n */\n\nimport * as v from '../../../../shared/src/valita.ts';\n\nexport const statusSchema = v.object({\n  tag: v.literal('status'),\n\n  /**\n   * The location from which litestream should perform the restore.\n   */\n  backupURL: v.string(),\n\n  /**\n   * The `replicaVersion` of the backup. If a subscriber's restored or\n   * existing replica is of a different version, it should delete it and\n   * retry the restore from litestream (i.e. equivalent to a\n   * `WrongReplicaVersion` response from a `/changes` subscription).\n   */\n  replicaVersion: v.string(),\n\n  /**\n   * The earliest watermark from which catchup is possible. If the\n   * subscriber's replica is older that this watermark, it should delete it\n   * and (retry the) restore from litestream (i.e. equivalent to a\n   * `WatermarkTooOld` response from a `/changes` subscription).\n   */\n  minWatermark: v.string(),\n\n  /**\n   * The size in bytes of the replication-manager's replica when the snapshot\n   * was reserved, used as an estimate of the size of the restored replica\n   * when reporting restore progress. Absent from older replication-managers.\n   */\n  replicaSize: v.number().optional(),\n});\n\nexport type SnapshotStatus = v.Infer<typeof statusSchema>;\n\nexport const statusMessageSchema = v.tuple([v.literal('status'), statusSchema]);\n\nexport const snapshotMessageSchema = v.union(statusMessageSchema);\n\nexport type SnapshotMessage = v.Infer<typeof statusMessageSchema>;\n"],"mappings":";;;;;;;;;AAUA,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;;;;CAKvB,WAAW,eAAE,OAAO;;;;;;;CAQpB,gBAAgB,eAAE,OAAO;;;;;;;CAQzB,cAAc,eAAE,OAAO;;;;;;CAOvB,aAAa,eAAE,OAAO,EAAE,SAAS;AACnC,CAAC;AAID,IAAa,sBAAsB,eAAE,MAAM,CAAC,eAAE,QAAQ,QAAQ,GAAG,YAAY,CAAC;AAEzC,eAAE,MAAM,mBAAmB"}

package/out/zero-cache/src/services/change-streamer/snapshot-reservations.d.ts

@@ -1,7 +1,7 @@
 import type { LogContext } from '@rocicorp/logger';
 import type { Source } from '../../types/streams.ts';
 import type { BackupConfig } from './change-streamer-service.ts';
-import type { SnapshotMessage } from './snapshot.ts';
+import type { SnapshotMessage } from './snapshot-message.ts';
 import type { ChangeLogReadSource } from './sqlite-change-log-read-router.ts';
 export declare class SnapshotReservations {
     #private;

package/out/zero-cache/src/services/change-streamer/snapshot-reservations.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"snapshot-reservations.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot-reservations.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAOnD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,8BAA8B,CAAC;AAC/D,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,eAAe,CAAC;AACnD,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,oCAAoC,CAAC;AAE5E,qBAAa,oBAAoB;;IAM/B,YACE,EAAE,EAAE,UAAU,EACd,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC,GAAG,SAAS,EAKjD;IAED,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,eAAe,CAAC,CAU5C;IAED,KAAK,CAAC,MAAM,EAAE,MAAM,QAEnB;IAED,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,eAAe,CAAC,GAAG,OAAO,CAElE;IAsCD,qBAAqB,YAOpB;IAED,kBAAkB,IAAI,MAAM,EAAE,CAI7B;IAED,0EAA0E;IAC1E,UAAU,CACR,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,MAAM,EACtB,YAAY,EAAE,MAAM,EACpB,MAAM,EAAE,mBAAmB,GAC1B,IAAI,CAmBN;IAeD;;;;OAIG;IACH,uBAAuB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAE/C;IAED,qBAAqB,aAKpB;CACF"}
+{"version":3,"file":"snapshot-reservations.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot-reservations.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAOnD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,8BAA8B,CAAC;AAC/D,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,uBAAuB,CAAC;AAC3D,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,oCAAoC,CAAC;AAE5E,qBAAa,oBAAoB;;IAM/B,YACE,EAAE,EAAE,UAAU,EACd,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC,GAAG,SAAS,EAKjD;IAED,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,eAAe,CAAC,CAU5C;IAED,KAAK,CAAC,MAAM,EAAE,MAAM,QAEnB;IAED,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,eAAe,CAAC,GAAG,OAAO,CAElE;IAsCD,qBAAqB,YAOpB;IAED,kBAAkB,IAAI,MAAM,EAAE,CAI7B;IAED,0EAA0E;IAC1E,UAAU,CACR,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,MAAM,EACtB,YAAY,EAAE,MAAM,EACpB,MAAM,EAAE,mBAAmB,GAC1B,IAAI,CAmBN;IAeD;;;;OAIG;IACH,uBAAuB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAE/C;IAED,qBAAqB,aAKpB;CACF"}

package/out/zero-cache/src/services/change-streamer/snapshot-reservations.js

@@ -17,7 +17,7 @@ var SnapshotReservations = class {
 		const instanceID = {};
 		const downstream = Subscription.create({ cleanup: () => this.#close(taskID, instanceID) });
 		this.#reservations.set(taskID, new Reservation(instanceID, downstream));
-		this.#lc.info?.(`created snasphot reservation for ${taskID}`);
+		this.#lc.info?.(`created snapshot reservation for ${taskID}`);
 		return downstream;
 	}
 	close(taskID) {

package/out/zero-cache/src/services/change-streamer/snapshot-reservations.js.map

@@ -1 +1 @@
-{"version":3,"file":"snapshot-reservations.js","names":["#lc","#backupConfig","#onClose","#reservations","#close","#metricAttrs","#replicaSize","#downstream","#watermark","#delayNoted"],"sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot-reservations.ts"],"sourcesContent":["import {statSync} from 'node:fs';\nimport type {LogContext} from '@rocicorp/logger';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {\n  litestreamMonitorMetricAttrs,\n  litestreamSnapshotReservationConfirmDuration,\n  litestreamSnapshotReservationDuration,\n} from '../litestream/metrics.ts';\nimport type {BackupConfig} from './change-streamer-service.ts';\nimport type {SnapshotMessage} from './snapshot.ts';\nimport type {ChangeLogReadSource} from './sqlite-change-log-read-router.ts';\n\nexport class SnapshotReservations {\n  readonly #lc: LogContext;\n  readonly #backupConfig: BackupConfig;\n  readonly #onClose: ((taskID: string) => void) | undefined;\n  readonly #reservations = new Map<string, Reservation>();\n\n  constructor(\n    lc: LogContext,\n    backupConfig: BackupConfig,\n    onClose?: ((taskID: string) => void) | undefined,\n  ) {\n    this.#lc = lc.withContext('component', 'snapshot-reserver');\n    this.#backupConfig = backupConfig;\n    this.#onClose = onClose;\n  }\n\n  open(taskID: string): Source<SnapshotMessage> {\n    this.close(taskID);\n\n    const instanceID = {};\n    const downstream = Subscription.create<SnapshotMessage>({\n      cleanup: () => this.#close(taskID, instanceID),\n    });\n    this.#reservations.set(taskID, new Reservation(instanceID, downstream));\n    this.#lc.info?.(`created snasphot reservation for ${taskID}`);\n    return downstream;\n  }\n\n  close(taskID: string) {\n    this.#close(taskID, undefined);\n  }\n\n  isCurrent(taskID: string, source: Source<SnapshotMessage>): boolean {\n    return this.#reservations.get(taskID)?.owns(source) ?? false;\n  }\n\n  #metricAttrs() {\n    return litestreamMonitorMetricAttrs(\n      this.#backupConfig.backupURL,\n      this.#backupConfig.litestreamVersion,\n      'view_syncer',\n    );\n  }\n\n  #close(taskID: string, cancelledInstanceID: InstanceID | undefined) {\n    const res = this.#reservations.get(taskID);\n    if (\n      res &&\n      (!cancelledInstanceID || res.instanceID === cancelledInstanceID)\n    ) {\n      // Note: delete first, so that the reservation is gone when close() is called.\n      this.#reservations.delete(taskID);\n      this.#onClose?.(taskID);\n      res.close();\n\n      const duration = Date.now() - res.startTime.getTime();\n      this.#lc.info?.(\n        `ended snapshot reservation for ${taskID} (${duration} ms)`,\n      );\n      // `result` reports how the reservation ended and `confirmed` whether it\n      // ever received its bounds. They are separate dimensions: a follower\n      // that gives up while waiting for a confirmation is\n      // `result=cancelled, confirmed=false`, which a single collapsed\n      // attribute cannot distinguish from a client that simply went away.\n      litestreamSnapshotReservationDuration().recordMs(duration, {\n        ...this.#metricAttrs(),\n        result: cancelledInstanceID ? 'cancelled' : 'closed',\n        confirmed: res.confirmed(),\n      });\n    }\n  }\n\n  confirmationsRequired() {\n    for (const res of this.#reservations.values()) {\n      if (!res.confirmed()) {\n        return true;\n      }\n    }\n    return false;\n  }\n\n  unconfirmedTaskIDs(): string[] {\n    return [...this.#reservations.entries()]\n      .filter(([, reservation]) => !reservation.confirmed())\n      .map(([taskID]) => taskID);\n  }\n\n  /** Confirms one reservation with the bounds of its pinned read source. */\n  confirmFor(\n    taskID: string,\n    replicaVersion: string,\n    minWatermark: string,\n    source: ChangeLogReadSource,\n  ): void {\n    const res = this.#reservations.get(taskID);\n    if (res && !res.confirmed()) {\n      this.#lc.info?.(\n        `reserving change-log entries since ${minWatermark} for ${taskID}`,\n      );\n      res.confirm(\n        this.#backupConfig.backupURL,\n        replicaVersion,\n        minWatermark,\n        this.#replicaSize(),\n      );\n      // Measured from `open()`, not from the first confirmation attempt: what\n      // matters is how long the follower waited before it could restore.\n      litestreamSnapshotReservationConfirmDuration().recordMs(\n        Date.now() - res.startTime.getTime(),\n        {...this.#metricAttrs(), source},\n      );\n    }\n  }\n\n  #replicaSize(): number | undefined {\n    const {replicaFile} = this.#backupConfig;\n    if (!replicaFile) {\n      return undefined;\n    }\n    try {\n      return statSync(replicaFile).size;\n    } catch (e) {\n      this.#lc.warn?.(`Unable to get the size of ${replicaFile}`, e);\n      return undefined;\n    }\n  }\n\n  /**\n   * Notes that a confirmation was deferred, returning true only the first time\n   * for this reservation. Confirmation is retried on every backup, so an\n   * undeduplicated count would measure backups rather than delayed followers.\n   */\n  noteConfirmationDelayed(taskID: string): boolean {\n    return this.#reservations.get(taskID)?.noteDelayed() ?? false;\n  }\n\n  getReservedWatermarks() {\n    return Array.from(\n      this.#reservations.values(),\n      ({reservedWatermark}) => reservedWatermark,\n    ).filter(watermark => watermark !== null);\n  }\n}\n\ntype InstanceID = {};\n\nclass Reservation {\n  readonly instanceID: InstanceID;\n  readonly startTime: Date = new Date();\n  readonly #downstream: Subscription<SnapshotMessage>;\n  #watermark: string | null = null;\n  #delayNoted = false;\n\n  constructor(\n    instanceID: InstanceID,\n    downstream: Subscription<SnapshotMessage>,\n  ) {\n    this.instanceID = instanceID;\n    this.#downstream = downstream;\n  }\n\n  get reservedWatermark() {\n    return this.#watermark;\n  }\n\n  confirmed() {\n    return this.#watermark !== null;\n  }\n\n  owns(source: Source<SnapshotMessage>): boolean {\n    return this.#downstream === source;\n  }\n\n  noteDelayed(): boolean {\n    if (this.#delayNoted) {\n      return false;\n    }\n    this.#delayNoted = true;\n    return true;\n  }\n\n  confirm(\n    backupURL: string,\n    replicaVersion: string,\n    minWatermark: string,\n    replicaSize: number | undefined,\n  ) {\n    if (this.#watermark === null) {\n      if (this.#downstream.active) {\n        this.#downstream.push([\n          'status',\n          {tag: 'status', backupURL, replicaVersion, minWatermark, replicaSize},\n        ]);\n      }\n      this.#watermark = minWatermark;\n    }\n  }\n\n  close() {\n    this.#downstream.cancel();\n  }\n}\n"],"mappings":";;;;AAaA,IAAa,uBAAb,MAAkC;CAChC;CACA;CACA;CACA,gCAAyB,IAAI,IAAyB;CAEtD,YACE,IACA,cACA,SACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,mBAAmB;EAC1D,KAAKC,gBAAgB;EACrB,KAAKC,WAAW;CAClB;CAEA,KAAK,QAAyC;EAC5C,KAAK,MAAM,MAAM;EAEjB,MAAM,aAAa,CAAC;EACpB,MAAM,aAAa,aAAa,OAAwB,EACtD,eAAe,KAAKE,OAAO,QAAQ,UAAU,EAC/C,CAAC;EACD,KAAKD,cAAc,IAAI,QAAQ,IAAI,YAAY,YAAY,UAAU,CAAC;EACtE,KAAKH,IAAI,OAAO,oCAAoC,QAAQ;EAC5D,OAAO;CACT;CAEA,MAAM,QAAgB;EACpB,KAAKI,OAAO,QAAQ,KAAA,CAAS;CAC/B;CAEA,UAAU,QAAgB,QAA0C;EAClE,OAAO,KAAKD,cAAc,IAAI,MAAM,GAAG,KAAK,MAAM,KAAK;CACzD;CAEA,eAAe;EACb,OAAO,6BACL,KAAKF,cAAc,WACnB,KAAKA,cAAc,mBACnB,aACF;CACF;CAEA,OAAO,QAAgB,qBAA6C;EAClE,MAAM,MAAM,KAAKE,cAAc,IAAI,MAAM;EACzC,IACE,QACC,CAAC,uBAAuB,IAAI,eAAe,sBAC5C;GAEA,KAAKA,cAAc,OAAO,MAAM;GAChC,KAAKD,WAAW,MAAM;GACtB,IAAI,MAAM;GAEV,MAAM,WAAW,KAAK,IAAI,IAAI,IAAI,UAAU,QAAQ;GACpD,KAAKF,IAAI,OACP,kCAAkC,OAAO,IAAI,SAAS,KACxD;GAMA,sCAAsC,EAAE,SAAS,UAAU;IACzD,GAAG,KAAKK,aAAa;IACrB,QAAQ,sBAAsB,cAAc;IAC5C,WAAW,IAAI,UAAU;GAC3B,CAAC;EACH;CACF;CAEA,wBAAwB;EACtB,KAAK,MAAM,OAAO,KAAKF,cAAc,OAAO,GAC1C,IAAI,CAAC,IAAI,UAAU,GACjB,OAAO;EAGX,OAAO;CACT;CAEA,qBAA+B;EAC7B,OAAO,CAAC,GAAG,KAAKA,cAAc,QAAQ,CAAC,EACpC,QAAQ,GAAG,iBAAiB,CAAC,YAAY,UAAU,CAAC,EACpD,KAAK,CAAC,YAAY,MAAM;CAC7B;;CAGA,WACE,QACA,gBACA,cACA,QACM;EACN,MAAM,MAAM,KAAKA,cAAc,IAAI,MAAM;EACzC,IAAI,OAAO,CAAC,IAAI,UAAU,GAAG;GAC3B,KAAKH,IAAI,OACP,sCAAsC,aAAa,OAAO,QAC5D;GACA,IAAI,QACF,KAAKC,cAAc,WACnB,gBACA,cACA,KAAKK,aAAa,CACpB;GAGA,6CAA6C,EAAE,SAC7C,KAAK,IAAI,IAAI,IAAI,UAAU,QAAQ,GACnC;IAAC,GAAG,KAAKD,aAAa;IAAG;GAAM,CACjC;EACF;CACF;CAEA,eAAmC;EACjC,MAAM,EAAC,gBAAe,KAAKJ;EAC3B,IAAI,CAAC,aACH;EAEF,IAAI;GACF,OAAO,SAAS,WAAW,EAAE;EAC/B,SAAS,GAAG;GACV,KAAKD,IAAI,OAAO,6BAA6B,eAAe,CAAC;GAC7D;EACF;CACF;;;;;;CAOA,wBAAwB,QAAyB;EAC/C,OAAO,KAAKG,cAAc,IAAI,MAAM,GAAG,YAAY,KAAK;CAC1D;CAEA,wBAAwB;EACtB,OAAO,MAAM,KACX,KAAKA,cAAc,OAAO,IACzB,EAAC,wBAAuB,iBAC3B,EAAE,QAAO,cAAa,cAAc,IAAI;CAC1C;AACF;AAIA,IAAM,cAAN,MAAkB;CAChB;CACA,4BAA2B,IAAI,KAAK;CACpC;CACA,aAA4B;CAC5B,cAAc;CAEd,YACE,YACA,YACA;EACA,KAAK,aAAa;EAClB,KAAKI,cAAc;CACrB;CAEA,IAAI,oBAAoB;EACtB,OAAO,KAAKC;CACd;CAEA,YAAY;EACV,OAAO,KAAKA,eAAe;CAC7B;CAEA,KAAK,QAA0C;EAC7C,OAAO,KAAKD,gBAAgB;CAC9B;CAEA,cAAuB;EACrB,IAAI,KAAKE,aACP,OAAO;EAET,KAAKA,cAAc;EACnB,OAAO;CACT;CAEA,QACE,WACA,gBACA,cACA,aACA;EACA,IAAI,KAAKD,eAAe,MAAM;GAC5B,IAAI,KAAKD,YAAY,QACnB,KAAKA,YAAY,KAAK,CACpB,UACA;IAAC,KAAK;IAAU;IAAW;IAAgB;IAAc;GAAW,CACtE,CAAC;GAEH,KAAKC,aAAa;EACpB;CACF;CAEA,QAAQ;EACN,KAAKD,YAAY,OAAO;CAC1B;AACF"}
+{"version":3,"file":"snapshot-reservations.js","names":["#lc","#backupConfig","#onClose","#reservations","#close","#metricAttrs","#replicaSize","#downstream","#watermark","#delayNoted"],"sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot-reservations.ts"],"sourcesContent":["import {statSync} from 'node:fs';\nimport type {LogContext} from '@rocicorp/logger';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {\n  litestreamMonitorMetricAttrs,\n  litestreamSnapshotReservationConfirmDuration,\n  litestreamSnapshotReservationDuration,\n} from '../litestream/metrics.ts';\nimport type {BackupConfig} from './change-streamer-service.ts';\nimport type {SnapshotMessage} from './snapshot-message.ts';\nimport type {ChangeLogReadSource} from './sqlite-change-log-read-router.ts';\n\nexport class SnapshotReservations {\n  readonly #lc: LogContext;\n  readonly #backupConfig: BackupConfig;\n  readonly #onClose: ((taskID: string) => void) | undefined;\n  readonly #reservations = new Map<string, Reservation>();\n\n  constructor(\n    lc: LogContext,\n    backupConfig: BackupConfig,\n    onClose?: ((taskID: string) => void) | undefined,\n  ) {\n    this.#lc = lc.withContext('component', 'snapshot-reserver');\n    this.#backupConfig = backupConfig;\n    this.#onClose = onClose;\n  }\n\n  open(taskID: string): Source<SnapshotMessage> {\n    this.close(taskID);\n\n    const instanceID = {};\n    const downstream = Subscription.create<SnapshotMessage>({\n      cleanup: () => this.#close(taskID, instanceID),\n    });\n    this.#reservations.set(taskID, new Reservation(instanceID, downstream));\n    this.#lc.info?.(`created snapshot reservation for ${taskID}`);\n    return downstream;\n  }\n\n  close(taskID: string) {\n    this.#close(taskID, undefined);\n  }\n\n  isCurrent(taskID: string, source: Source<SnapshotMessage>): boolean {\n    return this.#reservations.get(taskID)?.owns(source) ?? false;\n  }\n\n  #metricAttrs() {\n    return litestreamMonitorMetricAttrs(\n      this.#backupConfig.backupURL,\n      this.#backupConfig.litestreamVersion,\n      'view_syncer',\n    );\n  }\n\n  #close(taskID: string, cancelledInstanceID: InstanceID | undefined) {\n    const res = this.#reservations.get(taskID);\n    if (\n      res &&\n      (!cancelledInstanceID || res.instanceID === cancelledInstanceID)\n    ) {\n      // Note: delete first, so that the reservation is gone when close() is called.\n      this.#reservations.delete(taskID);\n      this.#onClose?.(taskID);\n      res.close();\n\n      const duration = Date.now() - res.startTime.getTime();\n      this.#lc.info?.(\n        `ended snapshot reservation for ${taskID} (${duration} ms)`,\n      );\n      // `result` reports how the reservation ended and `confirmed` whether it\n      // ever received its bounds. They are separate dimensions: a follower\n      // that gives up while waiting for a confirmation is\n      // `result=cancelled, confirmed=false`, which a single collapsed\n      // attribute cannot distinguish from a client that simply went away.\n      litestreamSnapshotReservationDuration().recordMs(duration, {\n        ...this.#metricAttrs(),\n        result: cancelledInstanceID ? 'cancelled' : 'closed',\n        confirmed: res.confirmed(),\n      });\n    }\n  }\n\n  confirmationsRequired() {\n    for (const res of this.#reservations.values()) {\n      if (!res.confirmed()) {\n        return true;\n      }\n    }\n    return false;\n  }\n\n  unconfirmedTaskIDs(): string[] {\n    return [...this.#reservations.entries()]\n      .filter(([, reservation]) => !reservation.confirmed())\n      .map(([taskID]) => taskID);\n  }\n\n  /** Confirms one reservation with the bounds of its pinned read source. */\n  confirmFor(\n    taskID: string,\n    replicaVersion: string,\n    minWatermark: string,\n    source: ChangeLogReadSource,\n  ): void {\n    const res = this.#reservations.get(taskID);\n    if (res && !res.confirmed()) {\n      this.#lc.info?.(\n        `reserving change-log entries since ${minWatermark} for ${taskID}`,\n      );\n      res.confirm(\n        this.#backupConfig.backupURL,\n        replicaVersion,\n        minWatermark,\n        this.#replicaSize(),\n      );\n      // Measured from `open()`, not from the first confirmation attempt: what\n      // matters is how long the follower waited before it could restore.\n      litestreamSnapshotReservationConfirmDuration().recordMs(\n        Date.now() - res.startTime.getTime(),\n        {...this.#metricAttrs(), source},\n      );\n    }\n  }\n\n  #replicaSize(): number | undefined {\n    const {replicaFile} = this.#backupConfig;\n    if (!replicaFile) {\n      return undefined;\n    }\n    try {\n      return statSync(replicaFile).size;\n    } catch (e) {\n      this.#lc.warn?.(`Unable to get the size of ${replicaFile}`, e);\n      return undefined;\n    }\n  }\n\n  /**\n   * Notes that a confirmation was deferred, returning true only the first time\n   * for this reservation. Confirmation is retried on every backup, so an\n   * undeduplicated count would measure backups rather than delayed followers.\n   */\n  noteConfirmationDelayed(taskID: string): boolean {\n    return this.#reservations.get(taskID)?.noteDelayed() ?? false;\n  }\n\n  getReservedWatermarks() {\n    return Array.from(\n      this.#reservations.values(),\n      ({reservedWatermark}) => reservedWatermark,\n    ).filter(watermark => watermark !== null);\n  }\n}\n\ntype InstanceID = {};\n\nclass Reservation {\n  readonly instanceID: InstanceID;\n  readonly startTime: Date = new Date();\n  readonly #downstream: Subscription<SnapshotMessage>;\n  #watermark: string | null = null;\n  #delayNoted = false;\n\n  constructor(\n    instanceID: InstanceID,\n    downstream: Subscription<SnapshotMessage>,\n  ) {\n    this.instanceID = instanceID;\n    this.#downstream = downstream;\n  }\n\n  get reservedWatermark() {\n    return this.#watermark;\n  }\n\n  confirmed() {\n    return this.#watermark !== null;\n  }\n\n  owns(source: Source<SnapshotMessage>): boolean {\n    return this.#downstream === source;\n  }\n\n  noteDelayed(): boolean {\n    if (this.#delayNoted) {\n      return false;\n    }\n    this.#delayNoted = true;\n    return true;\n  }\n\n  confirm(\n    backupURL: string,\n    replicaVersion: string,\n    minWatermark: string,\n    replicaSize: number | undefined,\n  ) {\n    if (this.#watermark === null) {\n      if (this.#downstream.active) {\n        this.#downstream.push([\n          'status',\n          {tag: 'status', backupURL, replicaVersion, minWatermark, replicaSize},\n        ]);\n      }\n      this.#watermark = minWatermark;\n    }\n  }\n\n  close() {\n    this.#downstream.cancel();\n  }\n}\n"],"mappings":";;;;AAaA,IAAa,uBAAb,MAAkC;CAChC;CACA;CACA;CACA,gCAAyB,IAAI,IAAyB;CAEtD,YACE,IACA,cACA,SACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,mBAAmB;EAC1D,KAAKC,gBAAgB;EACrB,KAAKC,WAAW;CAClB;CAEA,KAAK,QAAyC;EAC5C,KAAK,MAAM,MAAM;EAEjB,MAAM,aAAa,CAAC;EACpB,MAAM,aAAa,aAAa,OAAwB,EACtD,eAAe,KAAKE,OAAO,QAAQ,UAAU,EAC/C,CAAC;EACD,KAAKD,cAAc,IAAI,QAAQ,IAAI,YAAY,YAAY,UAAU,CAAC;EACtE,KAAKH,IAAI,OAAO,oCAAoC,QAAQ;EAC5D,OAAO;CACT;CAEA,MAAM,QAAgB;EACpB,KAAKI,OAAO,QAAQ,KAAA,CAAS;CAC/B;CAEA,UAAU,QAAgB,QAA0C;EAClE,OAAO,KAAKD,cAAc,IAAI,MAAM,GAAG,KAAK,MAAM,KAAK;CACzD;CAEA,eAAe;EACb,OAAO,6BACL,KAAKF,cAAc,WACnB,KAAKA,cAAc,mBACnB,aACF;CACF;CAEA,OAAO,QAAgB,qBAA6C;EAClE,MAAM,MAAM,KAAKE,cAAc,IAAI,MAAM;EACzC,IACE,QACC,CAAC,uBAAuB,IAAI,eAAe,sBAC5C;GAEA,KAAKA,cAAc,OAAO,MAAM;GAChC,KAAKD,WAAW,MAAM;GACtB,IAAI,MAAM;GAEV,MAAM,WAAW,KAAK,IAAI,IAAI,IAAI,UAAU,QAAQ;GACpD,KAAKF,IAAI,OACP,kCAAkC,OAAO,IAAI,SAAS,KACxD;GAMA,sCAAsC,EAAE,SAAS,UAAU;IACzD,GAAG,KAAKK,aAAa;IACrB,QAAQ,sBAAsB,cAAc;IAC5C,WAAW,IAAI,UAAU;GAC3B,CAAC;EACH;CACF;CAEA,wBAAwB;EACtB,KAAK,MAAM,OAAO,KAAKF,cAAc,OAAO,GAC1C,IAAI,CAAC,IAAI,UAAU,GACjB,OAAO;EAGX,OAAO;CACT;CAEA,qBAA+B;EAC7B,OAAO,CAAC,GAAG,KAAKA,cAAc,QAAQ,CAAC,EACpC,QAAQ,GAAG,iBAAiB,CAAC,YAAY,UAAU,CAAC,EACpD,KAAK,CAAC,YAAY,MAAM;CAC7B;;CAGA,WACE,QACA,gBACA,cACA,QACM;EACN,MAAM,MAAM,KAAKA,cAAc,IAAI,MAAM;EACzC,IAAI,OAAO,CAAC,IAAI,UAAU,GAAG;GAC3B,KAAKH,IAAI,OACP,sCAAsC,aAAa,OAAO,QAC5D;GACA,IAAI,QACF,KAAKC,cAAc,WACnB,gBACA,cACA,KAAKK,aAAa,CACpB;GAGA,6CAA6C,EAAE,SAC7C,KAAK,IAAI,IAAI,IAAI,UAAU,QAAQ,GACnC;IAAC,GAAG,KAAKD,aAAa;IAAG;GAAM,CACjC;EACF;CACF;CAEA,eAAmC;EACjC,MAAM,EAAC,gBAAe,KAAKJ;EAC3B,IAAI,CAAC,aACH;EAEF,IAAI;GACF,OAAO,SAAS,WAAW,EAAE;EAC/B,SAAS,GAAG;GACV,KAAKD,IAAI,OAAO,6BAA6B,eAAe,CAAC;GAC7D;EACF;CACF;;;;;;CAOA,wBAAwB,QAAyB;EAC/C,OAAO,KAAKG,cAAc,IAAI,MAAM,GAAG,YAAY,KAAK;CAC1D;CAEA,wBAAwB;EACtB,OAAO,MAAM,KACX,KAAKA,cAAc,OAAO,IACzB,EAAC,wBAAuB,iBAC3B,EAAE,QAAO,cAAa,cAAc,IAAI;CAC1C;AACF;AAIA,IAAM,cAAN,MAAkB;CAChB;CACA,4BAA2B,IAAI,KAAK;CACpC;CACA,aAA4B;CAC5B,cAAc;CAEd,YACE,YACA,YACA;EACA,KAAK,aAAa;EAClB,KAAKI,cAAc;CACrB;CAEA,IAAI,oBAAoB;EACtB,OAAO,KAAKC;CACd;CAEA,YAAY;EACV,OAAO,KAAKA,eAAe;CAC7B;CAEA,KAAK,QAA0C;EAC7C,OAAO,KAAKD,gBAAgB;CAC9B;CAEA,cAAuB;EACrB,IAAI,KAAKE,aACP,OAAO;EAET,KAAKA,cAAc;EACnB,OAAO;CACT;CAEA,QACE,WACA,gBACA,cACA,aACA;EACA,IAAI,KAAKD,eAAe,MAAM;GAC5B,IAAI,KAAKD,YAAY,QACnB,KAAKA,YAAY,KAAK,CACpB,UACA;IAAC,KAAK;IAAU;IAAW;IAAgB;IAAc;GAAW,CACtE,CAAC;GAEH,KAAKC,aAAa;EACpB;CACF;CAEA,QAAQ;EACN,KAAKD,YAAY,OAAO;CAC1B;AACF"}

package/out/zero-cache/src/services/change-streamer/snapshot.d.ts

@@ -12,93 +12,11 @@
  *   backed up changes.
  */
 import type { LogContext } from '@rocicorp/logger';
-import * as v from '../../../../shared/src/valita.ts';
-import { type NormalizedZeroConfig } from '../../config/normalize.ts';
-import type { Source } from '../../types/streams.ts';
-declare const statusSchema: v.ObjectType<{
-    tag: v.Type<"status">;
-    /**
-     * The location from which litestream should perform the restore.
-     */
-    backupURL: v.Type<string>;
-    /**
-     * The `replicaVersion` of the backup. If a subscriber's restored or
-     * existing replica is of a different version, it should delete it and
-     * retry the restore from litestream (i.e. equivalent to a
-     * `WrongReplicaVersion` response from a `/changes` subscription).
-     */
-    replicaVersion: v.Type<string>;
-    /**
-     * The earliest watermark from which catchup is possible. If the
-     * subscriber's replica is older that this watermark, it should delete it
-     * and (retry the) restore from litestream (i.e. equivalent to a
-     * `WatermarkTooOld` response from a `/changes` subscription).
-     */
-    minWatermark: v.Type<string>;
-    /**
-     * The size in bytes of the replication-manager's replica when the snapshot
-     * was reserved, used as an estimate of the size of the restored replica
-     * when reporting restore progress. Absent from older replication-managers.
-     */
-    replicaSize: v.Optional<number>;
-}, undefined>;
-export type SnapshotStatus = v.Infer<typeof statusSchema>;
-declare const statusMessageSchema: v.TupleType<[v.Type<"status">, v.ObjectType<{
-    tag: v.Type<"status">;
-    /**
-     * The location from which litestream should perform the restore.
-     */
-    backupURL: v.Type<string>;
-    /**
-     * The `replicaVersion` of the backup. If a subscriber's restored or
-     * existing replica is of a different version, it should delete it and
-     * retry the restore from litestream (i.e. equivalent to a
-     * `WrongReplicaVersion` response from a `/changes` subscription).
-     */
-    replicaVersion: v.Type<string>;
-    /**
-     * The earliest watermark from which catchup is possible. If the
-     * subscriber's replica is older that this watermark, it should delete it
-     * and (retry the) restore from litestream (i.e. equivalent to a
-     * `WatermarkTooOld` response from a `/changes` subscription).
-     */
-    minWatermark: v.Type<string>;
-    /**
-     * The size in bytes of the replication-manager's replica when the snapshot
-     * was reserved, used as an estimate of the size of the restored replica
-     * when reporting restore progress. Absent from older replication-managers.
-     */
-    replicaSize: v.Optional<number>;
-}, undefined>]>;
-export declare const snapshotMessageSchema: v.UnionType<[v.TupleType<[v.Type<"status">, v.ObjectType<{
-    tag: v.Type<"status">;
-    /**
-     * The location from which litestream should perform the restore.
-     */
-    backupURL: v.Type<string>;
-    /**
-     * The `replicaVersion` of the backup. If a subscriber's restored or
-     * existing replica is of a different version, it should delete it and
-     * retry the restore from litestream (i.e. equivalent to a
-     * `WrongReplicaVersion` response from a `/changes` subscription).
-     */
-    replicaVersion: v.Type<string>;
-    /**
-     * The earliest watermark from which catchup is possible. If the
-     * subscriber's replica is older that this watermark, it should delete it
-     * and (retry the) restore from litestream (i.e. equivalent to a
-     * `WatermarkTooOld` response from a `/changes` subscription).
-     */
-    minWatermark: v.Type<string>;
-    /**
-     * The size in bytes of the replication-manager's replica when the snapshot
-     * was reserved, used as an estimate of the size of the restored replica
-     * when reporting restore progress. Absent from older replication-managers.
-     */
-    replicaSize: v.Optional<number>;
-}, undefined>]>]>;
-export type SnapshotMessage = v.Infer<typeof statusMessageSchema>;
-export type ReserveSnapshot = (lc: LogContext, config: NormalizedZeroConfig) => Promise<Source<SnapshotMessage>>;
-export declare function reserveAndGetSnapshotStatus(lc: LogContext, config: NormalizedZeroConfig, reserve?: ReserveSnapshot): Promise<SnapshotStatus>;
-export {};
+import type { ReservationFollowup, SnapshotReserver } from './change-streamer-http.ts';
+import type { SnapshotStatus } from './subscribe.ts';
+export type { SnapshotStatus };
+export declare function reserveAndGetSnapshotStatus(lc: LogContext, taskID: string, changeStreamer: SnapshotReserver): Promise<{
+    reserved: SnapshotStatus;
+    followup: ReservationFollowup;
+}>;
 //# sourceMappingURL=snapshot.d.ts.map

package/out/zero-cache/src/services/change-streamer/snapshot.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"snapshot.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,KAAK,CAAC,MAAM,kCAAkC,CAAC;AACtD,OAAO,EAAC,KAAK,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AAEpE,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAGnD,QAAA,MAAM,YAAY;;IAGhB;;OAEG;;IAGH;;;;;OAKG;;IAGH;;;;;OAKG;;IAGH;;;;OAIG;;aAEH,CAAC;AAEH,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AAE1D,QAAA,MAAM,mBAAmB;;IA/BvB;;OAEG;;IAGH;;;;;OAKG;;IAGH;;;;;OAKG;;IAGH;;;;OAIG;;eAMmE,CAAC;AAEzE,eAAO,MAAM,qBAAqB;;IAjChC;;OAEG;;IAGH;;;;;OAKG;;IAGH;;;;;OAKG;;IAGH;;;;OAIG;;iBAQ4D,CAAC;AAElE,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAElE,MAAM,MAAM,eAAe,GAAG,CAC5B,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,oBAAoB,KACzB,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC;AAEtC,wBAAgB,2BAA2B,CACzC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,oBAAoB,EAC5B,OAAO,GAAE,eAAiC,GACzC,OAAO,CAAC,cAAc,CAAC,CAwEzB"}
+{"version":3,"file":"snapshot.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AASjD,OAAO,KAAK,EACV,mBAAmB,EACnB,gBAAgB,EACjB,MAAM,2BAA2B,CAAC;AACnC,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,gBAAgB,CAAC;AAEnD,YAAY,EAAC,cAAc,EAAC,CAAC;AAE7B,wBAAsB,2BAA2B,CAC/C,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,gBAAgB,GAC/B,OAAO,CAAC;IAAC,QAAQ,EAAE,cAAc,CAAC;IAAC,QAAQ,EAAE,mBAAmB,CAAA;CAAC,CAAC,CAmDpE"}