@rocicorp/zero 0.20.2025050600 → 0.20.2025050800

package/out/zero-cache/src/server/worker-dispatcher.js

@@ -2,7 +2,6 @@ import { LogContext } from '@rocicorp/logger';
 import UrlPattern from 'url-pattern';
 import { assert } from "../../../shared/src/asserts.js";
 import { h32 } from "../../../shared/src/hash.js";
-import { getSubscriberContext } from "../services/change-streamer/change-streamer-http.js";
 import { RunningState } from "../services/running-state.js";
 import { createWebSocketHandoffHandler, } from "../types/websocket-handoff.js";
 import { getConnectParams } from "../workers/connect-params.js";
@@ -55,8 +54,13 @@ export class WorkerDispatcher {
             assert(syncers.length === 0 && mutator === undefined, 'Dispatch to the change-streamer via the main port ' +
                 'is only allowed in multi-node mode');
             assert(changeStreamer, 'Received a change-streamer request without a change-streamer worker');
+            const url = new URL(req.url ?? '', 'http://unused/');
+            const path = parsePath(url);
+            if (!path) {
+                throw new Error(`Invalid URL: ${req.url}`);
+            }
             return {
-                payload: getSubscriberContext(req),
+                payload: path.action,
                 receiver: changeStreamer,
             };
         });

package/out/zero-cache/src/server/worker-dispatcher.js.map

@@ -1 +1 @@
-{"version":3,"file":"worker-dispatcher.js","sourceRoot":"","sources":["../../../../../zero-cache/src/server/worker-dispatcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAE5C,OAAO,UAAU,MAAM,aAAa,CAAC;AACrC,OAAO,EAAC,MAAM,EAAC,MAAM,gCAAgC,CAAC;AACtD,OAAO,EAAC,GAAG,EAAC,MAAM,6BAA6B,CAAC;AAChD,OAAO,EAAC,oBAAoB,EAAC,MAAM,qDAAqD,CAAC;AACzF,OAAO,EAAC,YAAY,EAAC,MAAM,8BAA8B,CAAC;AAI1D,OAAO,EACL,6BAA6B,GAG9B,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAC,gBAAgB,EAAC,MAAM,8BAA8B,CAAC;AAE9D,MAAM,OAAO,gBAAgB;IAClB,EAAE,GAAG,mBAAmB,CAAC;IACzB,GAAG,CAAa;IAEhB,MAAM,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAE5C,YACE,EAAc,EACd,MAAc,EACd,MAAc,EACd,OAAiB,EACjB,OAA2B,EAC3B,cAAkC;QAElC,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC;QAEd,SAAS,aAAa,CAAC,GAA0B;YAC/C,MAAM,EAAC,OAAO,EAAE,GAAG,EAAE,CAAC,EAAC,GAAG,GAAG,CAAC;YAC9B,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,gBAAgB,CAAC,CAAC;YAC/C,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;YAC5B,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC;YACvC,CAAC;YACD,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrC,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAC;YAC3C,CAAC;YACD,MAAM,EAAC,MAAM,EAAE,KAAK,EAAC,GAAG,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;YAChE,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC;YACzB,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,MAAM,WAAW,GAAG,6BAA6B,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE;YAC1D,MAAM,CACJ,OAAO,KAAK,SAAS,EACrB,yEAAyE,CAC1E,CAAC;YACF,OAAO,EAAC,OAAO,EAAE,aAAa,CAAC,GAAG,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,6BAA6B,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE;YAC1D,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,+CAA+C,CAAC,CAAC;YACxE,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;YAClC,MAAM,EAAC,aAAa,EAAC,GAAG,MAAM,CAAC;YAE/B,uEAAuE;YACvE,qEAAqE;YACrE,sEAAsE;YACtE,wEAAwE;YACxE,oEAAoE;YACpE,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,GAAG,GAAG,GAAG,aAAa,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;YAElE,EAAE,CAAC,KAAK,EAAE,CAAC,cAAc,aAAa,cAAc,MAAM,EAAE,CAAC,CAAC;YAC9D,OAAO,EAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,EAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,MAAM,qBAAqB,GAAG,6BAA6B,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE;YACpE,qEAAqE;YACrE,qEAAqE;YACrE,uEAAuE;YACvE,0DAA0D;YAC1D,0DAA0D;YAC1D,MAAM,CACJ,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,KAAK,SAAS,EAC7C,oDAAoD;gBAClD,oCAAoC,CACvC,CAAC;YACF,MAAM,CACJ,cAAc,EACd,qEAAqE,CACtE,CAAC;YACF,OAAO;gBACL,OAAO,EAAE,oBAAoB,CAAC,GAAG,CAAC;gBAClC,QAAQ,EAAE,cAAc;aACzB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,8EAA8E;QAC9E,MAAM,CAAC,aAAa,CAAmB,SAAS,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE;YAChE,MAAM,EAAC,OAAO,EAAE,IAAI,EAAC,GAAG,GAAG,CAAC;YAC5B,MAAM,EAAC,GAAG,EAAE,CAAC,EAAC,GAAG,OAAO,CAAC;YACzB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,gBAAgB,CAAC,CAAC;YAC/C,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;YAC5B,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC;YACvC,CAAC;YACD,MAAM,UAAU,GAAG,CAAC,MAA+B,EAAE,EAAE,CACrD,MAAM,CAAC,OAAO,EAAE,MAAgB,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YACvD,QAAQ,IAAI,CAAC,MAAM,EAAE,CAAC;gBACpB,KAAK,MAAM;oBACT,OAAO,UAAU,CAAC,WAAW,CAAC,CAAC;gBACjC,KAAK,aAAa;oBAChB,OAAO,UAAU,CAAC,qBAAqB,CAAC,CAAC;gBAC3C,KAAK,QAAQ;oBACX,OAAO,UAAU,CAAC,WAAW,CAAC,CAAC;gBACjC;oBACE,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC;YACzC,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,GAAG;QACD,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;IAC/B,CAAC;IAED,IAAI;QACF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC3B,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;IAC/B,CAAC;CACF;AAED,MAAM,WAAW,GAAG,IAAI,UAAU,CAAC,oCAAoC,CAAC,CAAC;AAEzE,MAAM,UAAU,SAAS,CACvB,GAAQ;IAIR,+CAA+C;IAC/C,OAAO,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,SAAS,CAAC;AACtD,CAAC,CAAC,qEAAqE;AACvE,4DAA4D"}
+{"version":3,"file":"worker-dispatcher.js","sourceRoot":"","sources":["../../../../../zero-cache/src/server/worker-dispatcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAE5C,OAAO,UAAU,MAAM,aAAa,CAAC;AACrC,OAAO,EAAC,MAAM,EAAC,MAAM,gCAAgC,CAAC;AACtD,OAAO,EAAC,GAAG,EAAC,MAAM,6BAA6B,CAAC;AAChD,OAAO,EAAC,YAAY,EAAC,MAAM,8BAA8B,CAAC;AAI1D,OAAO,EACL,6BAA6B,GAG9B,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAC,gBAAgB,EAAC,MAAM,8BAA8B,CAAC;AAE9D,MAAM,OAAO,gBAAgB;IAClB,EAAE,GAAG,mBAAmB,CAAC;IACzB,GAAG,CAAa;IAEhB,MAAM,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAE5C,YACE,EAAc,EACd,MAAc,EACd,MAAc,EACd,OAAiB,EACjB,OAA2B,EAC3B,cAAkC;QAElC,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC;QAEd,SAAS,aAAa,CAAC,GAA0B;YAC/C,MAAM,EAAC,OAAO,EAAE,GAAG,EAAE,CAAC,EAAC,GAAG,GAAG,CAAC;YAC9B,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,gBAAgB,CAAC,CAAC;YAC/C,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;YAC5B,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC;YACvC,CAAC;YACD,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrC,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAC;YAC3C,CAAC;YACD,MAAM,EAAC,MAAM,EAAE,KAAK,EAAC,GAAG,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;YAChE,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC;YACzB,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,MAAM,WAAW,GAAG,6BAA6B,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE;YAC1D,MAAM,CACJ,OAAO,KAAK,SAAS,EACrB,yEAAyE,CAC1E,CAAC;YACF,OAAO,EAAC,OAAO,EAAE,aAAa,CAAC,GAAG,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,6BAA6B,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE;YAC1D,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,+CAA+C,CAAC,CAAC;YACxE,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;YAClC,MAAM,EAAC,aAAa,EAAC,GAAG,MAAM,CAAC;YAE/B,uEAAuE;YACvE,qEAAqE;YACrE,sEAAsE;YACtE,wEAAwE;YACxE,oEAAoE;YACpE,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,GAAG,GAAG,GAAG,aAAa,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;YAElE,EAAE,CAAC,KAAK,EAAE,CAAC,cAAc,aAAa,cAAc,MAAM,EAAE,CAAC,CAAC;YAC9D,OAAO,EAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,EAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,MAAM,qBAAqB,GAAG,6BAA6B,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE;YACpE,qEAAqE;YACrE,qEAAqE;YACrE,uEAAuE;YACvE,0DAA0D;YAC1D,0DAA0D;YAC1D,MAAM,CACJ,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,KAAK,SAAS,EAC7C,oDAAoD;gBAClD,oCAAoC,CACvC,CAAC;YACF,MAAM,CACJ,cAAc,EACd,qEAAqE,CACtE,CAAC;YACF,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,EAAE,EAAE,gBAAgB,CAAC,CAAC;YACrD,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;YAC5B,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,MAAM,IAAI,KAAK,CAAC,gBAAgB,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;YAC7C,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI,CAAC,MAAM;gBACpB,QAAQ,EAAE,cAAc;aACzB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,8EAA8E;QAC9E,MAAM,CAAC,aAAa,CAAmB,SAAS,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE;YAChE,MAAM,EAAC,OAAO,EAAE,IAAI,EAAC,GAAG,GAAG,CAAC;YAC5B,MAAM,EAAC,GAAG,EAAE,CAAC,EAAC,GAAG,OAAO,CAAC;YACzB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,gBAAgB,CAAC,CAAC;YAC/C,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;YAC5B,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC;YACvC,CAAC;YACD,MAAM,UAAU,GAAG,CAAC,MAA+B,EAAE,EAAE,CACrD,MAAM,CAAC,OAAO,EAAE,MAAgB,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YACvD,QAAQ,IAAI,CAAC,MAAM,EAAE,CAAC;gBACpB,KAAK,MAAM;oBACT,OAAO,UAAU,CAAC,WAAW,CAAC,CAAC;gBACjC,KAAK,aAAa;oBAChB,OAAO,UAAU,CAAC,qBAAqB,CAAC,CAAC;gBAC3C,KAAK,QAAQ;oBACX,OAAO,UAAU,CAAC,WAAW,CAAC,CAAC;gBACjC;oBACE,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC;YACzC,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,GAAG;QACD,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;IAC/B,CAAC;IAED,IAAI;QACF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC3B,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;IAC/B,CAAC;CACF;AAED,MAAM,WAAW,GAAG,IAAI,UAAU,CAAC,oCAAoC,CAAC,CAAC;AAEzE,MAAM,UAAU,SAAS,CAAC,GAAQ;IAQhC,+CAA+C;IAC/C,OAAO,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,SAAS,CAAC;AACtD,CAAC,CAAC,qEAAqE;AACvE,4DAA4D"}

package/out/zero-cache/src/services/change-streamer/backup-monitor.d.ts

@@ -1,13 +1,48 @@
 import type { LogContext } from '@rocicorp/logger';
+import { Subscription } from '../../types/subscription.ts';
 import type { Service } from '../service.ts';
 import type { ChangeStreamerService } from './change-streamer.ts';
+import type { SnapshotMessage } from './snapshot.ts';
 export declare const CHECK_INTERVAL_MS: number;
+/**
+ * The BackupMonitor polls the litestream "/metrics" endpoint to track the
+ * watermark (label) value of the `litestream_replica_progress` gauge and
+ * schedules cleanup of change log entries that can be purged as a result.
+ *
+ * See: https: *github.com/rocicorp/litestream/pull/3
+ *
+ * Note that change log entries cannot simply be purged as soon as they
+ * have been applied and backed up by litestream. Consider the case in which
+ * litestream backs up new wal segments every minute, but it takes 5 minutes
+ * to restore a replica: if a zero-cache starts restoring a replica at
+ * minute 0, and new watermarks are replicated at minutes 1, 2, 3, 4, and 5,
+ * purging changelog records as soon as those watermarks are replicated would
+ * result in the zero-cache not being able to catch up from minute 0 once it
+ * has finished restoring the replica.
+ *
+ * The `/snapshot` reservation protocol is used to prevent premature change
+ * log cleanup:
+ * - Clients restoring a snapshot initiate a `/snapshot` request and hold that
+ *   request open while it restores its snapshot, prepares it, and
+ *   starts its subscription to the change stream. During this time, no
+ *   cleanups are scheduled.
+ * - When the subscription is started, the interval since the beginning of
+ *   of the reservation is tracked to increase the background cleanup delay
+ *   interval if needed. The reservation is ended (and request closed), and
+ *   cleanup scheduling is resumed with the current delay interval.
+ *
+ * Note that the reservation request is the primary mechanism by which
+ * premature change log cleanup is prevented. The cleanup delay interval is
+ * a secondary safeguard.
+ */
 export declare class BackupMonitor implements Service {
     #private;
     readonly id = "backup-monitor";
-    constructor(lc: LogContext, metricsEndpoint: string, changeStreamer: ChangeStreamerService, cleanupDelayMs: number | undefined);
+    constructor(lc: LogContext, backupURL: string, metricsEndpoint: string, changeStreamer: ChangeStreamerService, initialCleanupDelayMs: number);
     run(): Promise<void>;
-    readonly checkMetrics: () => Promise<void>;
+    startSnapshotReservation(taskID: string): Subscription<SnapshotMessage>;
+    endReservation(taskID: string, updateCleanupDelay?: boolean): void;
+    readonly checkWatermarksAndScheduleCleanup: () => Promise<void>;
     stop(): Promise<void>;
 }
 //# sourceMappingURL=backup-monitor.d.ts.map

package/out/zero-cache/src/services/change-streamer/backup-monitor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"backup-monitor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAEhE,eAAO,MAAM,iBAAiB,QAAY,CAAC;AA4B3C,qBAAa,aAAc,YAAW,OAAO;;IAC3C,QAAQ,CAAC,EAAE,oBAAoB;gBAU7B,EAAE,EAAE,UAAU,EACd,eAAe,EAAE,MAAM,EACvB,cAAc,EAAE,qBAAqB,EACrC,cAAc,EAAE,MAAM,GAAG,SAAS;IAQpC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAUpB,QAAQ,CAAC,YAAY,sBAsDnB;IAEF,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAQtB"}
+{"version":3,"file":"backup-monitor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAEzD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAChE,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,eAAe,CAAC;AAEnD,eAAO,MAAM,iBAAiB,QAAY,CAAC;AAQ3C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,qBAAa,aAAc,YAAW,OAAO;;IAC3C,QAAQ,CAAC,EAAE,oBAAoB;gBAe7B,EAAE,EAAE,UAAU,EACd,SAAS,EAAE,MAAM,EACjB,eAAe,EAAE,MAAM,EACvB,cAAc,EAAE,qBAAqB,EACrC,qBAAqB,EAAE,MAAM;IAgB/B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAYpB,wBAAwB,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,CAAC,eAAe,CAAC;IAgBvE,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,kBAAkB,UAAO;IAoBxD,QAAQ,CAAC,iCAAiC,sBAWxC;IAgEF,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAYtB"}

package/out/zero-cache/src/services/change-streamer/backup-monitor.js

@@ -1,102 +1,167 @@
 import parsePrometheusTextFormat from 'parse-prometheus-text-format';
 import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
+import { Subscription } from "../../types/subscription.js";
 import { RunningState } from "../running-state.js";
 export const CHECK_INTERVAL_MS = 60 * 1000;
-const DEFAULT_CLEANUP_DELAY_MS = 60 * 1000;
-// The BackupMonitor polls the litestream "/metrics" endpoint to track the
-// watermark (label) value of the `litestream_replica_progress` gauge and
-// schedule cleanup of change log entries that can be purged as a result.
-//
-// See: https://github.com/rocicorp/litestream/pull/3
-//
-// Note that change log purging involves two time values:
-// 1. The time at which a watermark has been backed up by litestream
-// 2. The time it takes to restore the replica from the backup
-//
-// Namely, it is not safe to simply purge change log entries as soon as they
-// have been applied and backed up by litestream. Consider the case in which
-// litestream backs up new wal segments every minute, but it takes 5 minutes
-// to restore a replica: if a zero-cache starts restoring a replica at
-// minute 0, and new watermarks are replicated at minutes 1, 2, 3, 4, and 5,
-// purging changelog records as soon as those watermarks are replicated would
-// result in the zero-cache not being able to catch up from minute 0 once it
-// has finished restoring the replica.
-//
-// Consequently, cleanup is delayed by an interval equivalent to the time it
-// takes for the litestream restore, which is calculated by the runner (when
-// it performed the restore or the initial sync). In addition to this delay,
-// the ChangeStreamerService itself adds at least 30 seconds of padding before
-// purging records, ensuring all active subscribers are past the watermark
-// before purging preceding records.
+const MIN_CLEANUP_DELAY_MS = 30 * 1000;
+/**
+ * The BackupMonitor polls the litestream "/metrics" endpoint to track the
+ * watermark (label) value of the `litestream_replica_progress` gauge and
+ * schedules cleanup of change log entries that can be purged as a result.
+ *
+ * See: https: *github.com/rocicorp/litestream/pull/3
+ *
+ * Note that change log entries cannot simply be purged as soon as they
+ * have been applied and backed up by litestream. Consider the case in which
+ * litestream backs up new wal segments every minute, but it takes 5 minutes
+ * to restore a replica: if a zero-cache starts restoring a replica at
+ * minute 0, and new watermarks are replicated at minutes 1, 2, 3, 4, and 5,
+ * purging changelog records as soon as those watermarks are replicated would
+ * result in the zero-cache not being able to catch up from minute 0 once it
+ * has finished restoring the replica.
+ *
+ * The `/snapshot` reservation protocol is used to prevent premature change
+ * log cleanup:
+ * - Clients restoring a snapshot initiate a `/snapshot` request and hold that
+ *   request open while it restores its snapshot, prepares it, and
+ *   starts its subscription to the change stream. During this time, no
+ *   cleanups are scheduled.
+ * - When the subscription is started, the interval since the beginning of
+ *   of the reservation is tracked to increase the background cleanup delay
+ *   interval if needed. The reservation is ended (and request closed), and
+ *   cleanup scheduling is resumed with the current delay interval.
+ *
+ * Note that the reservation request is the primary mechanism by which
+ * premature change log cleanup is prevented. The cleanup delay interval is
+ * a secondary safeguard.
+ */
 export class BackupMonitor {
     id = 'backup-monitor';
     #lc;
+    #backupURL;
     #metricsEndpoint;
     #changeStreamer;
     #state = new RunningState(this.id);
+    #reservations = new Map();
+    #watermarks = new Map();
+    #lastWatermark = '';
     #cleanupDelayMs;
-    #cleanupTimers = new Map();
     #checkMetricsTimer;
-    constructor(lc, metricsEndpoint, changeStreamer, cleanupDelayMs) {
+    constructor(lc, backupURL, metricsEndpoint, changeStreamer, initialCleanupDelayMs) {
         this.#lc = lc.withContext('component', this.id);
+        this.#backupURL = backupURL;
         this.#metricsEndpoint = metricsEndpoint;
         this.#changeStreamer = changeStreamer;
-        this.#cleanupDelayMs = cleanupDelayMs ?? DEFAULT_CLEANUP_DELAY_MS;
+        this.#cleanupDelayMs = Math.max(initialCleanupDelayMs, MIN_CLEANUP_DELAY_MS);
+        this.#lc.info?.(`backup monitor started ${initialCleanupDelayMs} ms after snapshot restore`);
     }
     run() {
         this.#lc.info?.(`monitoring backups at ${this.#metricsEndpoint} with ` +
             `${this.#cleanupDelayMs} ms cleanup delay`);
-        this.#checkMetricsTimer = setInterval(this.checkMetrics, CHECK_INTERVAL_MS);
+        this.#checkMetricsTimer = setInterval(this.checkWatermarksAndScheduleCleanup, CHECK_INTERVAL_MS);
         return this.#state.stopped();
     }
+    startSnapshotReservation(taskID) {
+        this.#lc.info?.(`pausing change-log cleanup while ${taskID} snapshots`);
+        // In the case of retries, only track the last reservation.
+        this.#reservations.get(taskID)?.sub.cancel();
+        const sub = Subscription.create({
+            // If the reservation still exists when the connection closes
+            // (e.g. subscriber crashed), clean it up without updating the
+            // cleanup delay.
+            cleanup: () => this.endReservation(taskID, false),
+        });
+        this.#reservations.set(taskID, { start: new Date(), sub });
+        sub.push(['status', { tag: 'status', backupURL: this.#backupURL }]);
+        return sub;
+    }
+    endReservation(taskID, updateCleanupDelay = true) {
+        const res = this.#reservations.get(taskID);
+        if (res === undefined) {
+            return;
+        }
+        this.#reservations.delete(taskID);
+        const { start, sub } = res;
+        sub.cancel(); // closes the connection if still open
+        if (updateCleanupDelay) {
+            const duration = Date.now() - start.getTime();
+            this.#lc.info?.(`snapshot initialized by ${taskID} in ${duration} ms`);
+            if (duration > this.#cleanupDelayMs) {
+                this.#cleanupDelayMs = duration;
+                this.#lc.info?.(`increased cleanup delay to ${duration} ms`);
+            }
+        }
+    }
     // Exported for testing
-    checkMetrics = async () => {
+    checkWatermarksAndScheduleCleanup = async () => {
         try {
-            const resp = await fetch(this.#metricsEndpoint);
-            if (!resp.ok) {
-                this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, await resp.text());
-                return;
-            }
-            const families = parsePrometheusTextFormat(await resp.text());
-            for (const family of families) {
-                if (family.type === 'GAUGE' &&
-                    family.name === 'litestream_replica_progress') {
-                    for (const metric of family.metrics) {
-                        const watermark = metric.labels?.watermark;
-                        if (watermark && !this.#cleanupTimers.has(watermark)) {
-                            const time = new Date(parseFloat(metric.value) * 1000);
-                            const now = Date.now();
-                            const cleanupAt = Math.max(time.getTime() + this.#cleanupDelayMs, now);
-                            this.#lc.info?.(`replicated watermark=${watermark} to ${metric.labels?.name}` +
-                                ` at ${time.toISOString()}. scheduling cleanup at ` +
-                                new Date(cleanupAt).toISOString() +
-                                ` (in ${cleanupAt - now} ms)`);
-                            this.#cleanupTimers.set(watermark, setTimeout(() => {
-                                this.#changeStreamer.scheduleCleanup(watermark);
-                                // Clean up all previous watermarks, leaving the latest one in the
-                                // map to avoid redundant scheduling.
-                                for (const [key, timer] of this.#cleanupTimers.entries()) {
-                                    if (key === watermark) {
-                                        break;
-                                    }
-                                    this.#cleanupTimers.delete(key);
-                                    clearTimeout(timer);
-                                }
-                            }, cleanupAt - now));
-                            this.#lc.debug?.('timers', [...this.#cleanupTimers.entries()]);
-                        }
-                    }
-                }
-            }
+            await this.#checkWatermarks();
         }
         catch (e) {
             this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);
         }
+        try {
+            this.#scheduleCleanup();
+        }
+        catch (e) {
+            this.#lc.warn?.(`error scheduling cleanup`, e);
+        }
     };
+    async #checkWatermarks() {
+        const resp = await fetch(this.#metricsEndpoint);
+        if (!resp.ok) {
+            this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, await resp.text());
+            return;
+        }
+        const families = parsePrometheusTextFormat(await resp.text());
+        for (const family of families) {
+            if (family.type === 'GAUGE' &&
+                family.name === 'litestream_replica_progress') {
+                for (const metric of family.metrics) {
+                    const watermark = metric.labels?.watermark;
+                    if (watermark &&
+                        watermark > this.#lastWatermark &&
+                        !this.#watermarks.has(watermark)) {
+                        const time = new Date(parseFloat(metric.value) * 1000);
+                        this.#lc.info?.(`replicated watermark=${watermark} to ${metric.labels?.name}` +
+                            ` at ${time.toISOString()}.`);
+                        this.#watermarks.set(watermark, time);
+                    }
+                }
+            }
+        }
+    }
+    #scheduleCleanup() {
+        if (this.#reservations.size > 0) {
+            this.#lc.info?.(`watermark cleanup paused for snapshot(s): ${[...this.#reservations.keys()]}`);
+            return;
+        }
+        const latestCleanupTime = Date.now() - this.#cleanupDelayMs;
+        let maxWatermark = '';
+        for (const [watermark, backupTime] of this.#watermarks.entries()) {
+            if (backupTime.getTime() <= latestCleanupTime &&
+                watermark > maxWatermark) {
+                maxWatermark = watermark;
+            }
+        }
+        if (maxWatermark.length) {
+            this.#changeStreamer.scheduleCleanup(maxWatermark);
+            for (const watermark of this.#watermarks.keys()) {
+                if (watermark <= maxWatermark) {
+                    this.#watermarks.delete(watermark);
+                }
+            }
+            this.#lastWatermark = maxWatermark;
+        }
+    }
     stop() {
         clearInterval(this.#checkMetricsTimer);
-        for (const cleanupTimer of this.#cleanupTimers.values()) {
-            clearTimeout(cleanupTimer);
+        for (const { sub } of this.#reservations.values()) {
+            // Close any pending reservations. This commonly happens when a new
+            // replication-manager makes a `/snapshot` reservation on the existing
+            // replication-manager, and then shuts it down when it takes over the
+            // replication slot.
+            sub.cancel();
         }
         this.#state.stop(this.#lc);
         return promiseVoid;

package/out/zero-cache/src/services/change-streamer/backup-monitor.js.map

@@ -1 +1 @@
-{"version":3,"file":"backup-monitor.js","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"names":[],"mappings":"AACA,OAAO,yBAAyB,MAAM,8BAA8B,CAAC;AACrE,OAAO,EAAC,WAAW,EAAC,MAAM,6CAA6C,CAAC;AACxE,OAAO,EAAC,YAAY,EAAC,MAAM,qBAAqB,CAAC;AAIjD,MAAM,CAAC,MAAM,iBAAiB,GAAG,EAAE,GAAG,IAAI,CAAC;AAC3C,MAAM,wBAAwB,GAAG,EAAE,GAAG,IAAI,CAAC;AAE3C,0EAA0E;AAC1E,yEAAyE;AACzE,yEAAyE;AACzE,EAAE;AACF,qDAAqD;AACrD,EAAE;AACF,yDAAyD;AACzD,oEAAoE;AACpE,8DAA8D;AAC9D,EAAE;AACF,4EAA4E;AAC5E,4EAA4E;AAC5E,4EAA4E;AAC5E,sEAAsE;AACtE,4EAA4E;AAC5E,6EAA6E;AAC7E,4EAA4E;AAC5E,sCAAsC;AACtC,EAAE;AACF,4EAA4E;AAC5E,4EAA4E;AAC5E,4EAA4E;AAC5E,8EAA8E;AAC9E,0EAA0E;AAC1E,oCAAoC;AACpC,MAAM,OAAO,aAAa;IACf,EAAE,GAAG,gBAAgB,CAAC;IACtB,GAAG,CAAa;IAChB,gBAAgB,CAAS;IACzB,eAAe,CAAwB;IACvC,MAAM,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACnC,eAAe,CAAS;IACxB,cAAc,GAAG,IAAI,GAAG,EAA0B,CAAC;IAC5D,kBAAkB,CAA6B;IAE/C,YACE,EAAc,EACd,eAAuB,EACvB,cAAqC,EACrC,cAAkC;QAElC,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC,WAAW,CAAC,WAAW,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,gBAAgB,GAAG,eAAe,CAAC;QACxC,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC;QACtC,IAAI,CAAC,eAAe,GAAG,cAAc,IAAI,wBAAwB,CAAC;IACpE,CAAC;IAED,GAAG;QACD,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CACb,yBAAyB,IAAI,CAAC,gBAAgB,QAAQ;YACpD,GAAG,IAAI,CAAC,eAAe,mBAAmB,CAC7C,CAAC;QACF,IAAI,CAAC,kBAAkB,GAAG,WAAW,CAAC,IAAI,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAC;QAC5E,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;IAC/B,CAAC;IAED,uBAAuB;IACd,YAAY,GAAG,KAAK,IAAI,EAAE;QACjC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;YAChD,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;gBACb,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CACb,8BAA8B,IAAI,CAAC,gBAAgB,EAAE,EACrD,MAAM,IAAI,CAAC,IAAI,EAAE,CAClB,CAAC;gBACF,OAAO;YACT,CAAC;YACD,MAAM,QAAQ,GAAG,yBAAyB,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YAC9D,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;gBAC9B,IACE,MAAM,CAAC,IAAI,KAAK,OAAO;oBACvB,MAAM,CAAC,IAAI,KAAK,6BAA6B,EAC7C,CAAC;oBACD,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;wBACpC,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC;wBAC3C,IAAI,SAAS,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;4BACrD,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC;4BACvD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;4BACvB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CACxB,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,eAAe,EACrC,GAAG,CACJ,CAAC;4BACF,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CACb,wBAAwB,SAAS,OAAO,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE;gCAC3D,OAAO,IAAI,CAAC,WAAW,EAAE,0BAA0B;gCACnD,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE;gCACjC,QAAQ,SAAS,GAAG,GAAG,MAAM,CAChC,CAAC;4BACF,IAAI,CAAC,cAAc,CAAC,GAAG,CACrB,SAAS,EACT,UAAU,CAAC,GAAG,EAAE;gCACd,IAAI,CAAC,eAAe,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;gCAChD,kEAAkE;gCAClE,qCAAqC;gCACrC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE,CAAC;oCACzD,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;wCACtB,MAAM;oCACR,CAAC;oCACD,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;oCAChC,YAAY,CAAC,KAAK,CAAC,CAAC;gCACtB,CAAC;4BACH,CAAC,EAAE,SAAS,GAAG,GAAG,CAAC,CACpB,CAAC;4BACF,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE,CAAC,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;wBACjE,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,8BAA8B,IAAI,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC,CAAC;QAC5E,CAAC;IACH,CAAC,CAAC;IAEF,IAAI;QACF,aAAa,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QACvC,KAAK,MAAM,YAAY,IAAI,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,EAAE,CAAC;YACxD,YAAY,CAAC,YAAY,CAAC,CAAC;QAC7B,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC3B,OAAO,WAAW,CAAC;IACrB,CAAC;CACF"}
+{"version":3,"file":"backup-monitor.js","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"names":[],"mappings":"AACA,OAAO,yBAAyB,MAAM,8BAA8B,CAAC;AACrE,OAAO,EAAC,WAAW,EAAC,MAAM,6CAA6C,CAAC;AACxE,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AACzD,OAAO,EAAC,YAAY,EAAC,MAAM,qBAAqB,CAAC;AAKjD,MAAM,CAAC,MAAM,iBAAiB,GAAG,EAAE,GAAG,IAAI,CAAC;AAC3C,MAAM,oBAAoB,GAAG,EAAE,GAAG,IAAI,CAAC;AAOvC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,MAAM,OAAO,aAAa;IACf,EAAE,GAAG,gBAAgB,CAAC;IACtB,GAAG,CAAa;IAChB,UAAU,CAAS;IACnB,gBAAgB,CAAS;IACzB,eAAe,CAAwB;IACvC,MAAM,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEnC,aAAa,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC/C,WAAW,GAAG,IAAI,GAAG,EAAgB,CAAC;IAE/C,cAAc,GAAW,EAAE,CAAC;IAC5B,eAAe,CAAS;IACxB,kBAAkB,CAA6B;IAE/C,YACE,EAAc,EACd,SAAiB,EACjB,eAAuB,EACvB,cAAqC,EACrC,qBAA6B;QAE7B,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC,WAAW,CAAC,WAAW,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,gBAAgB,GAAG,eAAe,CAAC;QACxC,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC;QACtC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAC7B,qBAAqB,EACrB,oBAAoB,CACrB,CAAC;QAEF,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CACb,0BAA0B,qBAAqB,4BAA4B,CAC5E,CAAC;IACJ,CAAC;IAED,GAAG;QACD,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CACb,yBAAyB,IAAI,CAAC,gBAAgB,QAAQ;YACpD,GAAG,IAAI,CAAC,eAAe,mBAAmB,CAC7C,CAAC;QACF,IAAI,CAAC,kBAAkB,GAAG,WAAW,CACnC,IAAI,CAAC,iCAAiC,EACtC,iBAAiB,CAClB,CAAC;QACF,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;IAC/B,CAAC;IAED,wBAAwB,CAAC,MAAc;QACrC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,oCAAoC,MAAM,YAAY,CAAC,CAAC;QACxE,2DAA2D;QAC3D,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC;QAE7C,MAAM,GAAG,GAAG,YAAY,CAAC,MAAM,CAAkB;YAC/C,6DAA6D;YAC7D,8DAA8D;YAC9D,iBAAiB;YACjB,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,KAAK,CAAC;SAClD,CAAC,CAAC;QACH,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,EAAE,EAAC,KAAK,EAAE,IAAI,IAAI,EAAE,EAAE,GAAG,EAAC,CAAC,CAAC;QACzD,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAC,GAAG,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,CAAC,UAAU,EAAC,CAAC,CAAC,CAAC;QAClE,OAAO,GAAG,CAAC;IACb,CAAC;IAED,cAAc,CAAC,MAAc,EAAE,kBAAkB,GAAG,IAAI;QACtD,MAAM,GAAG,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC3C,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACtB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAClC,MAAM,EAAC,KAAK,EAAE,GAAG,EAAC,GAAG,GAAG,CAAC;QACzB,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,sCAAsC;QAEpD,IAAI,kBAAkB,EAAE,CAAC;YACvB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;YAC9C,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,2BAA2B,MAAM,OAAO,QAAQ,KAAK,CAAC,CAAC;YACvE,IAAI,QAAQ,GAAG,IAAI,CAAC,eAAe,EAAE,CAAC;gBACpC,IAAI,CAAC,eAAe,GAAG,QAAQ,CAAC;gBAChC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,8BAA8B,QAAQ,KAAK,CAAC,CAAC;YAC/D,CAAC;QACH,CAAC;IACH,CAAC;IAED,uBAAuB;IACd,iCAAiC,GAAG,KAAK,IAAI,EAAE;QACtD,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAChC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,8BAA8B,IAAI,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC,CAAC;QAC5E,CAAC;QACD,IAAI,CAAC;YACH,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC1B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,0BAA0B,EAAE,CAAC,CAAC,CAAC;QACjD,CAAC;IACH,CAAC,CAAC;IAEF,KAAK,CAAC,gBAAgB;QACpB,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAChD,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;YACb,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CACb,8BAA8B,IAAI,CAAC,gBAAgB,EAAE,EACrD,MAAM,IAAI,CAAC,IAAI,EAAE,CAClB,CAAC;YACF,OAAO;QACT,CAAC;QACD,MAAM,QAAQ,GAAG,yBAAyB,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9D,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;YAC9B,IACE,MAAM,CAAC,IAAI,KAAK,OAAO;gBACvB,MAAM,CAAC,IAAI,KAAK,6BAA6B,EAC7C,CAAC;gBACD,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;oBACpC,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC;oBAC3C,IACE,SAAS;wBACT,SAAS,GAAG,IAAI,CAAC,cAAc;wBAC/B,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,EAChC,CAAC;wBACD,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC;wBACvD,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CACb,wBAAwB,SAAS,OAAO,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE;4BAC3D,OAAO,IAAI,CAAC,WAAW,EAAE,GAAG,CAC/B,CAAC;wBACF,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;oBACxC,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,gBAAgB;QACd,IAAI,IAAI,CAAC,aAAa,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAChC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CACb,6CAA6C,CAAC,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,EAAE,CAC9E,CAAC;YACF,OAAO;QACT,CAAC;QACD,MAAM,iBAAiB,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC;QAC5D,IAAI,YAAY,GAAG,EAAE,CAAC;QACtB,KAAK,MAAM,CAAC,SAAS,EAAE,UAAU,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,EAAE,CAAC;YACjE,IACE,UAAU,CAAC,OAAO,EAAE,IAAI,iBAAiB;gBACzC,SAAS,GAAG,YAAY,EACxB,CAAC;gBACD,YAAY,GAAG,SAAS,CAAC;YAC3B,CAAC;QACH,CAAC;QACD,IAAI,YAAY,CAAC,MAAM,EAAE,CAAC;YACxB,IAAI,CAAC,eAAe,CAAC,eAAe,CAAC,YAAY,CAAC,CAAC;YACnD,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,CAAC;gBAChD,IAAI,SAAS,IAAI,YAAY,EAAE,CAAC;oBAC9B,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;gBACrC,CAAC;YACH,CAAC;YACD,IAAI,CAAC,cAAc,GAAG,YAAY,CAAC;QACrC,CAAC;IACH,CAAC;IAED,IAAI;QACF,aAAa,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QACvC,KAAK,MAAM,EAAC,GAAG,EAAC,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,EAAE,CAAC;YAChD,mEAAmE;YACnE,sEAAsE;YACtE,qEAAqE;YACrE,oBAAoB;YACpB,GAAG,CAAC,MAAM,EAAE,CAAC;QACf,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC3B,OAAO,WAAW,CAAC;IACrB,CAAC;CACF"}

package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts

@@ -1,21 +1,23 @@
 import { LogContext } from '@rocicorp/logger';
 import { IncomingMessage } from 'node:http';
-import type { PostgresDB } from '../../types/pg.ts';
 import { type Worker } from '../../types/processes.ts';
 import type { ShardID } from '../../types/shards.ts';
 import { type Source } from '../../types/streams.ts';
 import { HttpService, type Options } from '../http-service.ts';
+import type { BackupMonitor } from './backup-monitor.ts';
 import { type ChangeStreamer, type Downstream, type SubscriberContext } from './change-streamer.ts';
+import { type SnapshotMessage } from './snapshot.ts';
 export declare class ChangeStreamerHttpServer extends HttpService {
     #private;
     readonly id = "change-streamer-http-server";
     constructor(lc: LogContext, opts: Options, parent: Worker);
-    setDelegate(delegate: ChangeStreamer): void;
+    setDelegates(changeStreamer: ChangeStreamer, backupMonitor: BackupMonitor | null): void;
     protected _respondToKeepalive(): boolean;
 }
 export declare class ChangeStreamerHttpClient implements ChangeStreamer {
     #private;
-    constructor(lc: LogContext, shardID: ShardID, changeDB: PostgresDB);
+    constructor(lc: LogContext, shardID: ShardID, changeDB: string);
+    reserveSnapshot(taskID: string): Promise<Source<SnapshotMessage>>;
     subscribe(ctx: SubscriberContext): Promise<Source<Downstream>>;
 }
 type RequestHeaders = Pick<IncomingMessage, 'url' | 'headers'>;

package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-http.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAC,eAAe,EAAC,MAAM,WAAW,CAAC;AAI1C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAClD,OAAO,EAAC,KAAK,MAAM,EAAC,MAAM,0BAA0B,CAAC;AACrD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAsB,KAAK,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAIxE,OAAO,EAAC,WAAW,EAAE,KAAK,OAAO,EAAC,MAAM,oBAAoB,CAAC;AAC7D,OAAO,EAGL,KAAK,cAAc,EACnB,KAAK,UAAU,EACf,KAAK,iBAAiB,EACvB,MAAM,sBAAsB,CAAC;AAY9B,qBAAa,wBAAyB,SAAQ,WAAW;;IACvD,QAAQ,CAAC,EAAE,iCAAiC;gBAGhC,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM;IAkBzD,WAAW,CAAC,QAAQ,EAAE,cAAc;IAOpC,SAAS,CAAC,mBAAmB,IAAI,OAAO;CA6BzC;AAED,qBAAa,wBAAyB,YAAW,cAAc;;gBAKjD,EAAE,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU;IAM5D,SAAS,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;CAgBrE;AAED,KAAK,cAAc,GAAG,IAAI,CAAC,eAAe,EAAE,KAAK,GAAG,SAAS,CAAC,CAAC;AAE/D,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,cAAc,GAAG,iBAAiB,CAa3E"}
+{"version":3,"file":"change-streamer-http.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAC,eAAe,EAAC,MAAM,WAAW,CAAC;AAM1C,OAAO,EAAC,KAAK,MAAM,EAAC,MAAM,0BAA0B,CAAC;AACrD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAsB,KAAK,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAIxE,OAAO,EAAC,WAAW,EAAE,KAAK,OAAO,EAAC,MAAM,oBAAoB,CAAC;AAC7D,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAGL,KAAK,cAAc,EACnB,KAAK,UAAU,EACf,KAAK,iBAAiB,EACvB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAwB,KAAK,eAAe,EAAC,MAAM,eAAe,CAAC;AAW1E,qBAAa,wBAAyB,SAAQ,WAAW;;IACvD,QAAQ,CAAC,EAAE,iCAAiC;gBAIhC,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM;IAoBzD,YAAY,CACV,cAAc,EAAE,cAAc,EAC9B,aAAa,EAAE,aAAa,GAAG,IAAI;IASrC,SAAS,CAAC,mBAAmB,IAAI,OAAO;CAwEzC;AAED,qBAAa,wBAAyB,YAAW,cAAc;;gBAKjD,EAAE,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM;IAyBxD,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IASjE,SAAS,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;CAQrE;AAED,KAAK,cAAc,GAAG,IAAI,CAAC,eAAe,EAAE,KAAK,GAAG,SAAS,CAAC,CAAC;AAE/D,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,cAAc,GAAG,iBAAiB,CAc3E"}

package/out/zero-cache/src/services/change-streamer/change-streamer-http.js

@@ -4,6 +4,7 @@ import { IncomingMessage } from 'node:http';
 import WebSocket from 'ws';
 import { assert } from "../../../../shared/src/asserts.js";
 import { must } from "../../../../shared/src/must.js";
+import { pgClient } from "../../types/pg.js";
 import {} from "../../types/processes.js";
 import { streamIn, streamOut } from "../../types/streams.js";
 import { URLParams } from "../../types/url-params.js";
@@ -12,50 +13,81 @@ import { closeWithError, PROTOCOL_ERROR } from "../../types/ws.js";
 import { HttpService } from "../http-service.js";
 import { downstreamSchema, PROTOCOL_VERSION, } from "./change-streamer.js";
 import { discoverChangeStreamerAddress } from "./schema/tables.js";
+import { snapshotMessageSchema } from "./snapshot.js";
 const MIN_SUPPORTED_PROTOCOL_VERSION = 1;
-const DIRECT_PATH_PATTERN = '/replication/:version/changes';
-const TENANT_PATH_PATTERN = '/:tenant' + DIRECT_PATH_PATTERN;
-const PATH_REGEX = /(?<tenant>[^/]+\/)?\/replication\/v(?<version>\d+)\/changes$/;
+const SNAPSHOT_PATH_PATTERN = '/replication/:version/snapshot';
+const CHANGES_PATH_PATTERN = '/replication/:version/changes';
+const PATH_REGEX = /\/replication\/v(?<version>\d+)\/(changes|snapshot)$/;
+const SNAPSHOT_PATH = `/replication/v${PROTOCOL_VERSION}/snapshot`;
 const CHANGES_PATH = `/replication/v${PROTOCOL_VERSION}/changes`;
 export class ChangeStreamerHttpServer extends HttpService {
     id = 'change-streamer-http-server';
-    #delegate = null;
+    #changeStreamer = null;
+    #backupMonitor = null;
     constructor(lc, opts, parent) {
         super('change-streamer-http-server', lc, opts, async (fastify) => {
             await fastify.register(websocket);
-            // fastify does not support optional path components, so we just
-            // register both patterns.
-            fastify.get(DIRECT_PATH_PATTERN, { websocket: true }, this.#subscribe);
-            fastify.get(TENANT_PATH_PATTERN, { websocket: true }, this.#subscribe);
-            installWebSocketReceiver(lc, fastify.websocketServer, this.#handleSubscription, parent);
+            fastify.get(CHANGES_PATH_PATTERN, { websocket: true }, this.#subscribe);
+            fastify.get(SNAPSHOT_PATH_PATTERN, { websocket: true }, this.#reserveSnapshot);
+            installWebSocketReceiver(lc, fastify.websocketServer, this.#receiveWebsocket, parent);
         });
     }
-    setDelegate(delegate) {
-        assert(this.#delegate === null, 'delegate already set');
-        this.#delegate = delegate;
+    setDelegates(changeStreamer, backupMonitor) {
+        assert(this.#changeStreamer === null, 'delegate already set');
+        this.#changeStreamer = changeStreamer;
+        this.#backupMonitor = backupMonitor;
     }
     // Only respond to LB health checks (on "/keepalive") if the delegate is
     // initialized. Container health checks (on "/") are always acknowledged.
     _respondToKeepalive() {
-        return this.#delegate !== null;
+        return this.#changeStreamer !== null;
     }
-    #mustDelegate() {
-        return must(this.#delegate, 'received request before change-streamer is ready');
+    #getChangeStreamer() {
+        return must(this.#changeStreamer, 'received request before change-streamer is ready');
     }
-    #subscribe = async (ws, req) => {
-        let ctx;
+    #getBackupMonitor() {
+        return must(this.#backupMonitor, 'received request before change-streamer is ready');
+    }
+    // Called when receiving a web socket via the main dispatcher handoff.
+    #receiveWebsocket = (ws, action, msg) => {
+        switch (action) {
+            case 'snapshot':
+                return this.#reserveSnapshot(ws, msg);
+            case 'changes':
+                return this.#subscribe(ws, msg);
+            default:
+                closeWithError(this._lc, ws, `invalid action "${action}" received in handoff`);
+                return;
+        }
+    };
+    #reserveSnapshot = (ws, req) => {
         try {
-            ctx = getSubscriberContext(req);
+            const url = new URL(req.url ?? '', req.headers.origin ?? 'http://localhost');
+            const taskID = url.searchParams.get('taskID');
+            if (!taskID) {
+                throw new Error('Missing taskID in snapshot request');
+            }
+            const downstream = this.#getBackupMonitor().startSnapshotReservation(taskID);
+            void streamOut(this._lc, downstream, ws);
         }
         catch (err) {
             closeWithError(this._lc, ws, err, PROTOCOL_ERROR);
-            return;
         }
-        await this.#handleSubscription(ws, ctx);
     };
-    #handleSubscription = async (ws, ctx) => {
-        const downstream = await this.#mustDelegate().subscribe(ctx);
-        await streamOut(this._lc, downstream, ws);
+    #subscribe = async (ws, req) => {
+        try {
+            const ctx = getSubscriberContext(req);
+            const downstream = await this.#getChangeStreamer().subscribe(ctx);
+            if (ctx.initial && ctx.taskID && this.#backupMonitor) {
+                // Now that the change-streamer knows about the subscriber and watermark,
+                // end the reservation to safely resume scheduling cleanup.
+                this.#backupMonitor.endReservation(ctx.taskID);
+            }
+            void streamOut(this._lc, downstream, ws);
+        }
+        catch (err) {
+            closeWithError(this._lc, ws, err, PROTOCOL_ERROR);
+        }
     };
 }
 export class ChangeStreamerHttpClient {
@@ -65,15 +97,31 @@ export class ChangeStreamerHttpClient {
     constructor(lc, shardID, changeDB) {
         this.#lc = lc;
         this.#shardID = shardID;
-        this.#changeDB = changeDB;
+        // Create a pg client with a single short-lived connection for the purpose
+        // of change-streamer discovery (i.e. ChangeDB as DNS).
+        this.#changeDB = pgClient(lc, changeDB, {
+            max: 1,
+            ['idle_timeout']: 15,
+            connection: { ['application_name']: 'change-streamer-discovery' },
+        });
     }
-    async subscribe(ctx) {
+    async #resolveChangeStreamer(path) {
         const address = await discoverChangeStreamerAddress(this.#shardID, this.#changeDB);
         if (!address) {
             throw new Error(`no change-streamer is running`);
         }
-        const uri = new URL(CHANGES_PATH, `http://${address}/`);
+        const uri = new URL(path, `http://${address}/`);
         this.#lc.info?.(`connecting to change-streamer@${uri}`);
+        return uri;
+    }
+    async reserveSnapshot(taskID) {
+        const uri = await this.#resolveChangeStreamer(SNAPSHOT_PATH);
+        const params = new URLSearchParams({ taskID });
+        const ws = new WebSocket(uri + `?${params.toString()}`);
+        return streamIn(this.#lc, ws, snapshotMessageSchema);
+    }
+    async subscribe(ctx) {
+        const uri = await this.#resolveChangeStreamer(CHANGES_PATH);
         const params = getParams(ctx);
         const ws = new WebSocket(uri + `?${params.toString()}`);
         return streamIn(this.#lc, ws, downstreamSchema);
@@ -86,6 +134,7 @@ export function getSubscriberContext(req) {
     return {
         protocolVersion,
         id: params.get('id', true),
+        taskID: params.get('taskID', false),
         mode: params.get('mode', false) === 'backup' ? 'backup' : 'serving',
         replicaVersion: params.get('replicaVersion', true),
         watermark: params.get('watermark', true),
@@ -113,6 +162,7 @@ function getParams(ctx) {
     assert(protocolVersion === PROTOCOL_VERSION, `replicator should be setting protocolVersion to ${PROTOCOL_VERSION}`);
     return new URLSearchParams({
         ...stringParams,
+        taskID: ctx.taskID ? ctx.taskID : '',
         initial: ctx.initial ? 'true' : 'false',
     });
 }

package/out/zero-cache/src/services/change-streamer/change-streamer-http.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-http.js","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAC,eAAe,EAAC,MAAM,WAAW,CAAC;AAC1C,OAAO,SAAS,MAAM,IAAI,CAAC;AAC3B,OAAO,EAAC,MAAM,EAAC,MAAM,mCAAmC,CAAC;AACzD,OAAO,EAAC,IAAI,EAAC,MAAM,gCAAgC,CAAC;AAEpD,OAAO,EAAa,MAAM,0BAA0B,CAAC;AAErD,OAAO,EAAC,QAAQ,EAAE,SAAS,EAAc,MAAM,wBAAwB,CAAC;AACxE,OAAO,EAAC,SAAS,EAAC,MAAM,2BAA2B,CAAC;AACpD,OAAO,EAAC,wBAAwB,EAAC,MAAM,kCAAkC,CAAC;AAC1E,OAAO,EAAC,cAAc,EAAE,cAAc,EAAC,MAAM,mBAAmB,CAAC;AACjE,OAAO,EAAC,WAAW,EAAe,MAAM,oBAAoB,CAAC;AAC7D,OAAO,EACL,gBAAgB,EAChB,gBAAgB,GAIjB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAC,6BAA6B,EAAC,MAAM,oBAAoB,CAAC;AAEjE,MAAM,8BAA8B,GAAG,CAAC,CAAC;AAEzC,MAAM,mBAAmB,GAAG,+BAA+B,CAAC;AAC5D,MAAM,mBAAmB,GAAG,UAAU,GAAG,mBAAmB,CAAC;AAC7D,MAAM,UAAU,GACd,8DAA8D,CAAC;AAEjE,MAAM,YAAY,GAAG,iBAAiB,gBAAgB,UAAU,CAAC;AAEjE,MAAM,OAAO,wBAAyB,SAAQ,WAAW;IAC9C,EAAE,GAAG,6BAA6B,CAAC;IAC5C,SAAS,GAA0B,IAAI,CAAC;IAExC,YAAY,EAAc,EAAE,IAAa,EAAE,MAAc;QACvD,KAAK,CAAC,6BAA6B,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAC,OAAO,EAAC,EAAE;YAC7D,MAAM,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;YAElC,gEAAgE;YAChE,0BAA0B;YAC1B,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,EAAC,SAAS,EAAE,IAAI,EAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YACrE,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,EAAC,SAAS,EAAE,IAAI,EAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YAErE,wBAAwB,CACtB,EAAE,EACF,OAAO,CAAC,eAAe,EACvB,IAAI,CAAC,mBAAmB,EACxB,MAAM,CACP,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED,WAAW,CAAC,QAAwB;QAClC,MAAM,CAAC,IAAI,CAAC,SAAS,KAAK,IAAI,EAAE,sBAAsB,CAAC,CAAC;QACxD,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;IAC5B,CAAC;IAED,wEAAwE;IACxE,yEAAyE;IAC/D,mBAAmB;QAC3B,OAAO,IAAI,CAAC,SAAS,KAAK,IAAI,CAAC;IACjC,CAAC;IAED,aAAa;QACX,OAAO,IAAI,CACT,IAAI,CAAC,SAAS,EACd,kDAAkD,CACnD,CAAC;IACJ,CAAC;IAEQ,UAAU,GAAG,KAAK,EAAE,EAAa,EAAE,GAAmB,EAAE,EAAE;QACjE,IAAI,GAAsB,CAAC;QAC3B,IAAI,CAAC;YACH,GAAG,GAAG,oBAAoB,CAAC,GAAG,CAAC,CAAC;QAClC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,cAAc,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,cAAc,CAAC,CAAC;YAClD,OAAO;QACT,CAAC;QACD,MAAM,IAAI,CAAC,mBAAmB,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IAC1C,CAAC,CAAC;IAEO,mBAAmB,GAAG,KAAK,EAClC,EAAa,EACb,GAAsB,EACtB,EAAE;QACF,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAC7D,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,EAAE,EAAE,CAAC,CAAC;IAC5C,CAAC,CAAC;CACH;AAED,MAAM,OAAO,wBAAwB;IAC1B,GAAG,CAAa;IAChB,QAAQ,CAAU;IAClB,SAAS,CAAa;IAE/B,YAAY,EAAc,EAAE,OAAgB,EAAE,QAAoB;QAChE,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC;QACd,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QACxB,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,GAAsB;QACpC,MAAM,OAAO,GAAG,MAAM,6BAA6B,CACjD,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,SAAS,CACf,CAAC;QACF,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACnD,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,YAAY,EAAE,UAAU,OAAO,GAAG,CAAC,CAAC;QAExD,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,iCAAiC,GAAG,EAAE,CAAC,CAAC;QACxD,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,EAAE,GAAG,IAAI,SAAS,CAAC,GAAG,GAAG,IAAI,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAExD,OAAO,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAClD,CAAC;CACF;AAID,MAAM,UAAU,oBAAoB,CAAC,GAAmB;IACtD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,EAAE,EAAE,GAAG,CAAC,OAAO,CAAC,MAAM,IAAI,kBAAkB,CAAC,CAAC;IAC7E,MAAM,eAAe,GAAG,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAChD,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,GAAG,CAAC,CAAC;IAElC,OAAO;QACL,eAAe;QACf,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC;QAC1B,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;QACnE,cAAc,EAAE,MAAM,CAAC,GAAG,CAAC,gBAAgB,EAAE,IAAI,CAAC;QAClD,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,CAAC;QACxC,OAAO,EAAE,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC;KACtC,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,QAAgB;IACjC,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACxC,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,iBAAiB,QAAQ,EAAE,CAAC,CAAC;IAC/C,CAAC;IACD,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACxC,IACE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QACf,CAAC,GAAG,gBAAgB;QACpB,CAAC,GAAG,8BAA8B,EAClC,CAAC;QACD,MAAM,IAAI,KAAK,CACb,sCAAsC,CAAC,IAAI;YACzC,0BAA0B,8BAA8B,SAAS,gBAAgB,GAAG,CACvF,CAAC;IACJ,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,6DAA6D;AAC7D,SAAS,SAAS,CAAC,GAAsB;IACvC,2DAA2D;IAC3D,MAAM,EAAC,eAAe,EAAE,GAAG,YAAY,EAAC,GAAG,GAAG,CAAC;IAC/C,MAAM,CACJ,eAAe,KAAK,gBAAgB,EACpC,mDAAmD,gBAAgB,EAAE,CACtE,CAAC;IACF,OAAO,IAAI,eAAe,CAAC;QACzB,GAAG,YAAY;QACf,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO;KACxC,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"change-streamer-http.js","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAC,eAAe,EAAC,MAAM,WAAW,CAAC;AAC1C,OAAO,SAAS,MAAM,IAAI,CAAC;AAC3B,OAAO,EAAC,MAAM,EAAC,MAAM,mCAAmC,CAAC;AACzD,OAAO,EAAC,IAAI,EAAC,MAAM,gCAAgC,CAAC;AAEpD,OAAO,EAAC,QAAQ,EAAkB,MAAM,mBAAmB,CAAC;AAC5D,OAAO,EAAa,MAAM,0BAA0B,CAAC;AAErD,OAAO,EAAC,QAAQ,EAAE,SAAS,EAAc,MAAM,wBAAwB,CAAC;AACxE,OAAO,EAAC,SAAS,EAAC,MAAM,2BAA2B,CAAC;AACpD,OAAO,EAAC,wBAAwB,EAAC,MAAM,kCAAkC,CAAC;AAC1E,OAAO,EAAC,cAAc,EAAE,cAAc,EAAC,MAAM,mBAAmB,CAAC;AACjE,OAAO,EAAC,WAAW,EAAe,MAAM,oBAAoB,CAAC;AAE7D,OAAO,EACL,gBAAgB,EAChB,gBAAgB,GAIjB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAC,6BAA6B,EAAC,MAAM,oBAAoB,CAAC;AACjE,OAAO,EAAC,qBAAqB,EAAuB,MAAM,eAAe,CAAC;AAE1E,MAAM,8BAA8B,GAAG,CAAC,CAAC;AAEzC,MAAM,qBAAqB,GAAG,gCAAgC,CAAC;AAC/D,MAAM,oBAAoB,GAAG,+BAA+B,CAAC;AAC7D,MAAM,UAAU,GAAG,sDAAsD,CAAC;AAE1E,MAAM,aAAa,GAAG,iBAAiB,gBAAgB,WAAW,CAAC;AACnE,MAAM,YAAY,GAAG,iBAAiB,gBAAgB,UAAU,CAAC;AAEjE,MAAM,OAAO,wBAAyB,SAAQ,WAAW;IAC9C,EAAE,GAAG,6BAA6B,CAAC;IAC5C,eAAe,GAA0B,IAAI,CAAC;IAC9C,cAAc,GAAyB,IAAI,CAAC;IAE5C,YAAY,EAAc,EAAE,IAAa,EAAE,MAAc;QACvD,KAAK,CAAC,6BAA6B,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAC,OAAO,EAAC,EAAE;YAC7D,MAAM,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;YAElC,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAAE,EAAC,SAAS,EAAE,IAAI,EAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YACtE,OAAO,CAAC,GAAG,CACT,qBAAqB,EACrB,EAAC,SAAS,EAAE,IAAI,EAAC,EACjB,IAAI,CAAC,gBAAgB,CACtB,CAAC;YAEF,wBAAwB,CACtB,EAAE,EACF,OAAO,CAAC,eAAe,EACvB,IAAI,CAAC,iBAAiB,EACtB,MAAM,CACP,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED,YAAY,CACV,cAA8B,EAC9B,aAAmC;QAEnC,MAAM,CAAC,IAAI,CAAC,eAAe,KAAK,IAAI,EAAE,sBAAsB,CAAC,CAAC;QAC9D,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC;QACtC,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;IACtC,CAAC;IAED,wEAAwE;IACxE,yEAAyE;IAC/D,mBAAmB;QAC3B,OAAO,IAAI,CAAC,eAAe,KAAK,IAAI,CAAC;IACvC,CAAC;IAED,kBAAkB;QAChB,OAAO,IAAI,CACT,IAAI,CAAC,eAAe,EACpB,kDAAkD,CACnD,CAAC;IACJ,CAAC;IAED,iBAAiB;QACf,OAAO,IAAI,CACT,IAAI,CAAC,cAAc,EACnB,kDAAkD,CACnD,CAAC;IACJ,CAAC;IAED,sEAAsE;IAC7D,iBAAiB,GAAG,CAC3B,EAAa,EACb,MAA8B,EAC9B,GAA0B,EAC1B,EAAE;QACF,QAAQ,MAAM,EAAE,CAAC;YACf,KAAK,UAAU;gBACb,OAAO,IAAI,CAAC,gBAAgB,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;YACxC,KAAK,SAAS;gBACZ,OAAO,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;YAClC;gBACE,cAAc,CACZ,IAAI,CAAC,GAAG,EACR,EAAE,EACF,mBAAmB,MAAM,uBAAuB,CACjD,CAAC;gBACF,OAAO;QACX,CAAC;IACH,CAAC,CAAC;IAEO,gBAAgB,GAAG,CAAC,EAAa,EAAE,GAAmB,EAAE,EAAE;QACjE,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,GAAG,CACjB,GAAG,CAAC,GAAG,IAAI,EAAE,EACb,GAAG,CAAC,OAAO,CAAC,MAAM,IAAI,kBAAkB,CACzC,CAAC;YACF,MAAM,MAAM,GAAG,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAC9C,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;YACxD,CAAC;YACD,MAAM,UAAU,GACd,IAAI,CAAC,iBAAiB,EAAE,CAAC,wBAAwB,CAAC,MAAM,CAAC,CAAC;YAC5D,KAAK,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,EAAE,EAAE,CAAC,CAAC;QAC3C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,cAAc,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,cAAc,CAAC,CAAC;QACpD,CAAC;IACH,CAAC,CAAC;IAEO,UAAU,GAAG,KAAK,EAAE,EAAa,EAAE,GAAmB,EAAE,EAAE;QACjE,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,oBAAoB,CAAC,GAAG,CAAC,CAAC;YAEtC,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YAClE,IAAI,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,MAAM,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;gBACrD,yEAAyE;gBACzE,2DAA2D;gBAC3D,IAAI,CAAC,cAAc,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACjD,CAAC;YACD,KAAK,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,EAAE,EAAE,CAAC,CAAC;QAC3C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,cAAc,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,cAAc,CAAC,CAAC;QACpD,CAAC;IACH,CAAC,CAAC;CACH;AAED,MAAM,OAAO,wBAAwB;IAC1B,GAAG,CAAa;IAChB,QAAQ,CAAU;IAClB,SAAS,CAAa;IAE/B,YAAY,EAAc,EAAE,OAAgB,EAAE,QAAgB;QAC5D,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC;QACd,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QACxB,0EAA0E;QAC1E,uDAAuD;QACvD,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC,EAAE,EAAE,QAAQ,EAAE;YACtC,GAAG,EAAE,CAAC;YACN,CAAC,cAAc,CAAC,EAAE,EAAE;YACpB,UAAU,EAAE,EAAC,CAAC,kBAAkB,CAAC,EAAE,2BAA2B,EAAC;SAChE,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,sBAAsB,CAAC,IAAY;QACvC,MAAM,OAAO,GAAG,MAAM,6BAA6B,CACjD,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,SAAS,CACf,CAAC;QACF,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACnD,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,UAAU,OAAO,GAAG,CAAC,CAAC;QAChD,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,iCAAiC,GAAG,EAAE,CAAC,CAAC;QACxD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,MAAc;QAClC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAAC,aAAa,CAAC,CAAC;QAE7D,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC,EAAC,MAAM,EAAC,CAAC,CAAC;QAC7C,MAAM,EAAE,GAAG,IAAI,SAAS,CAAC,GAAG,GAAG,IAAI,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAExD,OAAO,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,EAAE,qBAAqB,CAAC,CAAC;IACvD,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,GAAsB;QACpC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAAC,YAAY,CAAC,CAAC;QAE5D,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,EAAE,GAAG,IAAI,SAAS,CAAC,GAAG,GAAG,IAAI,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAExD,OAAO,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAClD,CAAC;CACF;AAID,MAAM,UAAU,oBAAoB,CAAC,GAAmB;IACtD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,EAAE,EAAE,GAAG,CAAC,OAAO,CAAC,MAAM,IAAI,kBAAkB,CAAC,CAAC;IAC7E,MAAM,eAAe,GAAG,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAChD,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,GAAG,CAAC,CAAC;IAElC,OAAO;QACL,eAAe;QACf,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC;QAC1B,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC;QACnC,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;QACnE,cAAc,EAAE,MAAM,CAAC,GAAG,CAAC,gBAAgB,EAAE,IAAI,CAAC;QAClD,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,CAAC;QACxC,OAAO,EAAE,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC;KACtC,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,QAAgB;IACjC,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACxC,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,iBAAiB,QAAQ,EAAE,CAAC,CAAC;IAC/C,CAAC;IACD,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACxC,IACE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QACf,CAAC,GAAG,gBAAgB;QACpB,CAAC,GAAG,8BAA8B,EAClC,CAAC;QACD,MAAM,IAAI,KAAK,CACb,sCAAsC,CAAC,IAAI;YACzC,0BAA0B,8BAA8B,SAAS,gBAAgB,GAAG,CACvF,CAAC;IACJ,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,6DAA6D;AAC7D,SAAS,SAAS,CAAC,GAAsB;IACvC,2DAA2D;IAC3D,MAAM,EAAC,eAAe,EAAE,GAAG,YAAY,EAAC,GAAG,GAAG,CAAC;IAC/C,MAAM,CACJ,eAAe,KAAK,gBAAgB,EACpC,mDAAmD,gBAAgB,EAAE,CACtE,CAAC;IACF,OAAO,IAAI,eAAe,CAAC;QACzB,GAAG,YAAY;QACf,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;QACpC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO;KACxC,CAAC,CAAC;AACL,CAAC"}

package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts

@@ -45,12 +45,17 @@ export interface ChangeStreamer {
      */
     subscribe(ctx: SubscriberContext): Promise<Source<Downstream>>;
 }
-export declare const PROTOCOL_VERSION = 2;
+export declare const PROTOCOL_VERSION = 3;
 export type SubscriberContext = {
     /**
      * The supported change-streamer protocol version.
      */
     protocolVersion: number;
+    /**
+     * Task ID. This is used to link the request with a preceding snapshot
+     * reservation.
+     */
+    taskID: string | null;
     /**
      * Subscriber id. This is only used for debugging.
      */