@rocicorp/zero 1.10.0-canary.0 → 1.10.0-canary.1

package/out/zero-cache/src/services/change-streamer/change-log-transaction-hash.js

@@ -0,0 +1,39 @@
+import { createHash } from "node:crypto";
+//#region ../zero-cache/src/services/change-streamer/change-log-transaction-hash.ts
+/**
+* Computes a stable digest over the fields shared by the Postgres and SQLite
+* change logs. Values are length-prefixed so different field boundaries cannot
+* produce the same byte stream.
+*/
+var ChangeLogTransactionHasher = class {
+	#hash;
+	constructor() {
+		this.#hash = createHash("sha256");
+		writeString(this.#hash, "zero-change-log-transaction-v1");
+	}
+	add(row) {
+		writeString(this.#hash, row.watermark);
+		writeString(this.#hash, String(row.pos));
+		writeString(this.#hash, row.tag);
+		writeString(this.#hash, row.change);
+		if (row.precommit === null) this.#hash.update(Uint8Array.of(0));
+		else {
+			this.#hash.update(Uint8Array.of(1));
+			writeString(this.#hash, row.precommit);
+		}
+	}
+	digest() {
+		return this.#hash.digest("hex");
+	}
+};
+function writeString(hash, value) {
+	const bytes = Buffer.from(value);
+	const length = Buffer.allocUnsafe(4);
+	length.writeUInt32BE(bytes.byteLength);
+	hash.update(length);
+	hash.update(bytes);
+}
+//#endregion
+export { ChangeLogTransactionHasher };
+
+//# sourceMappingURL=change-log-transaction-hash.js.map

package/out/zero-cache/src/services/change-streamer/change-log-transaction-hash.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"change-log-transaction-hash.js","names":["#hash"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-log-transaction-hash.ts"],"sourcesContent":["import {createHash, type Hash} from 'node:crypto';\n\nexport type ChangeLogHashRow = {\n  readonly watermark: string;\n  readonly pos: number;\n  readonly tag: string;\n  readonly change: string;\n  readonly precommit: string | null;\n};\n\n/**\n * Computes a stable digest over the fields shared by the Postgres and SQLite\n * change logs. Values are length-prefixed so different field boundaries cannot\n * produce the same byte stream.\n */\nexport class ChangeLogTransactionHasher {\n  readonly #hash: Hash;\n\n  constructor() {\n    this.#hash = createHash('sha256');\n    writeString(this.#hash, 'zero-change-log-transaction-v1');\n  }\n\n  add(row: ChangeLogHashRow): void {\n    writeString(this.#hash, row.watermark);\n    writeString(this.#hash, String(row.pos));\n    writeString(this.#hash, row.tag);\n    writeString(this.#hash, row.change);\n    if (row.precommit === null) {\n      this.#hash.update(Uint8Array.of(0));\n    } else {\n      this.#hash.update(Uint8Array.of(1));\n      writeString(this.#hash, row.precommit);\n    }\n  }\n\n  digest(): string {\n    return this.#hash.digest('hex');\n  }\n}\n\nfunction writeString(hash: Hash, value: string): void {\n  const bytes = Buffer.from(value);\n  const length = Buffer.allocUnsafe(4);\n  length.writeUInt32BE(bytes.byteLength);\n  hash.update(length);\n  hash.update(bytes);\n}\n"],"mappings":";;;;;;;AAeA,IAAa,6BAAb,MAAwC;CACtC;CAEA,cAAc;EACZ,KAAKA,QAAQ,WAAW,QAAQ;EAChC,YAAY,KAAKA,OAAO,gCAAgC;CAC1D;CAEA,IAAI,KAA6B;EAC/B,YAAY,KAAKA,OAAO,IAAI,SAAS;EACrC,YAAY,KAAKA,OAAO,OAAO,IAAI,GAAG,CAAC;EACvC,YAAY,KAAKA,OAAO,IAAI,GAAG;EAC/B,YAAY,KAAKA,OAAO,IAAI,MAAM;EAClC,IAAI,IAAI,cAAc,MACpB,KAAKA,MAAM,OAAO,WAAW,GAAG,CAAC,CAAC;OAC7B;GACL,KAAKA,MAAM,OAAO,WAAW,GAAG,CAAC,CAAC;GAClC,YAAY,KAAKA,OAAO,IAAI,SAAS;EACvC;CACF;CAEA,SAAiB;EACf,OAAO,KAAKA,MAAM,OAAO,KAAK;CAChC;AACF;AAEA,SAAS,YAAY,MAAY,OAAqB;CACpD,MAAM,QAAQ,OAAO,KAAK,KAAK;CAC/B,MAAM,SAAS,OAAO,YAAY,CAAC;CACnC,OAAO,cAAc,MAAM,UAAU;CACrC,KAAK,OAAO,MAAM;CAClB,KAAK,OAAO,KAAK;AACnB"}

package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts

@@ -4,8 +4,7 @@ import { type Worker } from '../../types/processes.ts';
 import { type ShardID } from '../../types/shards.ts';
 import { type Source } from '../../types/streams.ts';
 import { HttpService } from '../http-service.ts';
-import type { BackupMonitor } from './backup-monitor.ts';
-import { type ChangeStreamer, type ChangeStreamerService, type Downstream, type SubscriberContext } from './change-streamer.ts';
+import { type ChangeStreamer, type ChangeStreamerService, type SerializedDownstream, type SubscriberContext } from './change-streamer.ts';
 import { type SnapshotMessage } from './snapshot.ts';
 type Options = {
     port: number;
@@ -15,7 +14,7 @@ type Options = {
 export declare class ChangeStreamerHttpServer extends HttpService {
     #private;
     readonly id = "change-streamer-http-server";
-    constructor(lc: LogContext, opts: Options, parent: Worker, changeStreamer: ChangeStreamerService, backupMonitor: BackupMonitor | null);
+    constructor(lc: LogContext, opts: Options, parent: Worker, changeStreamer: ChangeStreamerService);
     protected _onStart(): void;
     protected _onStop(): Promise<void>;
 }
@@ -23,7 +22,7 @@ export declare class ChangeStreamerHttpClient implements ChangeStreamer {
     #private;
     constructor(lc: LogContext, shardID: ShardID, changeDB: string, changeStreamerURI: string | undefined);
     reserveSnapshot(taskID: string): Promise<Source<SnapshotMessage>>;
-    subscribe(ctx: SubscriberContext): Promise<Source<Downstream>>;
+    subscribe(ctx: SubscriberContext): Promise<Source<SerializedDownstream>>;
 }
 type RequestHeaders = Pick<IncomingMessage, 'url' | 'headers'>;
 export declare function getSubscriberContext(req: RequestHeaders): SubscriberContext;

package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-http.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,WAAW,CAAC;AAE/C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAMjD,OAAO,EAAC,KAAK,MAAM,EAAC,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAC,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAIL,KAAK,MAAM,EACZ,MAAM,wBAAwB,CAAC;AAIhC,OAAO,EAAC,WAAW,EAAC,MAAM,oBAAoB,CAAC;AAC/C,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAGL,KAAK,cAAc,EACnB,KAAK,qBAAqB,EAC1B,KAAK,UAAU,EACf,KAAK,iBAAiB,EACvB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAwB,KAAK,eAAe,EAAC,MAAM,eAAe,CAAC;AAW1E,KAAK,OAAO,GAAG;IACb,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,qBAAa,wBAAyB,SAAQ,WAAW;;IACvD,QAAQ,CAAC,EAAE,iCAAiC;gBAO1C,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,OAAO,EACb,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,qBAAqB,EACrC,aAAa,EAAE,aAAa,GAAG,IAAI;cA4GlB,QAAQ,IAAI,IAAI;cAWV,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAKlD;AAED,qBAAa,wBAAyB,YAAW,cAAc;;gBAO3D,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,MAAM,EAChB,iBAAiB,EAAE,MAAM,GAAG,SAAS;IA8BjC,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IASjE,SAAS,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;CAQrE;AAED,KAAK,cAAc,GAAG,IAAI,CAAC,eAAe,EAAE,KAAK,GAAG,SAAS,CAAC,CAAC;AAE/D,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,cAAc,GAAG,iBAAiB,CAc3E"}
+{"version":3,"file":"change-streamer-http.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,WAAW,CAAC;AAE/C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,EAAC,KAAK,MAAM,EAAC,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAC,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAKL,KAAK,MAAM,EACZ,MAAM,wBAAwB,CAAC;AAIhC,OAAO,EAAC,WAAW,EAAC,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAGL,KAAK,cAAc,EACnB,KAAK,qBAAqB,EAC1B,KAAK,oBAAoB,EACzB,KAAK,iBAAiB,EACvB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAwB,KAAK,eAAe,EAAC,MAAM,eAAe,CAAC;AAW1E,KAAK,OAAO,GAAG;IACb,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,qBAAa,wBAAyB,SAAQ,WAAW;;IACvD,QAAQ,CAAC,EAAE,iCAAiC;gBAM1C,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,OAAO,EACb,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,qBAAqB;cAgGpB,QAAQ,IAAI,IAAI;cAWV,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAKlD;AAED,qBAAa,wBAAyB,YAAW,cAAc;;gBAO3D,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,MAAM,EAChB,iBAAiB,EAAE,MAAM,GAAG,SAAS;IA8BjC,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IASjE,SAAS,CACb,GAAG,EAAE,iBAAiB,GACrB,OAAO,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;CAQzC;AAED,KAAK,cAAc,GAAG,IAAI,CAAC,eAAe,EAAE,KAAK,GAAG,SAAS,CAAC,CAAC;AAE/D,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,cAAc,GAAG,iBAAiB,CAkB3E"}

package/out/zero-cache/src/services/change-streamer/change-streamer-http.js

@@ -1,19 +1,18 @@
 import { assert } from "../../../../shared/src/asserts.js";
-import { must } from "../../../../shared/src/must.js";
 import "../../types/processes.js";
 import { pgClient } from "../../types/pg.js";
 import { HttpService } from "../http-service.js";
 import { PROTOCOL_ERROR, closeWithError } from "../../types/ws.js";
 import { installWebSocketReceiver } from "../../types/websocket-handoff.js";
+import { streamIn, streamInStringified, streamOut, streamOutStringified } from "../../types/streams.js";
 import { discoverChangeStreamerAddress } from "./schema/tables.js";
-import { streamIn, streamOut, streamOutStringified } from "../../types/streams.js";
 import { URLParams } from "../../types/url-params.js";
 import { downstreamSchema } from "./change-streamer.js";
 import { snapshotMessageSchema } from "./snapshot.js";
 import WebSocket$1 from "ws";
 import websocket from "@fastify/websocket";
 //#region ../zero-cache/src/services/change-streamer/change-streamer-http.ts
-var MIN_SUPPORTED_PROTOCOL_VERSION = 1;
+var MIN_SUPPORTED_PROTOCOL_VERSION = 4;
 var SNAPSHOT_PATH_PATTERN = "/replication/:version/snapshot";
 var CHANGES_PATH_PATTERN = "/replication/:version/changes";
 var PATH_REGEX = /\/replication\/v(?<version>\d+)\/(changes|snapshot)$/;
@@ -24,8 +23,7 @@ var ChangeStreamerHttpServer = class extends HttpService {
 	#lc;
 	#opts;
 	#changeStreamer;
-	#backupMonitor;
-	constructor(lc, opts, parent, changeStreamer, backupMonitor) {
+	constructor(lc, opts, parent, changeStreamer) {
 		super("change-streamer-http-server", lc, opts, async (fastify) => {
 			await fastify.register(websocket);
 			fastify.get(CHANGES_PATH_PATTERN, { websocket: true }, this.#subscribe);
@@ -35,10 +33,6 @@ var ChangeStreamerHttpServer = class extends HttpService {
 		this.#lc = lc;
 		this.#opts = opts;
 		this.#changeStreamer = changeStreamer;
-		this.#backupMonitor = backupMonitor;
-	}
-	#getBackupMonitor() {
-		return must(this.#backupMonitor, "replication-manager is not configured with a ZERO_LITESTREAM_BACKUP_URL");
 	}
 	#receiveWebsocket = (ws, action, msg) => {
 		switch (action) {
@@ -49,13 +43,14 @@ var ChangeStreamerHttpServer = class extends HttpService {
 				return;
 		}
 	};
-	#reserveSnapshot = (ws, req) => {
+	#reserveSnapshot = async (ws, req) => {
+		this.#ensureChangeStreamerStarted("incoming snapshot reservation");
 		try {
 			const url = new URL(req.url ?? "", req.headers.origin ?? "http://localhost");
 			checkProtocolVersion(url.pathname);
 			const taskID = url.searchParams.get("taskID");
 			if (!taskID) throw new Error("Missing taskID in snapshot request");
-			const downstream = this.#getBackupMonitor().startSnapshotReservation(taskID);
+			const downstream = await this.#changeStreamer.startSnapshotReservation(taskID);
 			streamOut(this._lc, downstream, ws);
 		} catch (err) {
 			closeWithError(this._lc, ws, err, PROTOCOL_ERROR);
@@ -66,7 +61,6 @@ var ChangeStreamerHttpServer = class extends HttpService {
 			const ctx = getSubscriberContext(req);
 			if (ctx.mode === "serving") this.#ensureChangeStreamerStarted("incoming subscription");
 			const downstream = await this.#changeStreamer.subscribe(ctx);
-			if (ctx.initial && ctx.taskID && this.#backupMonitor) this.#backupMonitor.endReservation(ctx.taskID);
 			streamOutStringified(this._lc, downstream, ws);
 		} catch (err) {
 			closeWithError(this._lc, ws, err, PROTOCOL_ERROR);
@@ -119,7 +113,7 @@ var ChangeStreamerHttpClient = class {
 	}
 	async subscribe(ctx) {
 		const ws = new WebSocket$1(await this.#resolveChangeStreamer(CHANGES_PATH) + `?${getParams(ctx).toString()}`);
-		return streamIn(this.#lc, ws, downstreamSchema);
+		return streamInStringified(this.#lc, ws, downstreamSchema);
 	}
 };
 function getSubscriberContext(req) {
@@ -129,11 +123,12 @@ function getSubscriberContext(req) {
 	return {
 		protocolVersion,
 		id: params.get("id", true),
-		taskID: params.get("taskID", false),
+		taskID: params.get("taskID", true),
 		mode: params.get("mode", false) === "backup" ? "backup" : "serving",
 		replicaVersion: params.get("replicaVersion", true),
 		watermark: params.get("watermark", true),
-		initial: params.getBoolean("initial")
+		initial: params.getBoolean("initial"),
+		logsChangeStream: params.getBoolean("logsChangeStream")
 	};
 }
 function checkProtocolVersion(pathname) {
@@ -148,8 +143,8 @@ function getParams(ctx) {
 	assert(protocolVersion === 6, `replicator should be setting protocolVersion to 6`);
 	return new URLSearchParams({
 		...stringParams,
-		taskID: ctx.taskID ? ctx.taskID : "",
-		initial: ctx.initial ? "true" : "false"
+		initial: ctx.initial ? "true" : "false",
+		logsChangeStream: ctx.logsChangeStream ? "true" : "false"
 	});
 }
 //#endregion

package/out/zero-cache/src/services/change-streamer/change-streamer-http.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-http.js","names":["#lc","#opts","#changeStreamer","#backupMonitor","#subscribe","#reserveSnapshot","#receiveWebsocket","#getBackupMonitor","#ensureChangeStreamerStarted","#changeStreamerStarted","#shardID","#changeDB","#changeStreamerURI","#resolveChangeStreamer"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport websocket from '@fastify/websocket';\nimport type {LogContext} from '@rocicorp/logger';\nimport WebSocket from 'ws';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport type {IncomingMessageSubset} from '../../types/http.ts';\nimport {pgClient, type PostgresDB} from '../../types/pg.ts';\nimport {type Worker} from '../../types/processes.ts';\nimport {type ShardID} from '../../types/shards.ts';\nimport {\n  streamIn,\n  streamOut,\n  streamOutStringified,\n  type Source,\n} from '../../types/streams.ts';\nimport {URLParams} from '../../types/url-params.ts';\nimport {installWebSocketReceiver} from '../../types/websocket-handoff.ts';\nimport {closeWithError, PROTOCOL_ERROR} from '../../types/ws.ts';\nimport {HttpService} from '../http-service.ts';\nimport type {BackupMonitor} from './backup-monitor.ts';\nimport {\n  downstreamSchema,\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type ChangeStreamerService,\n  type Downstream,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport {discoverChangeStreamerAddress} from './schema/tables.ts';\nimport {snapshotMessageSchema, type SnapshotMessage} from './snapshot.ts';\n\nconst MIN_SUPPORTED_PROTOCOL_VERSION = 1;\n\nconst SNAPSHOT_PATH_PATTERN = '/replication/:version/snapshot';\nconst CHANGES_PATH_PATTERN = '/replication/:version/changes';\nconst PATH_REGEX = /\\/replication\\/v(?<version>\\d+)\\/(changes|snapshot)$/;\n\nconst SNAPSHOT_PATH = `/replication/v${PROTOCOL_VERSION}/snapshot`;\nconst CHANGES_PATH = `/replication/v${PROTOCOL_VERSION}/changes`;\n\ntype Options = {\n  port: number;\n  keepaliveTimeoutMs: number | undefined;\n  startupDelayMs: number;\n};\n\nexport class ChangeStreamerHttpServer extends HttpService {\n  readonly id = 'change-streamer-http-server';\n  readonly #lc: LogContext;\n  readonly #opts: Options;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #backupMonitor: BackupMonitor | null;\n\n  constructor(\n    lc: LogContext,\n    opts: Options,\n    parent: Worker,\n    changeStreamer: ChangeStreamerService,\n    backupMonitor: BackupMonitor | null,\n  ) {\n    super('change-streamer-http-server', lc, opts, async fastify => {\n      await fastify.register(websocket);\n\n      fastify.get(CHANGES_PATH_PATTERN, {websocket: true}, this.#subscribe);\n      fastify.get(\n        SNAPSHOT_PATH_PATTERN,\n        {websocket: true},\n        this.#reserveSnapshot,\n      );\n\n      installWebSocketReceiver<'snapshot' | 'changes'>(\n        lc,\n        fastify.websocketServer,\n        this.#receiveWebsocket,\n        parent,\n      );\n    });\n\n    this.#lc = lc;\n    this.#opts = opts;\n    this.#changeStreamer = changeStreamer;\n    this.#backupMonitor = backupMonitor;\n  }\n\n  #getBackupMonitor() {\n    return must(\n      this.#backupMonitor,\n      'replication-manager is not configured with a ZERO_LITESTREAM_BACKUP_URL',\n    );\n  }\n\n  // Called when receiving a web socket via the main dispatcher handoff.\n  readonly #receiveWebsocket = (\n    ws: WebSocket,\n    action: 'changes' | 'snapshot',\n    msg: IncomingMessageSubset,\n  ) => {\n    switch (action) {\n      case 'snapshot':\n        return this.#reserveSnapshot(ws, msg);\n      case 'changes':\n        return this.#subscribe(ws, msg);\n      default:\n        closeWithError(\n          this._lc,\n          ws,\n          `invalid action \"${action}\" received in handoff`,\n        );\n        return;\n    }\n  };\n\n  readonly #reserveSnapshot = (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const url = new URL(\n        req.url ?? '',\n        req.headers.origin ?? 'http://localhost',\n      );\n      checkProtocolVersion(url.pathname);\n      const taskID = url.searchParams.get('taskID');\n      if (!taskID) {\n        throw new Error('Missing taskID in snapshot request');\n      }\n      const downstream =\n        this.#getBackupMonitor().startSnapshotReservation(taskID);\n      void streamOut(this._lc, downstream, ws);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  readonly #subscribe = async (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const ctx = getSubscriberContext(req);\n      if (ctx.mode === 'serving') {\n        this.#ensureChangeStreamerStarted('incoming subscription');\n      }\n\n      const downstream = await this.#changeStreamer.subscribe(ctx);\n      if (ctx.initial && ctx.taskID && this.#backupMonitor) {\n        // Now that the change-streamer knows about the subscriber and watermark,\n        // end the reservation to safely resume scheduling cleanup.\n        this.#backupMonitor.endReservation(ctx.taskID);\n      }\n      void streamOutStringified(this._lc, downstream, ws);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  #changeStreamerStarted = false;\n\n  #ensureChangeStreamerStarted(reason: string) {\n    if (!this.#changeStreamerStarted && this._state.shouldRun()) {\n      this.#lc.info?.(`starting ChangeStreamerService: ${reason}`);\n      void this.#changeStreamer\n        .run()\n        .catch(e =>\n          this.#lc.warn?.(`ChangeStreamerService ended with error`, e),\n        )\n        .finally(() => this.stop());\n\n      this.#changeStreamerStarted = true;\n    }\n  }\n\n  protected override _onStart(): void {\n    const {startupDelayMs} = this.#opts;\n    this._state.setTimeout(\n      () =>\n        this.#ensureChangeStreamerStarted(\n          `startup delay elapsed (${startupDelayMs} ms)`,\n        ),\n      startupDelayMs,\n    );\n  }\n\n  protected override async _onStop(): Promise<void> {\n    if (this.#changeStreamerStarted) {\n      await this.#changeStreamer.stop();\n    }\n  }\n}\n\nexport class ChangeStreamerHttpClient implements ChangeStreamer {\n  readonly #lc: LogContext;\n  readonly #shardID: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #changeStreamerURI: string | undefined;\n\n  constructor(\n    lc: LogContext,\n    shardID: ShardID,\n    changeDB: string,\n    changeStreamerURI: string | undefined,\n  ) {\n    this.#lc = lc;\n    this.#shardID = shardID;\n    // Create a pg client with a single short-lived connection for the purpose\n    // of change-streamer discovery (i.e. ChangeDB as DNS).\n    this.#changeDB = pgClient(lc, changeDB, 'change-streamer-discovery', {\n      max: 1,\n      ['idle_timeout']: 15,\n    });\n    this.#changeStreamerURI = changeStreamerURI;\n  }\n\n  async #resolveChangeStreamer(path: string) {\n    let baseURL = this.#changeStreamerURI;\n    if (!baseURL) {\n      const address = await discoverChangeStreamerAddress(\n        this.#shardID,\n        this.#changeDB,\n      );\n      if (!address) {\n        throw new Error(`no change-streamer is running`);\n      }\n      baseURL = address.includes('://') ? `${address}/` : `ws://${address}/`;\n    }\n    const uri = new URL(path, baseURL);\n    this.#lc.info?.(`connecting to change-streamer@${uri}`);\n    return uri;\n  }\n\n  async reserveSnapshot(taskID: string): Promise<Source<SnapshotMessage>> {\n    const uri = await this.#resolveChangeStreamer(SNAPSHOT_PATH);\n\n    const params = new URLSearchParams({taskID});\n    const ws = new WebSocket(uri + `?${params.toString()}`);\n\n    return streamIn(this.#lc, ws, snapshotMessageSchema);\n  }\n\n  async subscribe(ctx: SubscriberContext): Promise<Source<Downstream>> {\n    const uri = await this.#resolveChangeStreamer(CHANGES_PATH);\n\n    const params = getParams(ctx);\n    const ws = new WebSocket(uri + `?${params.toString()}`);\n\n    return streamIn(this.#lc, ws, downstreamSchema);\n  }\n}\n\ntype RequestHeaders = Pick<IncomingMessage, 'url' | 'headers'>;\n\nexport function getSubscriberContext(req: RequestHeaders): SubscriberContext {\n  const url = new URL(req.url ?? '', req.headers.origin ?? 'http://localhost');\n  const protocolVersion = checkProtocolVersion(url.pathname);\n  const params = new URLParams(url);\n\n  return {\n    protocolVersion,\n    id: params.get('id', true),\n    taskID: params.get('taskID', false),\n    mode: params.get('mode', false) === 'backup' ? 'backup' : 'serving',\n    replicaVersion: params.get('replicaVersion', true),\n    watermark: params.get('watermark', true),\n    initial: params.getBoolean('initial'),\n  };\n}\n\nfunction checkProtocolVersion(pathname: string): number {\n  const match = PATH_REGEX.exec(pathname);\n  if (!match) {\n    throw new Error(`invalid path: ${pathname}`);\n  }\n  const v = Number(match.groups?.version);\n  if (\n    Number.isNaN(v) ||\n    v > PROTOCOL_VERSION ||\n    v < MIN_SUPPORTED_PROTOCOL_VERSION\n  ) {\n    throw new Error(\n      `Cannot service client at protocol v${v}. ` +\n        `Supported protocols: [v${MIN_SUPPORTED_PROTOCOL_VERSION} ... v${PROTOCOL_VERSION}]`,\n    );\n  }\n  return v;\n}\n\n// This is called from the client-side (i.e. the replicator).\nfunction getParams(ctx: SubscriberContext): URLSearchParams {\n  // The protocolVersion is hard-coded into the CHANGES_PATH.\n  const {protocolVersion, ...stringParams} = ctx;\n  assert(\n    protocolVersion === PROTOCOL_VERSION,\n    `replicator should be setting protocolVersion to ${PROTOCOL_VERSION}`,\n  );\n  return new URLSearchParams({\n    ...stringParams,\n    taskID: ctx.taskID ? ctx.taskID : '',\n    initial: ctx.initial ? 'true' : 'false',\n  });\n}\n"],"mappings":";;;;;;;;;;;;;;;AAgCA,IAAM,iCAAiC;AAEvC,IAAM,wBAAwB;AAC9B,IAAM,uBAAuB;AAC7B,IAAM,aAAa;AAEnB,IAAM,gBAAgB;AACtB,IAAM,eAAe;AAQrB,IAAa,2BAAb,cAA8C,YAAY;CACxD,KAAc;CACd;CACA;CACA;CACA;CAEA,YACE,IACA,MACA,QACA,gBACA,eACA;EACA,MAAM,+BAA+B,IAAI,MAAM,OAAM,YAAW;GAC9D,MAAM,QAAQ,SAAS,SAAS;GAEhC,QAAQ,IAAI,sBAAsB,EAAC,WAAW,KAAI,GAAG,KAAKI,UAAU;GACpE,QAAQ,IACN,uBACA,EAAC,WAAW,KAAI,GAChB,KAAKC,gBACP;GAEA,yBACE,IACA,QAAQ,iBACR,KAAKC,mBACL,MACF;EACF,CAAC;EAED,KAAKN,MAAM;EACX,KAAKC,QAAQ;EACb,KAAKC,kBAAkB;EACvB,KAAKC,iBAAiB;CACxB;CAEA,oBAAoB;EAClB,OAAO,KACL,KAAKA,gBACL,yEACF;CACF;CAGA,qBACE,IACA,QACA,QACG;EACH,QAAQ,QAAR;GACE,KAAK,YACH,OAAO,KAAKE,iBAAiB,IAAI,GAAG;GACtC,KAAK,WACH,OAAO,KAAKD,WAAW,IAAI,GAAG;GAChC;IACE,eACE,KAAK,KACL,IACA,mBAAmB,OAAO,sBAC5B;IACA;EACJ;CACF;CAEA,oBAA6B,IAAe,QAAwB;EAClE,IAAI;GACF,MAAM,MAAM,IAAI,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,kBACxB;GACA,qBAAqB,IAAI,QAAQ;GACjC,MAAM,SAAS,IAAI,aAAa,IAAI,QAAQ;GAC5C,IAAI,CAAC,QACH,MAAM,IAAI,MAAM,oCAAoC;GAEtD,MAAM,aACJ,KAAKG,kBAAkB,EAAE,yBAAyB,MAAM;GAC1D,UAAe,KAAK,KAAK,YAAY,EAAE;EACzC,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;CAEA,aAAsB,OAAO,IAAe,QAAwB;EAClE,IAAI;GACF,MAAM,MAAM,qBAAqB,GAAG;GACpC,IAAI,IAAI,SAAS,WACf,KAAKC,6BAA6B,uBAAuB;GAG3D,MAAM,aAAa,MAAM,KAAKN,gBAAgB,UAAU,GAAG;GAC3D,IAAI,IAAI,WAAW,IAAI,UAAU,KAAKC,gBAGpC,KAAKA,eAAe,eAAe,IAAI,MAAM;GAE/C,qBAA0B,KAAK,KAAK,YAAY,EAAE;EACpD,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;CAEA,yBAAyB;CAEzB,6BAA6B,QAAgB;EAC3C,IAAI,CAAC,KAAKM,0BAA0B,KAAK,OAAO,UAAU,GAAG;GAC3D,KAAKT,IAAI,OAAO,mCAAmC,QAAQ;GAC3D,KAAUE,gBACP,IAAI,EACJ,OAAM,MACL,KAAKF,IAAI,OAAO,0CAA0C,CAAC,CAC7D,EACC,cAAc,KAAK,KAAK,CAAC;GAE5B,KAAKS,yBAAyB;EAChC;CACF;CAEA,WAAoC;EAClC,MAAM,EAAC,mBAAkB,KAAKR;EAC9B,KAAK,OAAO,iBAER,KAAKO,6BACH,0BAA0B,eAAe,KAC3C,GACF,cACF;CACF;CAEA,MAAyB,UAAyB;EAChD,IAAI,KAAKC,wBACP,MAAM,KAAKP,gBAAgB,KAAK;CAEpC;AACF;AAEA,IAAa,2BAAb,MAAgE;CAC9D;CACA;CACA;CACA;CAEA,YACE,IACA,SACA,UACA,mBACA;EACA,KAAKF,MAAM;EACX,KAAKU,WAAW;EAGhB,KAAKC,YAAY,SAAS,IAAI,UAAU,6BAA6B;GACnE,KAAK;IACJ,iBAAiB;EACpB,CAAC;EACD,KAAKC,qBAAqB;CAC5B;CAEA,MAAMC,uBAAuB,MAAc;EACzC,IAAI,UAAU,KAAKD;EACnB,IAAI,CAAC,SAAS;GACZ,MAAM,UAAU,MAAM,8BACpB,KAAKF,UACL,KAAKC,SACP;GACA,IAAI,CAAC,SACH,MAAM,IAAI,MAAM,+BAA+B;GAEjD,UAAU,QAAQ,SAAS,KAAK,IAAI,GAAG,QAAQ,KAAK,QAAQ,QAAQ;EACtE;EACA,MAAM,MAAM,IAAI,IAAI,MAAM,OAAO;EACjC,KAAKX,IAAI,OAAO,iCAAiC,KAAK;EACtD,OAAO;CACT;CAEA,MAAM,gBAAgB,QAAkD;EAItE,MAAM,KAAK,IAAI,YAAU,MAHP,KAAKa,uBAAuB,aAAa,IAG5B,IAAI,IADhB,gBAAgB,EAAC,OAAM,CACP,EAAO,SAAS,GAAG;EAEtD,OAAO,SAAS,KAAKb,KAAK,IAAI,qBAAqB;CACrD;CAEA,MAAM,UAAU,KAAqD;EAInE,MAAM,KAAK,IAAI,YAAU,MAHP,KAAKa,uBAAuB,YAAY,IAG3B,IADhB,UAAU,GACU,EAAO,SAAS,GAAG;EAEtD,OAAO,SAAS,KAAKb,KAAK,IAAI,gBAAgB;CAChD;AACF;AAIA,SAAgB,qBAAqB,KAAwC;CAC3E,MAAM,MAAM,IAAI,IAAI,IAAI,OAAO,IAAI,IAAI,QAAQ,UAAU,kBAAkB;CAC3E,MAAM,kBAAkB,qBAAqB,IAAI,QAAQ;CACzD,MAAM,SAAS,IAAI,UAAU,GAAG;CAEhC,OAAO;EACL;EACA,IAAI,OAAO,IAAI,MAAM,IAAI;EACzB,QAAQ,OAAO,IAAI,UAAU,KAAK;EAClC,MAAM,OAAO,IAAI,QAAQ,KAAK,MAAM,WAAW,WAAW;EAC1D,gBAAgB,OAAO,IAAI,kBAAkB,IAAI;EACjD,WAAW,OAAO,IAAI,aAAa,IAAI;EACvC,SAAS,OAAO,WAAW,SAAS;CACtC;AACF;AAEA,SAAS,qBAAqB,UAA0B;CACtD,MAAM,QAAQ,WAAW,KAAK,QAAQ;CACtC,IAAI,CAAC,OACH,MAAM,IAAI,MAAM,iBAAiB,UAAU;CAE7C,MAAM,IAAI,OAAO,MAAM,QAAQ,OAAO;CACtC,IACE,OAAO,MAAM,CAAC,KACd,IAAA,KACA,IAAI,gCAEJ,MAAM,IAAI,MACR,sCAAsC,EAAE,2BACZ,+BAA+B,SAC7D;CAEF,OAAO;AACT;AAGA,SAAS,UAAU,KAAyC;CAE1D,MAAM,EAAC,iBAAiB,GAAG,iBAAgB;CAC3C,OACE,oBAAA,GACA,mDACF;CACA,OAAO,IAAI,gBAAgB;EACzB,GAAG;EACH,QAAQ,IAAI,SAAS,IAAI,SAAS;EAClC,SAAS,IAAI,UAAU,SAAS;CAClC,CAAC;AACH"}
+{"version":3,"file":"change-streamer-http.js","names":["#lc","#opts","#changeStreamer","#subscribe","#reserveSnapshot","#receiveWebsocket","#ensureChangeStreamerStarted","#changeStreamerStarted","#shardID","#changeDB","#changeStreamerURI","#resolveChangeStreamer"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport websocket from '@fastify/websocket';\nimport type {LogContext} from '@rocicorp/logger';\nimport WebSocket from 'ws';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport type {IncomingMessageSubset} from '../../types/http.ts';\nimport {pgClient, type PostgresDB} from '../../types/pg.ts';\nimport {type Worker} from '../../types/processes.ts';\nimport {type ShardID} from '../../types/shards.ts';\nimport {\n  streamIn,\n  streamInStringified,\n  streamOut,\n  streamOutStringified,\n  type Source,\n} from '../../types/streams.ts';\nimport {URLParams} from '../../types/url-params.ts';\nimport {installWebSocketReceiver} from '../../types/websocket-handoff.ts';\nimport {closeWithError, PROTOCOL_ERROR} from '../../types/ws.ts';\nimport {HttpService} from '../http-service.ts';\nimport {\n  downstreamSchema,\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type ChangeStreamerService,\n  type SerializedDownstream,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport {discoverChangeStreamerAddress} from './schema/tables.ts';\nimport {snapshotMessageSchema, type SnapshotMessage} from './snapshot.ts';\n\nconst MIN_SUPPORTED_PROTOCOL_VERSION = 4;\n\nconst SNAPSHOT_PATH_PATTERN = '/replication/:version/snapshot';\nconst CHANGES_PATH_PATTERN = '/replication/:version/changes';\nconst PATH_REGEX = /\\/replication\\/v(?<version>\\d+)\\/(changes|snapshot)$/;\n\nconst SNAPSHOT_PATH = `/replication/v${PROTOCOL_VERSION}/snapshot`;\nconst CHANGES_PATH = `/replication/v${PROTOCOL_VERSION}/changes`;\n\ntype Options = {\n  port: number;\n  keepaliveTimeoutMs: number | undefined;\n  startupDelayMs: number;\n};\n\nexport class ChangeStreamerHttpServer extends HttpService {\n  readonly id = 'change-streamer-http-server';\n  readonly #lc: LogContext;\n  readonly #opts: Options;\n  readonly #changeStreamer: ChangeStreamerService;\n\n  constructor(\n    lc: LogContext,\n    opts: Options,\n    parent: Worker,\n    changeStreamer: ChangeStreamerService,\n  ) {\n    super('change-streamer-http-server', lc, opts, async fastify => {\n      await fastify.register(websocket);\n\n      fastify.get(CHANGES_PATH_PATTERN, {websocket: true}, this.#subscribe);\n      fastify.get(\n        SNAPSHOT_PATH_PATTERN,\n        {websocket: true},\n        this.#reserveSnapshot,\n      );\n\n      installWebSocketReceiver<'snapshot' | 'changes'>(\n        lc,\n        fastify.websocketServer,\n        this.#receiveWebsocket,\n        parent,\n      );\n    });\n\n    this.#lc = lc;\n    this.#opts = opts;\n    this.#changeStreamer = changeStreamer;\n  }\n\n  // Called when receiving a web socket via the main dispatcher handoff.\n  readonly #receiveWebsocket = (\n    ws: WebSocket,\n    action: 'changes' | 'snapshot',\n    msg: IncomingMessageSubset,\n  ) => {\n    switch (action) {\n      case 'snapshot':\n        return this.#reserveSnapshot(ws, msg);\n      case 'changes':\n        return this.#subscribe(ws, msg);\n      default:\n        closeWithError(\n          this._lc,\n          ws,\n          `invalid action \"${action}\" received in handoff`,\n        );\n        return;\n    }\n  };\n\n  readonly #reserveSnapshot = async (ws: WebSocket, req: RequestHeaders) => {\n    this.#ensureChangeStreamerStarted('incoming snapshot reservation');\n    try {\n      const url = new URL(\n        req.url ?? '',\n        req.headers.origin ?? 'http://localhost',\n      );\n      checkProtocolVersion(url.pathname);\n      const taskID = url.searchParams.get('taskID');\n      if (!taskID) {\n        throw new Error('Missing taskID in snapshot request');\n      }\n      const downstream =\n        await this.#changeStreamer.startSnapshotReservation(taskID);\n      void streamOut(this._lc, downstream, ws);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  readonly #subscribe = async (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const ctx = getSubscriberContext(req);\n      if (ctx.mode === 'serving') {\n        this.#ensureChangeStreamerStarted('incoming subscription');\n      }\n\n      const downstream = await this.#changeStreamer.subscribe(ctx);\n      void streamOutStringified(this._lc, downstream, ws);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  #changeStreamerStarted = false;\n\n  #ensureChangeStreamerStarted(reason: string) {\n    if (!this.#changeStreamerStarted && this._state.shouldRun()) {\n      this.#lc.info?.(`starting ChangeStreamerService: ${reason}`);\n      void this.#changeStreamer\n        .run()\n        .catch(e =>\n          this.#lc.warn?.(`ChangeStreamerService ended with error`, e),\n        )\n        .finally(() => this.stop());\n\n      this.#changeStreamerStarted = true;\n    }\n  }\n\n  protected override _onStart(): void {\n    const {startupDelayMs} = this.#opts;\n    this._state.setTimeout(\n      () =>\n        this.#ensureChangeStreamerStarted(\n          `startup delay elapsed (${startupDelayMs} ms)`,\n        ),\n      startupDelayMs,\n    );\n  }\n\n  protected override async _onStop(): Promise<void> {\n    if (this.#changeStreamerStarted) {\n      await this.#changeStreamer.stop();\n    }\n  }\n}\n\nexport class ChangeStreamerHttpClient implements ChangeStreamer {\n  readonly #lc: LogContext;\n  readonly #shardID: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #changeStreamerURI: string | undefined;\n\n  constructor(\n    lc: LogContext,\n    shardID: ShardID,\n    changeDB: string,\n    changeStreamerURI: string | undefined,\n  ) {\n    this.#lc = lc;\n    this.#shardID = shardID;\n    // Create a pg client with a single short-lived connection for the purpose\n    // of change-streamer discovery (i.e. ChangeDB as DNS).\n    this.#changeDB = pgClient(lc, changeDB, 'change-streamer-discovery', {\n      max: 1,\n      ['idle_timeout']: 15,\n    });\n    this.#changeStreamerURI = changeStreamerURI;\n  }\n\n  async #resolveChangeStreamer(path: string) {\n    let baseURL = this.#changeStreamerURI;\n    if (!baseURL) {\n      const address = await discoverChangeStreamerAddress(\n        this.#shardID,\n        this.#changeDB,\n      );\n      if (!address) {\n        throw new Error(`no change-streamer is running`);\n      }\n      baseURL = address.includes('://') ? `${address}/` : `ws://${address}/`;\n    }\n    const uri = new URL(path, baseURL);\n    this.#lc.info?.(`connecting to change-streamer@${uri}`);\n    return uri;\n  }\n\n  async reserveSnapshot(taskID: string): Promise<Source<SnapshotMessage>> {\n    const uri = await this.#resolveChangeStreamer(SNAPSHOT_PATH);\n\n    const params = new URLSearchParams({taskID});\n    const ws = new WebSocket(uri + `?${params.toString()}`);\n\n    return streamIn(this.#lc, ws, snapshotMessageSchema);\n  }\n\n  async subscribe(\n    ctx: SubscriberContext,\n  ): Promise<Source<SerializedDownstream>> {\n    const uri = await this.#resolveChangeStreamer(CHANGES_PATH);\n\n    const params = getParams(ctx);\n    const ws = new WebSocket(uri + `?${params.toString()}`);\n\n    return streamInStringified(this.#lc, ws, downstreamSchema);\n  }\n}\n\ntype RequestHeaders = Pick<IncomingMessage, 'url' | 'headers'>;\n\nexport function getSubscriberContext(req: RequestHeaders): SubscriberContext {\n  const url = new URL(req.url ?? '', req.headers.origin ?? 'http://localhost');\n  const protocolVersion = checkProtocolVersion(url.pathname);\n  const params = new URLParams(url);\n\n  return {\n    protocolVersion,\n    id: params.get('id', true),\n    taskID: params.get('taskID', true),\n    mode: params.get('mode', false) === 'backup' ? 'backup' : 'serving',\n    replicaVersion: params.get('replicaVersion', true),\n    watermark: params.get('watermark', true),\n    initial: params.getBoolean('initial'),\n    // Absent for subscribers that predate the parameter, which is the safe\n    // default: the barrier falls back to polling rather than waiting on an\n    // ACK that would never be attributed to a writer.\n    logsChangeStream: params.getBoolean('logsChangeStream'),\n  };\n}\n\nfunction checkProtocolVersion(pathname: string): number {\n  const match = PATH_REGEX.exec(pathname);\n  if (!match) {\n    throw new Error(`invalid path: ${pathname}`);\n  }\n  const v = Number(match.groups?.version);\n  if (\n    Number.isNaN(v) ||\n    v > PROTOCOL_VERSION ||\n    v < MIN_SUPPORTED_PROTOCOL_VERSION\n  ) {\n    throw new Error(\n      `Cannot service client at protocol v${v}. ` +\n        `Supported protocols: [v${MIN_SUPPORTED_PROTOCOL_VERSION} ... v${PROTOCOL_VERSION}]`,\n    );\n  }\n  return v;\n}\n\n// This is called from the client-side (i.e. the replicator).\nfunction getParams(ctx: SubscriberContext): URLSearchParams {\n  // The protocolVersion is hard-coded into the CHANGES_PATH.\n  const {protocolVersion, ...stringParams} = ctx;\n  assert(\n    protocolVersion === PROTOCOL_VERSION,\n    `replicator should be setting protocolVersion to ${PROTOCOL_VERSION}`,\n  );\n  return new URLSearchParams({\n    ...stringParams,\n    initial: ctx.initial ? 'true' : 'false',\n    logsChangeStream: ctx.logsChangeStream ? 'true' : 'false',\n  });\n}\n"],"mappings":";;;;;;;;;;;;;;AA+BA,IAAM,iCAAiC;AAEvC,IAAM,wBAAwB;AAC9B,IAAM,uBAAuB;AAC7B,IAAM,aAAa;AAEnB,IAAM,gBAAgB;AACtB,IAAM,eAAe;AAQrB,IAAa,2BAAb,cAA8C,YAAY;CACxD,KAAc;CACd;CACA;CACA;CAEA,YACE,IACA,MACA,QACA,gBACA;EACA,MAAM,+BAA+B,IAAI,MAAM,OAAM,YAAW;GAC9D,MAAM,QAAQ,SAAS,SAAS;GAEhC,QAAQ,IAAI,sBAAsB,EAAC,WAAW,KAAI,GAAG,KAAKG,UAAU;GACpE,QAAQ,IACN,uBACA,EAAC,WAAW,KAAI,GAChB,KAAKC,gBACP;GAEA,yBACE,IACA,QAAQ,iBACR,KAAKC,mBACL,MACF;EACF,CAAC;EAED,KAAKL,MAAM;EACX,KAAKC,QAAQ;EACb,KAAKC,kBAAkB;CACzB;CAGA,qBACE,IACA,QACA,QACG;EACH,QAAQ,QAAR;GACE,KAAK,YACH,OAAO,KAAKE,iBAAiB,IAAI,GAAG;GACtC,KAAK,WACH,OAAO,KAAKD,WAAW,IAAI,GAAG;GAChC;IACE,eACE,KAAK,KACL,IACA,mBAAmB,OAAO,sBAC5B;IACA;EACJ;CACF;CAEA,mBAA4B,OAAO,IAAe,QAAwB;EACxE,KAAKG,6BAA6B,+BAA+B;EACjE,IAAI;GACF,MAAM,MAAM,IAAI,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,kBACxB;GACA,qBAAqB,IAAI,QAAQ;GACjC,MAAM,SAAS,IAAI,aAAa,IAAI,QAAQ;GAC5C,IAAI,CAAC,QACH,MAAM,IAAI,MAAM,oCAAoC;GAEtD,MAAM,aACJ,MAAM,KAAKJ,gBAAgB,yBAAyB,MAAM;GAC5D,UAAe,KAAK,KAAK,YAAY,EAAE;EACzC,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;CAEA,aAAsB,OAAO,IAAe,QAAwB;EAClE,IAAI;GACF,MAAM,MAAM,qBAAqB,GAAG;GACpC,IAAI,IAAI,SAAS,WACf,KAAKI,6BAA6B,uBAAuB;GAG3D,MAAM,aAAa,MAAM,KAAKJ,gBAAgB,UAAU,GAAG;GAC3D,qBAA0B,KAAK,KAAK,YAAY,EAAE;EACpD,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;CAEA,yBAAyB;CAEzB,6BAA6B,QAAgB;EAC3C,IAAI,CAAC,KAAKK,0BAA0B,KAAK,OAAO,UAAU,GAAG;GAC3D,KAAKP,IAAI,OAAO,mCAAmC,QAAQ;GAC3D,KAAUE,gBACP,IAAI,EACJ,OAAM,MACL,KAAKF,IAAI,OAAO,0CAA0C,CAAC,CAC7D,EACC,cAAc,KAAK,KAAK,CAAC;GAE5B,KAAKO,yBAAyB;EAChC;CACF;CAEA,WAAoC;EAClC,MAAM,EAAC,mBAAkB,KAAKN;EAC9B,KAAK,OAAO,iBAER,KAAKK,6BACH,0BAA0B,eAAe,KAC3C,GACF,cACF;CACF;CAEA,MAAyB,UAAyB;EAChD,IAAI,KAAKC,wBACP,MAAM,KAAKL,gBAAgB,KAAK;CAEpC;AACF;AAEA,IAAa,2BAAb,MAAgE;CAC9D;CACA;CACA;CACA;CAEA,YACE,IACA,SACA,UACA,mBACA;EACA,KAAKF,MAAM;EACX,KAAKQ,WAAW;EAGhB,KAAKC,YAAY,SAAS,IAAI,UAAU,6BAA6B;GACnE,KAAK;IACJ,iBAAiB;EACpB,CAAC;EACD,KAAKC,qBAAqB;CAC5B;CAEA,MAAMC,uBAAuB,MAAc;EACzC,IAAI,UAAU,KAAKD;EACnB,IAAI,CAAC,SAAS;GACZ,MAAM,UAAU,MAAM,8BACpB,KAAKF,UACL,KAAKC,SACP;GACA,IAAI,CAAC,SACH,MAAM,IAAI,MAAM,+BAA+B;GAEjD,UAAU,QAAQ,SAAS,KAAK,IAAI,GAAG,QAAQ,KAAK,QAAQ,QAAQ;EACtE;EACA,MAAM,MAAM,IAAI,IAAI,MAAM,OAAO;EACjC,KAAKT,IAAI,OAAO,iCAAiC,KAAK;EACtD,OAAO;CACT;CAEA,MAAM,gBAAgB,QAAkD;EAItE,MAAM,KAAK,IAAI,YAAU,MAHP,KAAKW,uBAAuB,aAAa,IAG5B,IAAI,IADhB,gBAAgB,EAAC,OAAM,CACP,EAAO,SAAS,GAAG;EAEtD,OAAO,SAAS,KAAKX,KAAK,IAAI,qBAAqB;CACrD;CAEA,MAAM,UACJ,KACuC;EAIvC,MAAM,KAAK,IAAI,YAAU,MAHP,KAAKW,uBAAuB,YAAY,IAG3B,IADhB,UAAU,GACU,EAAO,SAAS,GAAG;EAEtD,OAAO,oBAAoB,KAAKX,KAAK,IAAI,gBAAgB;CAC3D;AACF;AAIA,SAAgB,qBAAqB,KAAwC;CAC3E,MAAM,MAAM,IAAI,IAAI,IAAI,OAAO,IAAI,IAAI,QAAQ,UAAU,kBAAkB;CAC3E,MAAM,kBAAkB,qBAAqB,IAAI,QAAQ;CACzD,MAAM,SAAS,IAAI,UAAU,GAAG;CAEhC,OAAO;EACL;EACA,IAAI,OAAO,IAAI,MAAM,IAAI;EACzB,QAAQ,OAAO,IAAI,UAAU,IAAI;EACjC,MAAM,OAAO,IAAI,QAAQ,KAAK,MAAM,WAAW,WAAW;EAC1D,gBAAgB,OAAO,IAAI,kBAAkB,IAAI;EACjD,WAAW,OAAO,IAAI,aAAa,IAAI;EACvC,SAAS,OAAO,WAAW,SAAS;EAIpC,kBAAkB,OAAO,WAAW,kBAAkB;CACxD;AACF;AAEA,SAAS,qBAAqB,UAA0B;CACtD,MAAM,QAAQ,WAAW,KAAK,QAAQ;CACtC,IAAI,CAAC,OACH,MAAM,IAAI,MAAM,iBAAiB,UAAU;CAE7C,MAAM,IAAI,OAAO,MAAM,QAAQ,OAAO;CACtC,IACE,OAAO,MAAM,CAAC,KACd,IAAA,KACA,IAAI,gCAEJ,MAAM,IAAI,MACR,sCAAsC,EAAE,2BACZ,+BAA+B,SAC7D;CAEF,OAAO;AACT;AAGA,SAAS,UAAU,KAAyC;CAE1D,MAAM,EAAC,iBAAiB,GAAG,iBAAgB;CAC3C,OACE,oBAAA,GACA,mDACF;CACA,OAAO,IAAI,gBAAgB;EACzB,GAAG;EACH,SAAS,IAAI,UAAU,SAAS;EAChC,kBAAkB,IAAI,mBAAmB,SAAS;CACpD,CAAC;AACH"}

package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts

@@ -2,36 +2,81 @@ import type { LogContext } from '@rocicorp/logger';
 import type { PostgresDB } from '../../types/pg.ts';
 import type { ShardID } from '../../types/shards.ts';
 import type { ChangeSource } from '../change-source/change-source.ts';
-import { type ChangeStreamData } from '../change-source/protocol/current/downstream.ts';
+import type { LitestreamVersion } from '../litestream/metrics.ts';
 import { type ReplicationStatusPublisher } from '../replicator/replication-status.ts';
 import type { SubscriptionState } from '../replicator/schema/replication-state.ts';
-import { type ChangeStreamerService } from './change-streamer.ts';
+import { type ChangeStreamerService, type SubscriberContext } from './change-streamer.ts';
+import { type SQLiteChangeLogCleanupGuard } from './sqlite-change-log-catchup.ts';
+import { type SQLiteChangeLogPurgeSchedulerOptions } from './sqlite-change-log-purge-scheduler.ts';
+import { type SQLiteChangeLogWriterOptions } from './sqlite-change-log-writer.ts';
 import { type PurgeLock, type TuningOptions as StorerOptions } from './storer.ts';
+export type BackupConfig = {
+    backupURL: string;
+    litestreamVersion: LitestreamVersion;
+};
+export type SQLiteCatchupOptions = {
+    /** The change-log database, i.e. `changeLogFileName(replicaFile)`. */
+    changeLogFile: string;
+    readBatchRows: number;
+    barrierTimeoutMs: number;
+    /**
+     * Backstop for the change-log writer's ACK, which is what normally releases
+     * the barrier. Defaults to the catchup coordinator's interval.
+     */
+    barrierPollIntervalMs?: number | undefined;
+    /**
+     * Slice 11 supplies the production canary selector.
+     *
+     * It does not need to exclude backup subscribers or the change-log writer:
+     * eligibility checks reject both before invoking this selector. Serving a
+     * writer from SQLite would make it wait on its own ACK.
+     */
+    shouldUse?: ((ctx: SubscriberContext) => boolean) | undefined;
+    /**
+     * Overrides the purge scheduler's guard, for tests. When absent, the
+     * service supplies the scheduler's real writer-serialized guard, or the
+     * catchup's no-op default when purging is not configured.
+     */
+    cleanupGuard?: SQLiteChangeLogCleanupGuard | undefined;
+    /**
+     * How long the change log may be unavailable before declining to serve from
+     * it is reported as a warning rather than a debug line. Defaults to
+     * {@link DEFAULT_CHANGE_LOG_UNAVAILABLE_WARN_THRESHOLD_MS}.
+     */
+    notReadyWarnThresholdMs?: number | undefined;
+};
 export type TuningOptions = StorerOptions & {
-    flowControlConsensusPaddingSeconds: number;
+    flowControlConsensusTimeoutProportion: number;
+    flowControlSlowSubscriberGracePeriodMs?: number | undefined;
+    sqliteCatchup?: SQLiteCatchupOptions | undefined;
+    /**
+     * Supplied when `sqliteChangeLogMode != off`, i.e. this is the gate on the
+     * change-log writer. Absent, nothing writes the log.
+     */
+    sqliteChangeLogWriter?: Omit<SQLiteChangeLogWriterOptions, 'onCommit' | 'onDisabled' | 'onRebuilt'> | undefined;
+    /**
+     * Also supplied when `sqliteChangeLogMode != off`, and deliberately not
+     * gated on the read path: in `write` mode no reader ever opens, and this is
+     * the configuration the purge scheduler actually ships in. The scheduler
+     * runs on the writer's own connection, so with the writer absent (mode
+     * `off`) it is never constructed, and a leftover file on disk is left to
+     * the replicator's cleanup.
+     */
+    sqliteChangeLogPurge?: SQLiteChangeLogPurgeSchedulerOptions | undefined;
+    /**
+     * Supplied at `sqliteChangeLogMode >= compare`, and the gate on deriving the
+     * stream's initialization parameters from the replica as well as from
+     * Postgres. Postgres stays authoritative; the two are compared and the
+     * result is reported. Not a new configuration option -- the mode ladder
+     * carries it, because a state in which the log's buffer and its cookies were
+     * at different rollout stages is what invariant 15 exists to forbid.
+     */
+    sqliteChangeLogCompare?: {
+        replicaFile: string;
+    } | undefined;
 };
 /**
  * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.
  */
-export declare function initializeStreamer(lc: LogContext, shard: ShardID, taskID: string, discoveryAddress: string, discoveryProtocol: string, changeDB: PostgresDB, changeSource: ChangeSource, replicationStatusPublisher: ReplicationStatusPublisher, subscriptionState: SubscriptionState, purgeLock: PurgeLock | null, autoReset: boolean, opts: TuningOptions, setTimeoutFn?: typeof setTimeout): Promise<ChangeStreamerService>;
-export type ChangeTag = ChangeStreamData[1]['tag'];
-/**
- * Internally all Downstream messages (not just commits) are given a watermark.
- * These are used for internal ordering for:
- * 1. Replaying new changes in the Storer
- * 2. Filtering old changes in the Subscriber
- *
- * However, only the watermark for `Commit` messages are exposed to
- * subscribers, as that is the only semantically correct watermark to
- * use for tracking a position in a replication stream.
- *
- * Additionally, the ChangeStreamData is eagerly stringified once, after which
- * the string is passed to the changeLog and all subscribers, eliminating
- * redundant stringification and reducing GC churn.
- */
-export type WatermarkedChange = [
-    watermark: string,
-    tag: ChangeTag,
-    json: string
-];
+export declare function initializeStreamer(lc: LogContext, shard: ShardID, taskID: string, discoveryAddress: string, discoveryProtocol: string, changeDB: PostgresDB, changeSource: ChangeSource, replicationStatusPublisher: ReplicationStatusPublisher, subscriptionState: SubscriptionState, backupConfig: BackupConfig | null, purgeLock: PurgeLock | null, autoReset: boolean, opts: TuningOptions, setTimeoutFn?: typeof setTimeout): Promise<ChangeStreamerService>;
 //# sourceMappingURL=change-streamer-service.d.ts.map

package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-service.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAWjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAClD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAGnD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAEL,KAAK,gBAAgB,EAEtB,MAAM,iDAAiD,CAAC;AACzD,OAAO,EAGL,KAAK,0BAA0B,EAChC,MAAM,qCAAqC,CAAC;AAC7C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,2CAA2C,CAAC;AAMjF,OAAO,EACL,KAAK,qBAAqB,EAG3B,MAAM,sBAAsB,CAAC;AAS9B,OAAO,EAEL,KAAK,SAAS,EACd,KAAK,aAAa,IAAI,aAAa,EACpC,MAAM,aAAa,CAAC;AAGrB,MAAM,MAAM,aAAa,GAAG,aAAa,GAAG;IAC1C,kCAAkC,EAAE,MAAM,CAAC;CAC5C,CAAC;AAEF;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,QAAQ,EAAE,UAAU,EACpB,YAAY,EAAE,YAAY,EAC1B,0BAA0B,EAAE,0BAA0B,EACtD,iBAAiB,EAAE,iBAAiB,EACpC,SAAS,EAAE,SAAS,GAAG,IAAI,EAC3B,SAAS,EAAE,OAAO,EAClB,IAAI,EAAE,aAAa,EACnB,YAAY,oBAAa,GACxB,OAAO,CAAC,qBAAqB,CAAC,CA4BhC;AAID,MAAM,MAAM,SAAS,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;AAEnD;;;;;;;;;;;;;GAaG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,MAAM;IACjB,GAAG,EAAE,SAAS;IACd,IAAI,EAAE,MAAM;CACb,CAAC"}
+{"version":3,"file":"change-streamer-service.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAWjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAClD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAGnD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,mCAAmC,CAAC;AAM3C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAGL,KAAK,0BAA0B,EAChC,MAAM,qCAAqC,CAAC;AAC7C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,2CAA2C,CAAC;AAUjF,OAAO,EACL,KAAK,qBAAqB,EAE1B,KAAK,iBAAiB,EAEvB,MAAM,sBAAsB,CAAC;AAU9B,OAAO,EAEL,KAAK,2BAA2B,EACjC,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAGL,KAAK,oCAAoC,EAC1C,MAAM,wCAAwC,CAAC;AAEhD,OAAO,EAEL,KAAK,4BAA4B,EAClC,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAEL,KAAK,SAAS,EACd,KAAK,aAAa,IAAI,aAAa,EACpC,MAAM,aAAa,CAAC;AAIrB,MAAM,MAAM,YAAY,GAAG;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,iBAAiB,CAAC;CACtC,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,sEAAsE;IACtE,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3C;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,iBAAiB,KAAK,OAAO,CAAC,GAAG,SAAS,CAAC;IAC9D;;;;OAIG;IACH,YAAY,CAAC,EAAE,2BAA2B,GAAG,SAAS,CAAC;IACvD;;;;OAIG;IACH,uBAAuB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC9C,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG,aAAa,GAAG;IAC1C,qCAAqC,EAAE,MAAM,CAAC;IAC9C,sCAAsC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5D,aAAa,CAAC,EAAE,oBAAoB,GAAG,SAAS,CAAC;IACjD;;;OAGG;IACH,qBAAqB,CAAC,EAClB,IAAI,CACF,4BAA4B,EAC5B,UAAU,GAAG,YAAY,GAAG,WAAW,CACxC,GACD,SAAS,CAAC;IACd;;;;;;;OAOG;IACH,oBAAoB,CAAC,EAAE,oCAAoC,GAAG,SAAS,CAAC;IACxE;;;;;;;OAOG;IACH,sBAAsB,CAAC,EAAE;QAAC,WAAW,EAAE,MAAM,CAAA;KAAC,GAAG,SAAS,CAAC;CAC5D,CAAC;AAEF;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,QAAQ,EAAE,UAAU,EACpB,YAAY,EAAE,YAAY,EAC1B,0BAA0B,EAAE,0BAA0B,EACtD,iBAAiB,EAAE,iBAAiB,EACpC,YAAY,EAAE,YAAY,GAAG,IAAI,EACjC,SAAS,EAAE,SAAS,GAAG,IAAI,EAC3B,SAAS,EAAE,OAAO,EAClB,IAAI,EAAE,aAAa,EACnB,YAAY,oBAAa,GACxB,OAAO,CAAC,qBAAqB,CAAC,CA4BhC"}