@rocicorp/zero 1.9.1-canary.1 → 1.10.0-canary.1

package/out/zero-cache/src/services/litestream/vfs-watermark-poller.d.ts

@@ -0,0 +1,57 @@
+import type { LogContext, LogLevel } from '@rocicorp/logger';
+import { type Worker } from '../../types/processes.ts';
+import type { Service } from '../service.ts';
+type LoadableDatabase = {
+    loadExtension(path: string): unknown;
+    close(): void;
+};
+type QueryDatabase = {
+    prepare(sql: string): {
+        get<T>(): T;
+    };
+    close(): void;
+};
+export type BackupWatermarkUpdate = [
+    'backupWatermarkUpdate',
+    {
+        watermark: string;
+        writeTimeMs: number;
+        backupTimeMs: number;
+    }
+];
+export type VfsBackupWatermarkPollerEnvOptions = {
+    readonly backupURL: string;
+    readonly remotePollIntervalMs: number;
+    readonly endpoint?: string | undefined;
+    readonly region?: string | undefined;
+    readonly logLevel?: LogLevel | undefined;
+    readonly logFile?: string | undefined;
+};
+export type VfsBackupWatermarkPollerOptions = {
+    readonly localDbFile: string;
+    readonly backupURL: string;
+    readonly endpoint?: string | undefined;
+    readonly region?: string | undefined;
+    readonly extensionPath: string;
+    readonly remotePollIntervalMs: number;
+};
+type VfsBackupWatermarkPollerDeps = {
+    readonly loaderFactory?: (() => LoadableDatabase) | undefined;
+    readonly databaseFactory?: ((lc: LogContext, uri: string) => QueryDatabase) | undefined;
+};
+export declare function buildLitestreamVfsReplicaURL({ backupURL, endpoint, region, }: {
+    readonly backupURL: string;
+    readonly endpoint?: string | undefined;
+    readonly region?: string | undefined;
+}): string;
+export declare function getVfsEnv(options: VfsBackupWatermarkPollerEnvOptions): NodeJS.ProcessEnv;
+export declare class VfsBackupWatermarkPoller implements Service {
+    #private;
+    readonly id = "backup-watermark-poller";
+    constructor(lc: LogContext, parent: Worker, options: VfsBackupWatermarkPollerOptions, deps?: VfsBackupWatermarkPollerDeps);
+    checkWatermarks(): void;
+    run(): Promise<void>;
+    stop(): Promise<void>;
+}
+export {};
+//# sourceMappingURL=vfs-watermark-poller.d.ts.map

package/out/zero-cache/src/services/litestream/vfs-watermark-poller.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vfs-watermark-poller.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/vfs-watermark-poller.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAE,QAAQ,EAAC,MAAM,kBAAkB,CAAC;AAK3D,OAAO,EAAC,KAAK,MAAM,EAAC,MAAM,0BAA0B,CAAC;AAErD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAI3C,KAAK,gBAAgB,GAAG;IACtB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC;IACrC,KAAK,IAAI,IAAI,CAAC;CACf,CAAC;AAEF,KAAK,aAAa,GAAG;IACnB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG;QAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAA;KAAC,CAAC;IACpC,KAAK,IAAI,IAAI,CAAC;CACf,CAAC;AAeF,MAAM,MAAM,qBAAqB,GAAG;IAClC,uBAAuB;IACvB;QACE,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;KACtB;CACF,CAAC;AAUF,MAAM,MAAM,kCAAkC,GAAG;IAC/C,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC;IACtC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAErC,QAAQ,CAAC,QAAQ,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAC;IACzC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACvC,CAAC;AAEF,MAAM,MAAM,+BAA+B,GAAG;IAC5C,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAErC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC;CACvC,CAAC;AAEF,KAAK,4BAA4B,GAAG;IAClC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC,MAAM,gBAAgB,CAAC,GAAG,SAAS,CAAC;IAC9D,QAAQ,CAAC,eAAe,CAAC,EACrB,CAAC,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,KAAK,aAAa,CAAC,GAChD,SAAS,CAAC;CACf,CAAC;AAEF,wBAAgB,4BAA4B,CAAC,EAC3C,SAAS,EACT,QAAQ,EACR,MAAM,GACP,EAAE;IACD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACtC,GAAG,MAAM,CAiBT;AAED,wBAAgB,SAAS,CACvB,OAAO,EAAE,kCAAkC,GAC1C,MAAM,CAAC,UAAU,CAWnB;AAKD,qBAAa,wBAAyB,YAAW,OAAO;;IACtD,QAAQ,CAAC,EAAE,6BAA6B;gBAatC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,+BAA+B,EACxC,IAAI,GAAE,4BAAiC;IAgEzC,eAAe;IAiDT,GAAG;IAkBT,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAItB"}

package/out/zero-cache/src/services/litestream/vfs-watermark-poller.js

@@ -0,0 +1,137 @@
+import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
+import { parse, valita_exports } from "../../../../shared/src/valita.js";
+import "../../types/processes.js";
+import { Database } from "../../../../zqlite/src/db.js";
+import { RunningState } from "../running-state.js";
+import Sqlite3Database from "@rocicorp/zero-sqlite3";
+//#region ../zero-cache/src/services/litestream/vfs-watermark-poller.ts
+var DEFAULT_VFS_LOG_LEVEL = "info";
+var watermarkRowSchema = valita_exports.object({
+	watermark: valita_exports.string(),
+	writeTimeMs: valita_exports.number()
+});
+var watermarkQuerySchema = watermarkRowSchema.extend({ backupTime: valita_exports.string() });
+function buildLitestreamVfsReplicaURL({ backupURL, endpoint, region }) {
+	if (endpoint === void 0 && region === void 0) return backupURL;
+	const url = new URL(backupURL);
+	if (url.protocol !== "s3:") return backupURL;
+	if (endpoint !== void 0 && !url.searchParams.has("endpoint")) url.searchParams.set("endpoint", endpoint);
+	if (region !== void 0 && !url.searchParams.has("region")) url.searchParams.set("region", region);
+	return url.toString();
+}
+function getVfsEnv(options) {
+	return {
+		LITESTREAM_REPLICA_URL: buildLitestreamVfsReplicaURL(options),
+		LITESTREAM_LOG_LEVEL: options.logLevel ?? DEFAULT_VFS_LOG_LEVEL,
+		LITESTREAM_LOG_FILE: options.logFile,
+		LITESTREAM_POLL_INTERVAL: `${options.remotePollIntervalMs}ms`
+	};
+}
+var MIN_CHECK_INTERVAL_MS = 1e3;
+var MAX_CHECK_INTERVAL_MS = 6e4;
+var VfsBackupWatermarkPoller = class {
+	id = "backup-watermark-poller";
+	#runningState = new RunningState(this.id);
+	#lc;
+	#parent;
+	#loader;
+	#localDb;
+	#openRemote;
+	#remoteDb = null;
+	#lastRemoteWatermark = null;
+	constructor(lc, parent, options, deps = {}) {
+		lc = lc.withContext("component", "vfs-watermark-poller");
+		this.#lc = lc;
+		this.#parent = parent;
+		const { databaseFactory } = deps;
+		const openDb = (url, type, schema) => {
+			try {
+				const db = databaseFactory ? databaseFactory(lc, url) : new Database(lc, url, { readonly: true });
+				const stmt = db.prepare(type === "local" ? `
+          SELECT stateVersion AS watermark, writeTimeMs FROM "_zero.replicationState"` : `
+          SELECT stateVersion AS watermark, writeTimeMs, litestream_time() as backupTime FROM "_zero.replicationState"`);
+				return {
+					getWatermark: () => parse(stmt.get(), schema),
+					close: () => db.close()
+				};
+			} catch (e) {
+				lc.error?.(`error opening ${url}`, e);
+				throw e;
+			}
+		};
+		this.#localDb = openDb(options.localDbFile, "local", watermarkRowSchema);
+		const loaderFactory = deps.loaderFactory ?? (() => new Sqlite3Database(":memory:"));
+		const remoteDbURL = `file:zero-backup.db?${new URLSearchParams({
+			vfs: "litestream",
+			mode: "ro",
+			replica_url: buildLitestreamVfsReplicaURL(options),
+			poll_interval: `${options.remotePollIntervalMs}ms`
+		}).toString()}`;
+		lc.info?.(`Loading VFS ${remoteDbURL} at ${options.extensionPath}`);
+		this.#loader = loaderFactory();
+		this.#loader.loadExtension(options.extensionPath);
+		this.#openRemote = () => openDb(remoteDbURL, "remote", watermarkQuerySchema);
+	}
+	checkWatermarks() {
+		const local = this.#localDb.getWatermark();
+		if (local.watermark !== this.#lastRemoteWatermark?.watermark) {
+			if (!this.#remoteDb) {
+				if (this.#lastRemoteWatermark) this.#lc.debug?.(`local and remote watermarks differ. resuming backup polling`, {
+					local,
+					remote: this.#lastRemoteWatermark
+				});
+				this.#remoteDb = this.#openRemote();
+			}
+			const remote = this.#remoteDb.getWatermark();
+			if (remote.watermark !== this.#lastRemoteWatermark?.watermark) {
+				this.#lc.debug?.(`backup watermark advanced from ${this.#lastRemoteWatermark?.watermark} to ${remote.watermark}`, {
+					local,
+					remote
+				});
+				this.#lastRemoteWatermark = remote;
+				const { watermark, writeTimeMs, backupTime } = remote;
+				const backupTimeMs = new Date(backupTime).getTime();
+				this.#parent.send(["backupWatermarkUpdate", {
+					watermark,
+					writeTimeMs,
+					backupTimeMs
+				}]);
+			}
+		}
+		if (local.watermark === this.#lastRemoteWatermark?.watermark) {
+			if (this.#remoteDb) {
+				this.#lc.debug?.(`local and remote watermarks match. pausing backup polling`, {
+					local,
+					remote: this.#lastRemoteWatermark
+				});
+				this.#remoteDb.close();
+				this.#remoteDb = null;
+			}
+		}
+	}
+	async run() {
+		this.#lc.info?.(`starting backup polling`);
+		let interval = MIN_CHECK_INTERVAL_MS;
+		while (this.#runningState.shouldRun()) {
+			try {
+				this.checkWatermarks();
+				interval = MIN_CHECK_INTERVAL_MS;
+			} catch (e) {
+				this.#lc.warn?.(`error checking local and remote watermarks`, e);
+				interval = Math.min(MAX_CHECK_INTERVAL_MS, interval * 2);
+			}
+			await this.#runningState.sleep(interval);
+		}
+		this.#remoteDb?.close();
+		this.#localDb.close();
+		this.#loader.close();
+	}
+	stop() {
+		this.#runningState.stop(this.#lc);
+		return promiseVoid;
+	}
+};
+//#endregion
+export { VfsBackupWatermarkPoller, getVfsEnv };
+
+//# sourceMappingURL=vfs-watermark-poller.js.map

package/out/zero-cache/src/services/litestream/vfs-watermark-poller.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vfs-watermark-poller.js","names":["#runningState","#lc","#parent","#loader","#localDb","#openRemote","#lastRemoteWatermark","#remoteDb"],"sources":["../../../../../../zero-cache/src/services/litestream/vfs-watermark-poller.ts"],"sourcesContent":["import type {LogContext, LogLevel} from '@rocicorp/logger';\nimport Sqlite3Database from '@rocicorp/zero-sqlite3';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {type Worker} from '../../types/processes.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {Service} from '../service.ts';\n\nconst DEFAULT_VFS_LOG_LEVEL = 'info';\n\ntype LoadableDatabase = {\n  loadExtension(path: string): unknown;\n  close(): void;\n};\n\ntype QueryDatabase = {\n  prepare(sql: string): {get<T>(): T};\n  close(): void;\n};\n\nconst watermarkRowSchema = v.object({\n  watermark: v.string(),\n  writeTimeMs: v.number(),\n});\n\nconst watermarkQuerySchema = watermarkRowSchema.extend({\n  backupTime: v.string(),\n});\n\ntype WatermarkQueryResult = v.Infer<typeof watermarkQuerySchema>;\n\ntype WatermarkRow = v.Infer<typeof watermarkRowSchema>;\n\nexport type BackupWatermarkUpdate = [\n  'backupWatermarkUpdate',\n  {\n    watermark: string;\n    writeTimeMs: number;\n    backupTimeMs: number;\n  },\n];\n\ntype WatermarkReader<Result extends WatermarkRow> = {\n  getWatermark(): Result;\n  close(): void;\n};\n\n// Values that must be passed via the ENV to the original process (worker).\n// Some of these will be configurable via the db URI once\n// https://github.com/benbjohnson/litestream/issues/1150 is released.\nexport type VfsBackupWatermarkPollerEnvOptions = {\n  readonly backupURL: string;\n  readonly remotePollIntervalMs: number;\n  readonly endpoint?: string | undefined;\n  readonly region?: string | undefined;\n\n  readonly logLevel?: LogLevel | undefined;\n  readonly logFile?: string | undefined;\n};\n\nexport type VfsBackupWatermarkPollerOptions = {\n  readonly localDbFile: string;\n  readonly backupURL: string;\n  readonly endpoint?: string | undefined;\n  readonly region?: string | undefined;\n\n  readonly extensionPath: string;\n  readonly remotePollIntervalMs: number;\n};\n\ntype VfsBackupWatermarkPollerDeps = {\n  readonly loaderFactory?: (() => LoadableDatabase) | undefined;\n  readonly databaseFactory?:\n    | ((lc: LogContext, uri: string) => QueryDatabase)\n    | undefined;\n};\n\nexport function buildLitestreamVfsReplicaURL({\n  backupURL,\n  endpoint,\n  region,\n}: {\n  readonly backupURL: string;\n  readonly endpoint?: string | undefined;\n  readonly region?: string | undefined;\n}): string {\n  if (endpoint === undefined && region === undefined) {\n    return backupURL;\n  }\n\n  const url = new URL(backupURL);\n  if (url.protocol !== 's3:') {\n    return backupURL;\n  }\n\n  if (endpoint !== undefined && !url.searchParams.has('endpoint')) {\n    url.searchParams.set('endpoint', endpoint);\n  }\n  if (region !== undefined && !url.searchParams.has('region')) {\n    url.searchParams.set('region', region);\n  }\n  return url.toString();\n}\n\nexport function getVfsEnv(\n  options: VfsBackupWatermarkPollerEnvOptions,\n): NodeJS.ProcessEnv {\n  return {\n    LITESTREAM_REPLICA_URL: buildLitestreamVfsReplicaURL(options),\n    LITESTREAM_LOG_LEVEL: options.logLevel ?? DEFAULT_VFS_LOG_LEVEL,\n    LITESTREAM_LOG_FILE: options.logFile,\n\n    // Supported by the rocicorp fork: https://github.com/rocicorp/litestream/pull/20\n    // TODO: Remove and use official build when\n    // https://github.com/benbjohnson/litestream/pull/1157 is released.\n    LITESTREAM_POLL_INTERVAL: `${options.remotePollIntervalMs}ms`,\n  };\n}\n\nconst MIN_CHECK_INTERVAL_MS = 1_000;\nconst MAX_CHECK_INTERVAL_MS = 60_000;\n\nexport class VfsBackupWatermarkPoller implements Service {\n  readonly id = 'backup-watermark-poller';\n  readonly #runningState = new RunningState(this.id);\n  readonly #lc: LogContext;\n  readonly #parent: Worker;\n  readonly #loader: LoadableDatabase;\n\n  readonly #localDb: WatermarkReader<WatermarkRow>;\n  readonly #openRemote: () => WatermarkReader<WatermarkQueryResult>;\n\n  #remoteDb: WatermarkReader<WatermarkQueryResult> | null = null;\n  #lastRemoteWatermark: WatermarkQueryResult | null = null;\n\n  constructor(\n    lc: LogContext,\n    parent: Worker,\n    options: VfsBackupWatermarkPollerOptions,\n    deps: VfsBackupWatermarkPollerDeps = {},\n  ) {\n    lc = lc.withContext('component', 'vfs-watermark-poller');\n    this.#lc = lc;\n    this.#parent = parent;\n\n    const {databaseFactory} = deps;\n    const openDb = <Result extends WatermarkRow>(\n      url: string,\n      type: 'local' | 'remote',\n      schema: v.Type<Result>,\n    ) => {\n      try {\n        const db = databaseFactory\n          ? databaseFactory(lc, url)\n          : new Database(lc, url, {readonly: true});\n        const stmt = db.prepare(\n          type === 'local'\n            ? /*sql*/ `\n          SELECT stateVersion AS watermark, writeTimeMs FROM \"_zero.replicationState\"`\n            : /*sql*/ `\n          SELECT stateVersion AS watermark, writeTimeMs, litestream_time() as backupTime FROM \"_zero.replicationState\"`,\n        );\n        return {\n          getWatermark: () => v.parse(stmt.get(), schema),\n          close: () => db.close(),\n        } satisfies WatermarkReader<Result>;\n      } catch (e) {\n        lc.error?.(`error opening ${url}`, e);\n        throw e;\n      }\n    };\n\n    // Open the local (non-VFS) connection before loading the litestream VFS\n    // extension. Loading the extension registers a process-global\n    // `sqlite3_auto_extension` hook that SQLite invokes on every connection\n    // opened *afterward* in this process (even ones that don't reference\n    // `vfs=litestream`), and a signature bug in that hook currently causes\n    // it to fail such connections. Opening #localDb first sidesteps it,\n    // since already-open connections aren't affected by the hook.\n    this.#localDb = openDb(options.localDbFile, 'local', watermarkRowSchema);\n\n    const loaderFactory =\n      deps.loaderFactory ?? (() => new Sqlite3Database(':memory:'));\n\n    const dbParams = new URLSearchParams({\n      vfs: 'litestream',\n      mode: 'ro',\n      // Support forthcoming in https://github.com/benbjohnson/litestream/issues/1150\n      replica_url: buildLitestreamVfsReplicaURL(options),\n      poll_interval: `${options.remotePollIntervalMs}ms`,\n    });\n    // Note: the filename is arbitrary and only referenced in debug logs.\n    const remoteDbURL = `file:zero-backup.db?${dbParams.toString()}`;\n\n    lc.info?.(`Loading VFS ${remoteDbURL} at ${options.extensionPath}`);\n    this.#loader = loaderFactory();\n    this.#loader.loadExtension(options.extensionPath);\n\n    this.#openRemote = () =>\n      openDb(remoteDbURL, 'remote', watermarkQuerySchema);\n  }\n\n  // public for testing\n  checkWatermarks() {\n    const local = this.#localDb.getWatermark();\n    if (local.watermark !== this.#lastRemoteWatermark?.watermark) {\n      if (!this.#remoteDb) {\n        if (this.#lastRemoteWatermark) {\n          this.#lc.debug?.(\n            `local and remote watermarks differ. resuming backup polling`,\n            {local, remote: this.#lastRemoteWatermark},\n          );\n        }\n        this.#remoteDb = this.#openRemote();\n      }\n      const remote = this.#remoteDb.getWatermark();\n      if (remote.watermark !== this.#lastRemoteWatermark?.watermark) {\n        this.#lc.debug?.(\n          `backup watermark advanced from ${this.#lastRemoteWatermark?.watermark} to ${remote.watermark}`,\n          {local, remote},\n        );\n        this.#lastRemoteWatermark = remote;\n        const {watermark, writeTimeMs, backupTime} = remote;\n        // Note: litestream_time() returns \"latest\" if no LTX files were\n        // loaded:\n        //\n        // https://github.com/benbjohnson/litestream/blob/6d61ef5d007756d62e473daee4c760ac395a55c6/vfs.go#L2448\n        //\n        // but in that case the watermark query would have failed, so it\n        // follows that litestream_time() should always return a valid\n        // timestamp if the watermark query itself succeeded.\n        const backupTimeMs = new Date(backupTime).getTime();\n        this.#parent.send<BackupWatermarkUpdate>([\n          'backupWatermarkUpdate',\n          {watermark, writeTimeMs, backupTimeMs},\n        ]);\n      }\n    }\n    // Note: This is not an `else`, as we specifically want to re-check\n    // this.#lastRemoteWatermark\n    if (local.watermark === this.#lastRemoteWatermark?.watermark) {\n      if (this.#remoteDb) {\n        this.#lc.debug?.(\n          `local and remote watermarks match. pausing backup polling`,\n          {local, remote: this.#lastRemoteWatermark},\n        );\n        this.#remoteDb.close();\n        this.#remoteDb = null;\n      }\n    }\n  }\n\n  async run() {\n    this.#lc.info?.(`starting backup polling`);\n    let interval = MIN_CHECK_INTERVAL_MS;\n    while (this.#runningState.shouldRun()) {\n      try {\n        this.checkWatermarks();\n        interval = MIN_CHECK_INTERVAL_MS;\n      } catch (e) {\n        this.#lc.warn?.(`error checking local and remote watermarks`, e);\n        interval = Math.min(MAX_CHECK_INTERVAL_MS, interval * 2);\n      }\n      await this.#runningState.sleep(interval);\n    }\n    this.#remoteDb?.close();\n    this.#localDb.close();\n    this.#loader.close();\n  }\n\n  stop(): Promise<void> {\n    this.#runningState.stop(this.#lc);\n    return promiseVoid;\n  }\n}\n"],"mappings":";;;;;;;AASA,IAAM,wBAAwB;AAY9B,IAAM,qBAAqB,eAAE,OAAO;CAClC,WAAW,eAAE,OAAO;CACpB,aAAa,eAAE,OAAO;AACxB,CAAC;AAED,IAAM,uBAAuB,mBAAmB,OAAO,EACrD,YAAY,eAAE,OAAO,EACvB,CAAC;AAkDD,SAAgB,6BAA6B,EAC3C,WACA,UACA,UAKS;CACT,IAAI,aAAa,KAAA,KAAa,WAAW,KAAA,GACvC,OAAO;CAGT,MAAM,MAAM,IAAI,IAAI,SAAS;CAC7B,IAAI,IAAI,aAAa,OACnB,OAAO;CAGT,IAAI,aAAa,KAAA,KAAa,CAAC,IAAI,aAAa,IAAI,UAAU,GAC5D,IAAI,aAAa,IAAI,YAAY,QAAQ;CAE3C,IAAI,WAAW,KAAA,KAAa,CAAC,IAAI,aAAa,IAAI,QAAQ,GACxD,IAAI,aAAa,IAAI,UAAU,MAAM;CAEvC,OAAO,IAAI,SAAS;AACtB;AAEA,SAAgB,UACd,SACmB;CACnB,OAAO;EACL,wBAAwB,6BAA6B,OAAO;EAC5D,sBAAsB,QAAQ,YAAY;EAC1C,qBAAqB,QAAQ;EAK7B,0BAA0B,GAAG,QAAQ,qBAAqB;CAC5D;AACF;AAEA,IAAM,wBAAwB;AAC9B,IAAM,wBAAwB;AAE9B,IAAa,2BAAb,MAAyD;CACvD,KAAc;CACd,gBAAyB,IAAI,aAAa,KAAK,EAAE;CACjD;CACA;CACA;CAEA;CACA;CAEA,YAA0D;CAC1D,uBAAoD;CAEpD,YACE,IACA,QACA,SACA,OAAqC,CAAC,GACtC;EACA,KAAK,GAAG,YAAY,aAAa,sBAAsB;EACvD,KAAKC,MAAM;EACX,KAAKC,UAAU;EAEf,MAAM,EAAC,oBAAmB;EAC1B,MAAM,UACJ,KACA,MACA,WACG;GACH,IAAI;IACF,MAAM,KAAK,kBACP,gBAAgB,IAAI,GAAG,IACvB,IAAI,SAAS,IAAI,KAAK,EAAC,UAAU,KAAI,CAAC;IAC1C,MAAM,OAAO,GAAG,QACd,SAAS,UACG;yFAEA;uHAEd;IACA,OAAO;KACL,oBAAoB,MAAQ,KAAK,IAAI,GAAG,MAAM;KAC9C,aAAa,GAAG,MAAM;IACxB;GACF,SAAS,GAAG;IACV,GAAG,QAAQ,iBAAiB,OAAO,CAAC;IACpC,MAAM;GACR;EACF;EASA,KAAKE,WAAW,OAAO,QAAQ,aAAa,SAAS,kBAAkB;EAEvE,MAAM,gBACJ,KAAK,wBAAwB,IAAI,gBAAgB,UAAU;EAU7D,MAAM,cAAc,uBAAuB,IARtB,gBAAgB;GACnC,KAAK;GACL,MAAM;GAEN,aAAa,6BAA6B,OAAO;GACjD,eAAe,GAAG,QAAQ,qBAAqB;EACjD,CAE2C,EAAS,SAAS;EAE7D,GAAG,OAAO,eAAe,YAAY,MAAM,QAAQ,eAAe;EAClE,KAAKD,UAAU,cAAc;EAC7B,KAAKA,QAAQ,cAAc,QAAQ,aAAa;EAEhD,KAAKE,oBACH,OAAO,aAAa,UAAU,oBAAoB;CACtD;CAGA,kBAAkB;EAChB,MAAM,QAAQ,KAAKD,SAAS,aAAa;EACzC,IAAI,MAAM,cAAc,KAAKE,sBAAsB,WAAW;GAC5D,IAAI,CAAC,KAAKC,WAAW;IACnB,IAAI,KAAKD,sBACP,KAAKL,IAAI,QACP,+DACA;KAAC;KAAO,QAAQ,KAAKK;IAAoB,CAC3C;IAEF,KAAKC,YAAY,KAAKF,YAAY;GACpC;GACA,MAAM,SAAS,KAAKE,UAAU,aAAa;GAC3C,IAAI,OAAO,cAAc,KAAKD,sBAAsB,WAAW;IAC7D,KAAKL,IAAI,QACP,kCAAkC,KAAKK,sBAAsB,UAAU,MAAM,OAAO,aACpF;KAAC;KAAO;IAAM,CAChB;IACA,KAAKA,uBAAuB;IAC5B,MAAM,EAAC,WAAW,aAAa,eAAc;IAS7C,MAAM,eAAe,IAAI,KAAK,UAAU,EAAE,QAAQ;IAClD,KAAKJ,QAAQ,KAA4B,CACvC,yBACA;KAAC;KAAW;KAAa;IAAY,CACvC,CAAC;GACH;EACF;EAGA,IAAI,MAAM,cAAc,KAAKI,sBAAsB;OAC7C,KAAKC,WAAW;IAClB,KAAKN,IAAI,QACP,6DACA;KAAC;KAAO,QAAQ,KAAKK;IAAoB,CAC3C;IACA,KAAKC,UAAU,MAAM;IACrB,KAAKA,YAAY;GACnB;;CAEJ;CAEA,MAAM,MAAM;EACV,KAAKN,IAAI,OAAO,yBAAyB;EACzC,IAAI,WAAW;EACf,OAAO,KAAKD,cAAc,UAAU,GAAG;GACrC,IAAI;IACF,KAAK,gBAAgB;IACrB,WAAW;GACb,SAAS,GAAG;IACV,KAAKC,IAAI,OAAO,8CAA8C,CAAC;IAC/D,WAAW,KAAK,IAAI,uBAAuB,WAAW,CAAC;GACzD;GACA,MAAM,KAAKD,cAAc,MAAM,QAAQ;EACzC;EACA,KAAKO,WAAW,MAAM;EACtB,KAAKH,SAAS,MAAM;EACpB,KAAKD,QAAQ,MAAM;CACrB;CAEA,OAAsB;EACpB,KAAKH,cAAc,KAAK,KAAKC,GAAG;EAChC,OAAO;CACT;AACF"}

package/out/zero-cache/src/services/replicator/change-log-cookies.d.ts

@@ -0,0 +1,169 @@
+/**
+ * The change log's *second* job.
+ *
+ * A change log is a buffer **and** a cookie jar. The buffer catches a
+ * reconnecting subscriber up to the head; the cookie jar holds the backfill
+ * progress state that the change-streamer hands the change-source on every
+ * `startStream`, so that the change source never has to persist backfill state
+ * upstream. See `change-log-db.ts` for the buffer.
+ *
+ * There are two cookies, mirroring the two `cdc` tables the Postgres change log
+ * has always carried (`change-streamer/schema/tables.ts`):
+ *
+ * - **table metadata**, the most current {@link TableMetadata} of a table, which
+ *   a {@link BackfillRequest} has to carry; and
+ * - **backfilling columns**, the opaque {@link BackfillID} of every column whose
+ *   backfill has not completed.
+ *
+ * Both are a *fold over the change stream*: the cookie set is only meaningful
+ * paired with the watermark it was folded up to. {@link cookieOps} is that fold,
+ * expressed transport-free so that the Postgres change log and the SQLite change
+ * log cannot drift from each other — they must agree on all eight transitions
+ * forever, and the failure mode if they disagree is a backfill that is silently
+ * never re-requested. The two interpreters are {@link ChangeLogCookieWriter}
+ * here and `Storer.#trackBackfillMetadata()` in
+ * `change-streamer/storer.ts`.
+ *
+ * The tables live in the change-log database rather than in the replica: the
+ * replicator is a subscriber *downstream* of the change-streamer's write loop,
+ * so its state version trails the log's head, and pairing its cookies with the
+ * log's resume watermark would lose every backfill that completed in between.
+ * The replica's own copies (`_zero.tableMetadata`, `_zero.column_metadata`) are
+ * the *seed* source; these are the *working* set.
+ */
+import type { Database } from '../../../../zqlite/src/db.ts';
+import type { BackfillID, Identifier, SchemaChange, TableMetadata } from '../change-source/protocol/current/data.ts';
+import { type BackfillRequest } from '../change-source/protocol/current/upstream.ts';
+export declare const CHANGE_LOG_TABLE_METADATA_TABLE = "_zero.changeLogTableMetadata";
+export declare const CHANGE_LOG_BACKFILLING_TABLE = "_zero.changeLogBackfilling";
+export declare const CREATE_CHANGE_LOG_COOKIE_SCHEMA = "\n  CREATE TABLE \"_zero.changeLogTableMetadata\" (\n    \"schema\"   TEXT NOT NULL,\n    \"table\"    TEXT NOT NULL,\n    \"metadata\" TEXT NOT NULL,\n    PRIMARY KEY (\"schema\", \"table\")\n  );\n\n  CREATE TABLE \"_zero.changeLogBackfilling\" (\n    \"schema\"   TEXT NOT NULL,\n    \"table\"    TEXT NOT NULL,\n    \"column\"   TEXT NOT NULL,\n    \"backfill\" TEXT NOT NULL,\n    PRIMARY KEY (\"schema\", \"table\", \"column\")\n  );\n";
+export declare const DROP_CHANGE_LOG_COOKIE_TABLES = "\n  DROP TABLE IF EXISTS \"_zero.changeLogTableMetadata\";\n  DROP TABLE IF EXISTS \"_zero.changeLogBackfilling\";\n";
+export type TableMetadataCookie = {
+    readonly schema: string;
+    readonly table: string;
+    readonly metadata: TableMetadata;
+};
+export type BackfillCookie = {
+    readonly schema: string;
+    readonly table: string;
+    readonly column: string;
+    readonly backfill: BackfillID;
+};
+/**
+ * The whole cookie set, as of the log's head. Both lists are ordered by their
+ * primary key so that two stores' sets can be compared directly.
+ */
+export type CookieSet = {
+    readonly tableMetadata: readonly TableMetadataCookie[];
+    readonly backfilling: readonly BackfillCookie[];
+};
+export declare const EMPTY_COOKIE_SET: CookieSet;
+/**
+ * A transport-free description of a cookie mutation. This is the whole of what
+ * a {@link SchemaChange} means to the cookie jar; everything else about the
+ * change belongs to the buffer.
+ */
+export type CookieOp = {
+    op: 'upsert-metadata';
+    table: Identifier;
+    metadata: TableMetadata;
+} | {
+    op: 'upsert-backfill';
+    table: Identifier;
+    column: string;
+    backfill: BackfillID;
+} | {
+    op: 'rename-table';
+    old: Identifier;
+    new: Identifier;
+} | {
+    op: 'drop-table';
+    table: Identifier;
+} | {
+    op: 'rename-column';
+    table: Identifier;
+    old: string;
+    new: string;
+} | {
+    op: 'drop-column';
+    table: Identifier;
+    column: string;
+} | {
+    op: 'complete-backfill';
+    table: Identifier;
+    columns: readonly string[];
+};
+export type CookieOpTag = CookieOp['op'];
+/**
+ * The fold, and the only place either store decides what a schema change means
+ * for the cookie set. Both interpreters are mechanical translations of what this
+ * returns, in the order it returns them.
+ *
+ * Changes that carry no cookie state — index changes, and the `create-table` /
+ * `add-column` variants whose optional `metadata` and `backfill` fields are
+ * absent — produce no ops.
+ */
+export declare function cookieOps(change: SchemaChange): CookieOp[];
+/**
+ * The SQLite interpreter of {@link cookieOps}, over prepared statements on the
+ * change-log database.
+ *
+ * The statements run inside the log's transaction, in stream order, at the point
+ * the schema change is appended (see `ChangeLogStreamWriter`). That is what
+ * makes the cookie set atomic with the head — a crash cannot leave cookies ahead
+ * of or behind the rows they were folded from — and what makes an aborted
+ * transaction discard its cookies with its rows, for free.
+ *
+ * Statements are prepared against the tables as they exist now, so an instance
+ * does not survive a reconcile that drops and recreates them.
+ */
+export declare class ChangeLogCookieWriter {
+    #private;
+    constructor(db: Database);
+    /** Applies the change's cookie ops, returning the ops that were applied. */
+    apply(change: SchemaChange): CookieOp[];
+}
+/**
+ * The in-memory interpreter of {@link cookieOps}: the third one, and the only
+ * one that is not a store.
+ *
+ * A replica-derived cookie set is a snapshot at the replica's state version,
+ * which trails the change log's head, so it cannot be compared with a
+ * Postgres-derived set directly. This carries it forward over the schema changes
+ * the log holds in between, which turns "the two stores disagree, but they are
+ * also at different positions" into an exact equality — and makes a
+ * disagreement between any two of the three interpreters visible as an
+ * inequality that neither store would have noticed on its own.
+ *
+ * `changes` must be the log's schema changes over `(from, to]` in stream order.
+ * Anything else produces a set that belongs to no watermark.
+ */
+export declare function foldCookies(cookies: CookieSet, changes: Iterable<SchemaChange>): CookieSet;
+/**
+ * Returns one {@link BackfillRequest} for each table with an active backfill.
+ * Each request contains the stored table metadata, if available.
+ */
+export declare function backfillRequestsFrom(cookies: CookieSet): BackfillRequest[];
+/**
+ * Reads the whole cookie set, ordered by primary key. This is what a
+ * change-streamer would hand `startStream`, and what the Postgres-derived and
+ * replica-derived sets are compared against.
+ */
+export declare function readCookies(db: Database): CookieSet;
+/**
+ * Replaces the cookie set wholesale.
+ *
+ * The cookies are unversioned point-in-time state, so a reconciliation that
+ * moves the head invalidates them: truncating the stream above `W` does not roll
+ * the cookie set back to `W`, and the truncated transactions can contain
+ * `create-table`, `add-column`, and `backfill-completed`. The caller must run
+ * this in the same transaction as the reconciliation that invalidated them.
+ */
+export declare function replaceCookies(db: Database, cookies: CookieSet): void;
+/** Retained cookie rows, by table. Both are normally zero. */
+export type CookieRowCounts = {
+    readonly tableMetadata: number;
+    readonly backfilling: number;
+};
+export declare function readCookieRowCounts(db: Database): CookieRowCounts;
+//# sourceMappingURL=change-log-cookies.d.ts.map

package/out/zero-cache/src/services/replicator/change-log-cookies.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"change-log-cookies.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/change-log-cookies.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAKH,OAAO,KAAK,EAAC,QAAQ,EAAY,MAAM,8BAA8B,CAAC;AACtE,OAAO,KAAK,EACV,UAAU,EACV,UAAU,EACV,YAAY,EACZ,aAAa,EACd,MAAM,2CAA2C,CAAC;AACnD,OAAO,EAEL,KAAK,eAAe,EACrB,MAAM,+CAA+C,CAAC;AAEvD,eAAO,MAAM,+BAA+B,iCAAiC,CAAC;AAC9E,eAAO,MAAM,4BAA4B,+BAA+B,CAAC;AAQzE,eAAO,MAAM,+BAA+B,8bAe3C,CAAC;AAEF,eAAO,MAAM,6BAA6B,yHAGzC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAChC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC;CAClC,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;CAC/B,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,SAAS,GAAG;IACtB,QAAQ,CAAC,aAAa,EAAE,SAAS,mBAAmB,EAAE,CAAC;IACvD,QAAQ,CAAC,WAAW,EAAE,SAAS,cAAc,EAAE,CAAC;CACjD,CAAC;AAEF,eAAO,MAAM,gBAAgB,EAAE,SAG9B,CAAC;AAEF;;;;GAIG;AACH,MAAM,MAAM,QAAQ,GAChB;IAAC,EAAE,EAAE,iBAAiB,CAAC;IAAC,KAAK,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,aAAa,CAAA;CAAC,GACnE;IACE,EAAE,EAAE,iBAAiB,CAAC;IACtB,KAAK,EAAE,UAAU,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,UAAU,CAAC;CACtB,GACD;IAAC,EAAE,EAAE,cAAc,CAAC;IAAC,GAAG,EAAE,UAAU,CAAC;IAAC,GAAG,EAAE,UAAU,CAAA;CAAC,GACtD;IAAC,EAAE,EAAE,YAAY,CAAC;IAAC,KAAK,EAAE,UAAU,CAAA;CAAC,GACrC;IAAC,EAAE,EAAE,eAAe,CAAC;IAAC,KAAK,EAAE,UAAU,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAC,GAClE;IAAC,EAAE,EAAE,aAAa,CAAC;IAAC,KAAK,EAAE,UAAU,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAC,GACtD;IAAC,EAAE,EAAE,mBAAmB,CAAC;IAAC,KAAK,EAAE,UAAU,CAAC;IAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAA;CAAC,CAAC;AAE7E,MAAM,MAAM,WAAW,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;AAEzC;;;;;;;;GAQG;AACH,wBAAgB,SAAS,CAAC,MAAM,EAAE,YAAY,GAAG,QAAQ,EAAE,CAmF1D;AAED;;;;;;;;;;;;GAYG;AACH,qBAAa,qBAAqB;;gBAUpB,EAAE,EAAE,QAAQ;IAuCxB,4EAA4E;IAC5E,KAAK,CAAC,MAAM,EAAE,YAAY,GAAG,QAAQ,EAAE;CAiFxC;AAUD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,WAAW,CACzB,OAAO,EAAE,SAAS,EAClB,OAAO,EAAE,QAAQ,CAAC,YAAY,CAAC,GAC9B,SAAS,CA6GX;AAID;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,SAAS,GAAG,eAAe,EAAE,CAuB1E;AAED;;;;GAIG;AACH,wBAAgB,WAAW,CAAC,EAAE,EAAE,QAAQ,GAAG,SAAS,CA6BnD;AAED;;;;;;;;GAQG;AACH,wBAAgB,cAAc,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,GAAG,IAAI,CAsBrE;AAED,8DAA8D;AAC9D,MAAM,MAAM,eAAe,GAAG;IAC5B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B,CAAC;AAEF,wBAAgB,mBAAmB,CAAC,EAAE,EAAE,QAAQ,GAAG,eAAe,CAUjE"}