@rocicorp/zero 1.10.0-canary.2 → 1.10.0-canary.20

package/out/zero-cache/src/services/litestream/vfs-watermark-poller.js

@@ -1,137 +0,0 @@
-import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
-import { parse, valita_exports } from "../../../../shared/src/valita.js";
-import "../../types/processes.js";
-import { Database } from "../../../../zqlite/src/db.js";
-import { RunningState } from "../running-state.js";
-import SQLite3Database from "@rocicorp/zero-sqlite3";
-//#region ../zero-cache/src/services/litestream/vfs-watermark-poller.ts
-var DEFAULT_VFS_LOG_LEVEL = "info";
-var watermarkRowSchema = valita_exports.object({
-	watermark: valita_exports.string(),
-	writeTimeMs: valita_exports.number()
-});
-var watermarkQuerySchema = watermarkRowSchema.extend({ backupTime: valita_exports.string() });
-function buildLitestreamVfsReplicaURL({ backupURL, endpoint, region }) {
-	if (endpoint === void 0 && region === void 0) return backupURL;
-	const url = new URL(backupURL);
-	if (url.protocol !== "s3:") return backupURL;
-	if (endpoint !== void 0 && !url.searchParams.has("endpoint")) url.searchParams.set("endpoint", endpoint);
-	if (region !== void 0 && !url.searchParams.has("region")) url.searchParams.set("region", region);
-	return url.toString();
-}
-function getVfsEnv(options) {
-	return {
-		LITESTREAM_REPLICA_URL: buildLitestreamVfsReplicaURL(options),
-		LITESTREAM_LOG_LEVEL: options.logLevel ?? DEFAULT_VFS_LOG_LEVEL,
-		LITESTREAM_LOG_FILE: options.logFile,
-		LITESTREAM_POLL_INTERVAL: `${options.remotePollIntervalMs}ms`
-	};
-}
-var MIN_CHECK_INTERVAL_MS = 1e3;
-var MAX_CHECK_INTERVAL_MS = 6e4;
-var VfsBackupWatermarkPoller = class {
-	id = "backup-watermark-poller";
-	#runningState = new RunningState(this.id);
-	#lc;
-	#parent;
-	#loader;
-	#localDb;
-	#openRemote;
-	#remoteDb = null;
-	#lastRemoteWatermark = null;
-	constructor(lc, parent, options, deps = {}) {
-		lc = lc.withContext("component", "vfs-watermark-poller");
-		this.#lc = lc;
-		this.#parent = parent;
-		const { databaseFactory } = deps;
-		const openDb = (url, type, schema) => {
-			try {
-				const db = databaseFactory ? databaseFactory(lc, url) : new Database(lc, url, { readonly: true });
-				const stmt = db.prepare(type === "local" ? `
-          SELECT stateVersion AS watermark, writeTimeMs FROM "_zero.replicationState"` : `
-          SELECT stateVersion AS watermark, writeTimeMs, litestream_time() as backupTime FROM "_zero.replicationState"`);
-				return {
-					getWatermark: () => parse(stmt.get(), schema),
-					close: () => db.close()
-				};
-			} catch (e) {
-				lc.error?.(`error opening ${url}`, e);
-				throw e;
-			}
-		};
-		this.#localDb = openDb(options.localDbFile, "local", watermarkRowSchema);
-		const loaderFactory = deps.loaderFactory ?? (() => new SQLite3Database(":memory:"));
-		const remoteDbURL = `file:zero-backup.db?${new URLSearchParams({
-			vfs: "litestream",
-			mode: "ro",
-			replica_url: buildLitestreamVfsReplicaURL(options),
-			poll_interval: `${options.remotePollIntervalMs}ms`
-		}).toString()}`;
-		lc.info?.(`Loading VFS ${remoteDbURL} at ${options.extensionPath}`);
-		this.#loader = loaderFactory();
-		this.#loader.loadExtension(options.extensionPath);
-		this.#openRemote = () => openDb(remoteDbURL, "remote", watermarkQuerySchema);
-	}
-	checkWatermarks() {
-		const local = this.#localDb.getWatermark();
-		if (local.watermark !== this.#lastRemoteWatermark?.watermark) {
-			if (!this.#remoteDb) {
-				if (this.#lastRemoteWatermark) this.#lc.debug?.(`local and remote watermarks differ. resuming backup polling`, {
-					local,
-					remote: this.#lastRemoteWatermark
-				});
-				this.#remoteDb = this.#openRemote();
-			}
-			const remote = this.#remoteDb.getWatermark();
-			if (remote.watermark !== this.#lastRemoteWatermark?.watermark) {
-				this.#lc.debug?.(`backup watermark advanced from ${this.#lastRemoteWatermark?.watermark} to ${remote.watermark}`, {
-					local,
-					remote
-				});
-				this.#lastRemoteWatermark = remote;
-				const { watermark, writeTimeMs, backupTime } = remote;
-				const backupTimeMs = new Date(backupTime).getTime();
-				this.#parent.send(["backupWatermarkUpdate", {
-					watermark,
-					writeTimeMs,
-					backupTimeMs
-				}]);
-			}
-		}
-		if (local.watermark === this.#lastRemoteWatermark?.watermark) {
-			if (this.#remoteDb) {
-				this.#lc.debug?.(`local and remote watermarks match. pausing backup polling`, {
-					local,
-					remote: this.#lastRemoteWatermark
-				});
-				this.#remoteDb.close();
-				this.#remoteDb = null;
-			}
-		}
-	}
-	async run() {
-		this.#lc.info?.(`starting backup polling`);
-		let interval = MIN_CHECK_INTERVAL_MS;
-		while (this.#runningState.shouldRun()) {
-			try {
-				this.checkWatermarks();
-				interval = MIN_CHECK_INTERVAL_MS;
-			} catch (e) {
-				this.#lc.warn?.(`error checking local and remote watermarks`, e);
-				interval = Math.min(MAX_CHECK_INTERVAL_MS, interval * 2);
-			}
-			await this.#runningState.sleep(interval);
-		}
-		this.#remoteDb?.close();
-		this.#localDb.close();
-		this.#loader.close();
-	}
-	stop() {
-		this.#runningState.stop(this.#lc);
-		return promiseVoid;
-	}
-};
-//#endregion
-export { VfsBackupWatermarkPoller, getVfsEnv };
-
-//# sourceMappingURL=vfs-watermark-poller.js.map

package/out/zero-cache/src/services/litestream/vfs-watermark-poller.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"vfs-watermark-poller.js","names":["#runningState","#lc","#parent","#loader","#localDb","#openRemote","#lastRemoteWatermark","#remoteDb"],"sources":["../../../../../../zero-cache/src/services/litestream/vfs-watermark-poller.ts"],"sourcesContent":["import type {LogContext, LogLevel} from '@rocicorp/logger';\nimport Sqlite3Database from '@rocicorp/zero-sqlite3';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {type Worker} from '../../types/processes.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {Service} from '../service.ts';\n\nconst DEFAULT_VFS_LOG_LEVEL = 'info';\n\ntype LoadableDatabase = {\n  loadExtension(path: string): unknown;\n  close(): void;\n};\n\ntype QueryDatabase = {\n  prepare(sql: string): {get<T>(): T};\n  close(): void;\n};\n\nconst watermarkRowSchema = v.object({\n  watermark: v.string(),\n  writeTimeMs: v.number(),\n});\n\nconst watermarkQuerySchema = watermarkRowSchema.extend({\n  backupTime: v.string(),\n});\n\ntype WatermarkQueryResult = v.Infer<typeof watermarkQuerySchema>;\n\ntype WatermarkRow = v.Infer<typeof watermarkRowSchema>;\n\nexport type BackupWatermarkUpdate = [\n  'backupWatermarkUpdate',\n  {\n    watermark: string;\n    writeTimeMs: number;\n    backupTimeMs: number;\n  },\n];\n\ntype WatermarkReader<Result extends WatermarkRow> = {\n  getWatermark(): Result;\n  close(): void;\n};\n\n// Values that must be passed via the ENV to the original process (worker).\n// Some of these will be configurable via the db URI once\n// https://github.com/benbjohnson/litestream/issues/1150 is released.\nexport type VfsBackupWatermarkPollerEnvOptions = {\n  readonly backupURL: string;\n  readonly remotePollIntervalMs: number;\n  readonly endpoint?: string | undefined;\n  readonly region?: string | undefined;\n\n  readonly logLevel?: LogLevel | undefined;\n  readonly logFile?: string | undefined;\n};\n\nexport type VfsBackupWatermarkPollerOptions = {\n  readonly localDbFile: string;\n  readonly backupURL: string;\n  readonly endpoint?: string | undefined;\n  readonly region?: string | undefined;\n\n  readonly extensionPath: string;\n  readonly remotePollIntervalMs: number;\n};\n\ntype VfsBackupWatermarkPollerDeps = {\n  readonly loaderFactory?: (() => LoadableDatabase) | undefined;\n  readonly databaseFactory?:\n    | ((lc: LogContext, uri: string) => QueryDatabase)\n    | undefined;\n};\n\nexport function buildLitestreamVfsReplicaURL({\n  backupURL,\n  endpoint,\n  region,\n}: {\n  readonly backupURL: string;\n  readonly endpoint?: string | undefined;\n  readonly region?: string | undefined;\n}): string {\n  if (endpoint === undefined && region === undefined) {\n    return backupURL;\n  }\n\n  const url = new URL(backupURL);\n  if (url.protocol !== 's3:') {\n    return backupURL;\n  }\n\n  if (endpoint !== undefined && !url.searchParams.has('endpoint')) {\n    url.searchParams.set('endpoint', endpoint);\n  }\n  if (region !== undefined && !url.searchParams.has('region')) {\n    url.searchParams.set('region', region);\n  }\n  return url.toString();\n}\n\nexport function getVfsEnv(\n  options: VfsBackupWatermarkPollerEnvOptions,\n): NodeJS.ProcessEnv {\n  return {\n    LITESTREAM_REPLICA_URL: buildLitestreamVfsReplicaURL(options),\n    LITESTREAM_LOG_LEVEL: options.logLevel ?? DEFAULT_VFS_LOG_LEVEL,\n    LITESTREAM_LOG_FILE: options.logFile,\n\n    // Supported by the rocicorp fork: https://github.com/rocicorp/litestream/pull/20\n    // TODO: Remove and use official build when\n    // https://github.com/benbjohnson/litestream/pull/1157 is released.\n    LITESTREAM_POLL_INTERVAL: `${options.remotePollIntervalMs}ms`,\n  };\n}\n\nconst MIN_CHECK_INTERVAL_MS = 1_000;\nconst MAX_CHECK_INTERVAL_MS = 60_000;\n\nexport class VfsBackupWatermarkPoller implements Service {\n  readonly id = 'backup-watermark-poller';\n  readonly #runningState = new RunningState(this.id);\n  readonly #lc: LogContext;\n  readonly #parent: Worker;\n  readonly #loader: LoadableDatabase;\n\n  readonly #localDb: WatermarkReader<WatermarkRow>;\n  readonly #openRemote: () => WatermarkReader<WatermarkQueryResult>;\n\n  #remoteDb: WatermarkReader<WatermarkQueryResult> | null = null;\n  #lastRemoteWatermark: WatermarkQueryResult | null = null;\n\n  constructor(\n    lc: LogContext,\n    parent: Worker,\n    options: VfsBackupWatermarkPollerOptions,\n    deps: VfsBackupWatermarkPollerDeps = {},\n  ) {\n    lc = lc.withContext('component', 'vfs-watermark-poller');\n    this.#lc = lc;\n    this.#parent = parent;\n\n    const {databaseFactory} = deps;\n    const openDb = <Result extends WatermarkRow>(\n      url: string,\n      type: 'local' | 'remote',\n      schema: v.Type<Result>,\n    ) => {\n      try {\n        const db = databaseFactory\n          ? databaseFactory(lc, url)\n          : new Database(lc, url, {readonly: true});\n        const stmt = db.prepare(\n          type === 'local'\n            ? /*sql*/ `\n          SELECT stateVersion AS watermark, writeTimeMs FROM \"_zero.replicationState\"`\n            : /*sql*/ `\n          SELECT stateVersion AS watermark, writeTimeMs, litestream_time() as backupTime FROM \"_zero.replicationState\"`,\n        );\n        return {\n          getWatermark: () => v.parse(stmt.get(), schema),\n          close: () => db.close(),\n        } satisfies WatermarkReader<Result>;\n      } catch (e) {\n        lc.error?.(`error opening ${url}`, e);\n        throw e;\n      }\n    };\n\n    // Open the local (non-VFS) connection before loading the litestream VFS\n    // extension. Loading the extension registers a process-global\n    // `sqlite3_auto_extension` hook that SQLite invokes on every connection\n    // opened *afterward* in this process (even ones that don't reference\n    // `vfs=litestream`), and a signature bug in that hook currently causes\n    // it to fail such connections. Opening #localDb first sidesteps it,\n    // since already-open connections aren't affected by the hook.\n    this.#localDb = openDb(options.localDbFile, 'local', watermarkRowSchema);\n\n    const loaderFactory =\n      deps.loaderFactory ?? (() => new Sqlite3Database(':memory:'));\n\n    const dbParams = new URLSearchParams({\n      vfs: 'litestream',\n      mode: 'ro',\n      // Support forthcoming in https://github.com/benbjohnson/litestream/issues/1150\n      replica_url: buildLitestreamVfsReplicaURL(options),\n      poll_interval: `${options.remotePollIntervalMs}ms`,\n    });\n    // Note: the filename is arbitrary and only referenced in debug logs.\n    const remoteDbURL = `file:zero-backup.db?${dbParams.toString()}`;\n\n    lc.info?.(`Loading VFS ${remoteDbURL} at ${options.extensionPath}`);\n    this.#loader = loaderFactory();\n    this.#loader.loadExtension(options.extensionPath);\n\n    this.#openRemote = () =>\n      openDb(remoteDbURL, 'remote', watermarkQuerySchema);\n  }\n\n  // public for testing\n  checkWatermarks() {\n    const local = this.#localDb.getWatermark();\n    if (local.watermark !== this.#lastRemoteWatermark?.watermark) {\n      if (!this.#remoteDb) {\n        if (this.#lastRemoteWatermark) {\n          this.#lc.debug?.(\n            `local and remote watermarks differ. resuming backup polling`,\n            {local, remote: this.#lastRemoteWatermark},\n          );\n        }\n        this.#remoteDb = this.#openRemote();\n      }\n      const remote = this.#remoteDb.getWatermark();\n      if (remote.watermark !== this.#lastRemoteWatermark?.watermark) {\n        this.#lc.debug?.(\n          `backup watermark advanced from ${this.#lastRemoteWatermark?.watermark} to ${remote.watermark}`,\n          {local, remote},\n        );\n        this.#lastRemoteWatermark = remote;\n        const {watermark, writeTimeMs, backupTime} = remote;\n        // Note: litestream_time() returns \"latest\" if no LTX files were\n        // loaded:\n        //\n        // https://github.com/benbjohnson/litestream/blob/6d61ef5d007756d62e473daee4c760ac395a55c6/vfs.go#L2448\n        //\n        // but in that case the watermark query would have failed, so it\n        // follows that litestream_time() should always return a valid\n        // timestamp if the watermark query itself succeeded.\n        const backupTimeMs = new Date(backupTime).getTime();\n        this.#parent.send<BackupWatermarkUpdate>([\n          'backupWatermarkUpdate',\n          {watermark, writeTimeMs, backupTimeMs},\n        ]);\n      }\n    }\n    // Note: This is not an `else`, as we specifically want to re-check\n    // this.#lastRemoteWatermark\n    if (local.watermark === this.#lastRemoteWatermark?.watermark) {\n      if (this.#remoteDb) {\n        this.#lc.debug?.(\n          `local and remote watermarks match. pausing backup polling`,\n          {local, remote: this.#lastRemoteWatermark},\n        );\n        this.#remoteDb.close();\n        this.#remoteDb = null;\n      }\n    }\n  }\n\n  async run() {\n    this.#lc.info?.(`starting backup polling`);\n    let interval = MIN_CHECK_INTERVAL_MS;\n    while (this.#runningState.shouldRun()) {\n      try {\n        this.checkWatermarks();\n        interval = MIN_CHECK_INTERVAL_MS;\n      } catch (e) {\n        this.#lc.warn?.(`error checking local and remote watermarks`, e);\n        interval = Math.min(MAX_CHECK_INTERVAL_MS, interval * 2);\n      }\n      await this.#runningState.sleep(interval);\n    }\n    this.#remoteDb?.close();\n    this.#localDb.close();\n    this.#loader.close();\n  }\n\n  stop(): Promise<void> {\n    this.#runningState.stop(this.#lc);\n    return promiseVoid;\n  }\n}\n"],"mappings":";;;;;;;AASA,IAAM,wBAAwB;AAY9B,IAAM,qBAAqB,eAAE,OAAO;CAClC,WAAW,eAAE,OAAO;CACpB,aAAa,eAAE,OAAO;AACxB,CAAC;AAED,IAAM,uBAAuB,mBAAmB,OAAO,EACrD,YAAY,eAAE,OAAO,EACvB,CAAC;AAkDD,SAAgB,6BAA6B,EAC3C,WACA,UACA,UAKS;CACT,IAAI,aAAa,KAAA,KAAa,WAAW,KAAA,GACvC,OAAO;CAGT,MAAM,MAAM,IAAI,IAAI,SAAS;CAC7B,IAAI,IAAI,aAAa,OACnB,OAAO;CAGT,IAAI,aAAa,KAAA,KAAa,CAAC,IAAI,aAAa,IAAI,UAAU,GAC5D,IAAI,aAAa,IAAI,YAAY,QAAQ;CAE3C,IAAI,WAAW,KAAA,KAAa,CAAC,IAAI,aAAa,IAAI,QAAQ,GACxD,IAAI,aAAa,IAAI,UAAU,MAAM;CAEvC,OAAO,IAAI,SAAS;AACtB;AAEA,SAAgB,UACd,SACmB;CACnB,OAAO;EACL,wBAAwB,6BAA6B,OAAO;EAC5D,sBAAsB,QAAQ,YAAY;EAC1C,qBAAqB,QAAQ;EAK7B,0BAA0B,GAAG,QAAQ,qBAAqB;CAC5D;AACF;AAEA,IAAM,wBAAwB;AAC9B,IAAM,wBAAwB;AAE9B,IAAa,2BAAb,MAAyD;CACvD,KAAc;CACd,gBAAyB,IAAI,aAAa,KAAK,EAAE;CACjD;CACA;CACA;CAEA;CACA;CAEA,YAA0D;CAC1D,uBAAoD;CAEpD,YACE,IACA,QACA,SACA,OAAqC,CAAC,GACtC;EACA,KAAK,GAAG,YAAY,aAAa,sBAAsB;EACvD,KAAKC,MAAM;EACX,KAAKC,UAAU;EAEf,MAAM,EAAC,oBAAmB;EAC1B,MAAM,UACJ,KACA,MACA,WACG;GACH,IAAI;IACF,MAAM,KAAK,kBACP,gBAAgB,IAAI,GAAG,IACvB,IAAI,SAAS,IAAI,KAAK,EAAC,UAAU,KAAI,CAAC;IAC1C,MAAM,OAAO,GAAG,QACd,SAAS,UACG;yFAEA;uHAEd;IACA,OAAO;KACL,oBAAoB,MAAQ,KAAK,IAAI,GAAG,MAAM;KAC9C,aAAa,GAAG,MAAM;IACxB;GACF,SAAS,GAAG;IACV,GAAG,QAAQ,iBAAiB,OAAO,CAAC;IACpC,MAAM;GACR;EACF;EASA,KAAKE,WAAW,OAAO,QAAQ,aAAa,SAAS,kBAAkB;EAEvE,MAAM,gBACJ,KAAK,wBAAwB,IAAI,gBAAgB,UAAU;EAU7D,MAAM,cAAc,uBAAuB,IARtB,gBAAgB;GACnC,KAAK;GACL,MAAM;GAEN,aAAa,6BAA6B,OAAO;GACjD,eAAe,GAAG,QAAQ,qBAAqB;EACjD,CAE2C,EAAS,SAAS;EAE7D,GAAG,OAAO,eAAe,YAAY,MAAM,QAAQ,eAAe;EAClE,KAAKD,UAAU,cAAc;EAC7B,KAAKA,QAAQ,cAAc,QAAQ,aAAa;EAEhD,KAAKE,oBACH,OAAO,aAAa,UAAU,oBAAoB;CACtD;CAGA,kBAAkB;EAChB,MAAM,QAAQ,KAAKD,SAAS,aAAa;EACzC,IAAI,MAAM,cAAc,KAAKE,sBAAsB,WAAW;GAC5D,IAAI,CAAC,KAAKC,WAAW;IACnB,IAAI,KAAKD,sBACP,KAAKL,IAAI,QACP,+DACA;KAAC;KAAO,QAAQ,KAAKK;IAAoB,CAC3C;IAEF,KAAKC,YAAY,KAAKF,YAAY;GACpC;GACA,MAAM,SAAS,KAAKE,UAAU,aAAa;GAC3C,IAAI,OAAO,cAAc,KAAKD,sBAAsB,WAAW;IAC7D,KAAKL,IAAI,QACP,kCAAkC,KAAKK,sBAAsB,UAAU,MAAM,OAAO,aACpF;KAAC;KAAO;IAAM,CAChB;IACA,KAAKA,uBAAuB;IAC5B,MAAM,EAAC,WAAW,aAAa,eAAc;IAS7C,MAAM,eAAe,IAAI,KAAK,UAAU,EAAE,QAAQ;IAClD,KAAKJ,QAAQ,KAA4B,CACvC,yBACA;KAAC;KAAW;KAAa;IAAY,CACvC,CAAC;GACH;EACF;EAGA,IAAI,MAAM,cAAc,KAAKI,sBAAsB;OAC7C,KAAKC,WAAW;IAClB,KAAKN,IAAI,QACP,6DACA;KAAC;KAAO,QAAQ,KAAKK;IAAoB,CAC3C;IACA,KAAKC,UAAU,MAAM;IACrB,KAAKA,YAAY;GACnB;;CAEJ;CAEA,MAAM,MAAM;EACV,KAAKN,IAAI,OAAO,yBAAyB;EACzC,IAAI,WAAW;EACf,OAAO,KAAKD,cAAc,UAAU,GAAG;GACrC,IAAI;IACF,KAAK,gBAAgB;IACrB,WAAW;GACb,SAAS,GAAG;IACV,KAAKC,IAAI,OAAO,8CAA8C,CAAC;IAC/D,WAAW,KAAK,IAAI,uBAAuB,WAAW,CAAC;GACzD;GACA,MAAM,KAAKD,cAAc,MAAM,QAAQ;EACzC;EACA,KAAKO,WAAW,MAAM;EACtB,KAAKH,SAAS,MAAM;EACpB,KAAKD,QAAQ,MAAM;CACrB;CAEA,OAAsB;EACpB,KAAKH,cAAc,KAAK,KAAKC,GAAG;EAChC,OAAO;CACT;AACF"}