@rocicorp/zero 1.9.1-canary.1 → 1.10.0-canary.1

package/out/zero-cache/src/services/change-source/protocol/current/downstream.d.ts

@@ -1,7 +1,7 @@
 import * as v from '../../../../../../shared/src/valita.ts';
 declare const begin: v.TupleType<[v.Type<"begin">, v.ObjectType<{
     tag: v.Type<"begin">;
-    json: v.Optional<"p" | "s">;
+    json: v.Optional<"s" | "p">;
     skipAck: v.Optional<boolean>;
 }, undefined>, v.ObjectType<{
     commitWatermark: v.Type<string>;
@@ -235,7 +235,7 @@ export type Commit = v.Infer<typeof commit>;
 export type Rollback = v.Infer<typeof rollback>;
 export declare const changeStreamDataSchema: v.UnionType<[v.TupleType<[v.Type<"begin">, v.ObjectType<{
     tag: v.Type<"begin">;
-    json: v.Optional<"p" | "s">;
+    json: v.Optional<"s" | "p">;
     skipAck: v.Optional<boolean>;
 }, undefined>, v.ObjectType<{
     commitWatermark: v.Type<string>;
@@ -470,7 +470,7 @@ export type ChangeStreamControl = v.Infer<typeof changeStreamControlSchema>;
 /** Downstream messages consist of data plane and control plane messages. */
 export declare const changeStreamMessageSchema: v.UnionType<[v.UnionType<[v.TupleType<[v.Type<"begin">, v.ObjectType<{
     tag: v.Type<"begin">;
-    json: v.Optional<"p" | "s">;
+    json: v.Optional<"s" | "p">;
     skipAck: v.Optional<boolean>;
 }, undefined>, v.ObjectType<{
     commitWatermark: v.Type<string>;

package/out/zero-cache/src/services/change-source/protocol/current/schema-change-tags.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Kept in a module that is not re-exported by the public change protocol so
+ * internal consumers can share the tag list without expanding the public API.
+ */
+export declare const schemaChangeTags: readonly ["create-table", "rename-table", "update-table-metadata", "add-column", "update-column", "drop-column", "drop-table", "create-index", "drop-index", "backfill-completed"];
+//# sourceMappingURL=schema-change-tags.d.ts.map

package/out/zero-cache/src/services/change-source/protocol/current/schema-change-tags.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema-change-tags.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/schema-change-tags.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,eAAO,MAAM,gBAAgB,oLAWnB,CAAC"}

package/out/zero-cache/src/services/change-source/protocol/current/schema-change-tags.js

@@ -0,0 +1,21 @@
+//#region ../zero-cache/src/services/change-source/protocol/current/schema-change-tags.ts
+/**
+* Kept in a module that is not re-exported by the public change protocol so
+* internal consumers can share the tag list without expanding the public API.
+*/
+var schemaChangeTags = [
+	"create-table",
+	"rename-table",
+	"update-table-metadata",
+	"add-column",
+	"update-column",
+	"drop-column",
+	"drop-table",
+	"create-index",
+	"drop-index",
+	"backfill-completed"
+];
+//#endregion
+export { schemaChangeTags };
+
+//# sourceMappingURL=schema-change-tags.js.map

package/out/zero-cache/src/services/change-source/protocol/current/schema-change-tags.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema-change-tags.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/schema-change-tags.ts"],"sourcesContent":["/**\n * Kept in a module that is not re-exported by the public change protocol so\n * internal consumers can share the tag list without expanding the public API.\n */\nexport const schemaChangeTags = [\n  'create-table',\n  'rename-table',\n  'update-table-metadata',\n  'add-column',\n  'update-column',\n  'drop-column',\n  'drop-table',\n  'create-index',\n  'drop-index',\n  'backfill-completed',\n] as const;\n"],"mappings":";;;;;AAIA,IAAa,mBAAmB;CAC9B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF"}

package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.d.ts

@@ -1,18 +1,17 @@
 import type { LogContext } from '@rocicorp/logger';
 import type { NormalizedZeroConfig } from '../../config/normalize.ts';
-import type { BackupMonitor } from './backup-monitor.ts';
+import type { ProcessManager } from '../life-cycle.ts';
+import { BackupMonitor } from './backup-monitor.ts';
 import type { ChangeStreamerService } from './change-streamer.ts';
-import { type BackupStateVerifier } from './litestream3-backup-monitor.ts';
-import { type VfsBackupWatermarkSource } from './vfs-backup-monitor.ts';
+import { type BackupStateVerifier } from './litestream3-prometheus-poller.ts';
 export type BackupCleanupMonitorFactoryOptions = {
     lc: LogContext;
     config: NormalizedZeroConfig;
+    processes: ProcessManager;
     replicaFile: string;
     changeStreamer: ChangeStreamerService;
-    initialCleanupDelayMs: number;
     verifyBackupState?: BackupStateVerifier | undefined;
-    vfsBackupWatermarkSource?: VfsBackupWatermarkSource | undefined;
     env?: NodeJS.ProcessEnv | undefined;
 };
-export declare function createBackupCleanupMonitor({ lc, config, replicaFile, changeStreamer, initialCleanupDelayMs, verifyBackupState, vfsBackupWatermarkSource, env, }: BackupCleanupMonitorFactoryOptions): BackupMonitor | null;
+export declare function createBackupCleanupMonitor({ lc, config, processes, replicaFile, changeStreamer, verifyBackupState, env, }: BackupCleanupMonitorFactoryOptions): BackupMonitor;
 //# sourceMappingURL=backup-cleanup-monitor-factory.d.ts.map

package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"backup-cleanup-monitor-factory.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AAKpE,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,qBAAqB,CAAC;AACvD,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAChE,OAAO,EAEL,KAAK,mBAAmB,EACzB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAEL,KAAK,wBAAwB,EAC9B,MAAM,yBAAyB,CAAC;AAEjC,MAAM,MAAM,kCAAkC,GAAG;IAC/C,EAAE,EAAE,UAAU,CAAC;IACf,MAAM,EAAE,oBAAoB,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,qBAAqB,CAAC;IACtC,qBAAqB,EAAE,MAAM,CAAC;IAC9B,iBAAiB,CAAC,EAAE,mBAAmB,GAAG,SAAS,CAAC;IACpD,wBAAwB,CAAC,EAAE,wBAAwB,GAAG,SAAS,CAAC;IAChE,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,GAAG,SAAS,CAAC;CACrC,CAAC;AAEF,wBAAgB,0BAA0B,CAAC,EACzC,EAAE,EACF,MAAM,EACN,WAAW,EACX,cAAc,EACd,qBAAqB,EACrB,iBAAiB,EACjB,wBAAwB,EACxB,GAAG,GACJ,EAAE,kCAAkC,GAAG,aAAa,GAAG,IAAI,CAgC3D"}
+{"version":3,"file":"backup-cleanup-monitor-factory.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AAKpE,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,kBAAkB,CAAC;AAMrD,OAAO,EAAyB,aAAa,EAAC,MAAM,qBAAqB,CAAC;AAC1E,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAChE,OAAO,EACL,KAAK,mBAAmB,EAEzB,MAAM,oCAAoC,CAAC;AAG5C,MAAM,MAAM,kCAAkC,GAAG;IAC/C,EAAE,EAAE,UAAU,CAAC;IACf,MAAM,EAAE,oBAAoB,CAAC;IAC7B,SAAS,EAAE,cAAc,CAAC;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,qBAAqB,CAAC;IACtC,iBAAiB,CAAC,EAAE,mBAAmB,GAAG,SAAS,CAAC;IACpD,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,GAAG,SAAS,CAAC;CACrC,CAAC;AAEF,wBAAgB,0BAA0B,CAAC,EACzC,EAAE,EACF,MAAM,EACN,SAAS,EACT,WAAW,EACX,cAAc,EACd,iBAAiB,EACjB,GAAiB,GAClB,EAAE,kCAAkC,GAAG,aAAa,CAgDpD"}

package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.js

@@ -1,15 +1,38 @@
 import { forkChildWorker } from "../../types/processes.js";
-import { BACKUP_WATERMARK_READER_URL } from "../../server/worker-urls.js";
+import { BACKUP_WATERMARK_POLLER_URL } from "../../server/worker-urls.js";
+import { getVfsEnv } from "../litestream/vfs-watermark-poller.js";
+import { Subscription } from "../../types/subscription.js";
 import { getLastBackupTime } from "../litestream/commands.js";
-import { VfsBackupWatermarkWorkerSource } from "../litestream/vfs-watermark-worker-source.js";
-import { Litestream3BackupMonitor } from "./litestream3-backup-monitor.js";
-import { VfsBackupMonitor } from "./vfs-backup-monitor.js";
+import { BackupMonitor } from "./backup-monitor.js";
+import { Litestream3PrometheusPoller } from "./litestream3-prometheus-poller.js";
+import { ReplicaPoller } from "./replica-poller.js";
 //#region ../zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.ts
-function createBackupCleanupMonitor({ lc, config, replicaFile, changeStreamer, initialCleanupDelayMs, verifyBackupState, vfsBackupWatermarkSource, env }) {
-	const { backupURL, port: metricsPort } = config.litestream;
-	if (!backupURL) return null;
-	if (config.litestream.backupUsingV5) return new VfsBackupMonitor(lc, backupURL, changeStreamer, initialCleanupDelayMs, config.litestream.vfsProbeIntervalMs, vfsBackupWatermarkSource ?? new VfsBackupWatermarkWorkerSource(lc, () => forkChildWorker(BACKUP_WATERMARK_READER_URL, env ?? process.env), config.litestream.vfsProbeTimeoutMs));
-	return new Litestream3BackupMonitor(lc, replicaFile, backupURL, `http://localhost:${metricsPort}/metrics`, changeStreamer, initialCleanupDelayMs, verifyBackupState ?? (() => getLastBackupTime(lc, config)));
+function createBackupCleanupMonitor({ lc, config, processes, replicaFile, changeStreamer, verifyBackupState, env = process.env }) {
+	const { litestream, replica } = config;
+	const { backupURL } = litestream;
+	let stream;
+	if (!backupURL) stream = new ReplicaPoller(lc, replicaFile).start();
+	else if (config.litestream.backupUsingV5) {
+		const { logLevel, endpoint, region, vfsLogFile: logFile, vfsProbeIntervalMs: remotePollIntervalMs } = litestream;
+		const sub = Subscription.create();
+		const vfsEnv = getVfsEnv({
+			backupURL,
+			endpoint,
+			region,
+			logLevel,
+			logFile,
+			remotePollIntervalMs
+		});
+		processes.addWorker(forkChildWorker(BACKUP_WATERMARK_POLLER_URL, {
+			...env,
+			...vfsEnv
+		}), "supporting", "backup-watermark-poller").onMessageType("backupWatermarkUpdate", (msg) => sub.push(msg));
+		stream = sub;
+	} else {
+		const { port: metricsPort } = litestream;
+		stream = new Litestream3PrometheusPoller(lc, replicaFile, backupURL, `http://localhost:${metricsPort}/metrics`, verifyBackupState ?? (() => getLastBackupTime(lc, litestream, replica.file))).start();
+	}
+	return new BackupMonitor(lc, stream, changeStreamer, replicaFile);
 }
 //#endregion
 export { createBackupCleanupMonitor };

package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.js.map

@@ -1 +1 @@
-{"version":3,"file":"backup-cleanup-monitor-factory.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport {BACKUP_WATERMARK_READER_URL} from '../../server/worker-urls.ts';\nimport {forkChildWorker} from '../../types/processes.ts';\nimport {getLastBackupTime} from '../litestream/commands.ts';\nimport {VfsBackupWatermarkWorkerSource} from '../litestream/vfs-watermark-worker-source.ts';\nimport type {BackupMonitor} from './backup-monitor.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\nimport {\n  Litestream3BackupMonitor,\n  type BackupStateVerifier,\n} from './litestream3-backup-monitor.ts';\nimport {\n  VfsBackupMonitor,\n  type VfsBackupWatermarkSource,\n} from './vfs-backup-monitor.ts';\n\nexport type BackupCleanupMonitorFactoryOptions = {\n  lc: LogContext;\n  config: NormalizedZeroConfig;\n  replicaFile: string;\n  changeStreamer: ChangeStreamerService;\n  initialCleanupDelayMs: number;\n  verifyBackupState?: BackupStateVerifier | undefined;\n  vfsBackupWatermarkSource?: VfsBackupWatermarkSource | undefined;\n  env?: NodeJS.ProcessEnv | undefined;\n};\n\nexport function createBackupCleanupMonitor({\n  lc,\n  config,\n  replicaFile,\n  changeStreamer,\n  initialCleanupDelayMs,\n  verifyBackupState,\n  vfsBackupWatermarkSource,\n  env,\n}: BackupCleanupMonitorFactoryOptions): BackupMonitor | null {\n  const {backupURL, port: metricsPort} = config.litestream;\n  if (!backupURL) {\n    return null;\n  }\n\n  if (config.litestream.backupUsingV5) {\n    return new VfsBackupMonitor(\n      lc,\n      backupURL,\n      changeStreamer,\n      initialCleanupDelayMs,\n      config.litestream.vfsProbeIntervalMs,\n      vfsBackupWatermarkSource ??\n        new VfsBackupWatermarkWorkerSource(\n          lc,\n          () =>\n            forkChildWorker(BACKUP_WATERMARK_READER_URL, env ?? process.env),\n          config.litestream.vfsProbeTimeoutMs,\n        ),\n    );\n  }\n\n  return new Litestream3BackupMonitor(\n    lc,\n    replicaFile,\n    backupURL,\n    `http://localhost:${metricsPort}/metrics`,\n    changeStreamer,\n    initialCleanupDelayMs,\n    verifyBackupState ?? (() => getLastBackupTime(lc, config)),\n  );\n}\n"],"mappings":";;;;;;;AA4BA,SAAgB,2BAA2B,EACzC,IACA,QACA,aACA,gBACA,uBACA,mBACA,0BACA,OAC2D;CAC3D,MAAM,EAAC,WAAW,MAAM,gBAAe,OAAO;CAC9C,IAAI,CAAC,WACH,OAAO;CAGT,IAAI,OAAO,WAAW,eACpB,OAAO,IAAI,iBACT,IACA,WACA,gBACA,uBACA,OAAO,WAAW,oBAClB,4BACE,IAAI,+BACF,UAEE,gBAAgB,6BAA6B,OAAO,QAAQ,GAAG,GACjE,OAAO,WAAW,iBACpB,CACJ;CAGF,OAAO,IAAI,yBACT,IACA,aACA,WACA,oBAAoB,YAAY,WAChC,gBACA,uBACA,4BAA4B,kBAAkB,IAAI,MAAM,EAC1D;AACF"}
+{"version":3,"file":"backup-cleanup-monitor-factory.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport {BACKUP_WATERMARK_POLLER_URL} from '../../server/worker-urls.ts';\nimport {forkChildWorker} from '../../types/processes.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {ProcessManager} from '../life-cycle.ts';\nimport {getLastBackupTime} from '../litestream/commands.ts';\nimport {\n  type BackupWatermarkUpdate,\n  getVfsEnv,\n} from '../litestream/vfs-watermark-poller.ts';\nimport {type BackedUpWatermark, BackupMonitor} from './backup-monitor.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\nimport {\n  type BackupStateVerifier,\n  Litestream3PrometheusPoller,\n} from './litestream3-prometheus-poller.ts';\nimport {ReplicaPoller} from './replica-poller.ts';\n\nexport type BackupCleanupMonitorFactoryOptions = {\n  lc: LogContext;\n  config: NormalizedZeroConfig;\n  processes: ProcessManager;\n  replicaFile: string;\n  changeStreamer: ChangeStreamerService;\n  verifyBackupState?: BackupStateVerifier | undefined;\n  env?: NodeJS.ProcessEnv | undefined;\n};\n\nexport function createBackupCleanupMonitor({\n  lc,\n  config,\n  processes,\n  replicaFile,\n  changeStreamer,\n  verifyBackupState,\n  env = process.env,\n}: BackupCleanupMonitorFactoryOptions): BackupMonitor {\n  const {litestream, replica} = config;\n  const {backupURL} = litestream;\n\n  let stream: Source<BackedUpWatermark>;\n\n  if (!backupURL) {\n    stream = new ReplicaPoller(lc, replicaFile).start();\n  } else if (config.litestream.backupUsingV5) {\n    const {\n      logLevel,\n      endpoint,\n      region,\n      vfsLogFile: logFile,\n      vfsProbeIntervalMs: remotePollIntervalMs,\n    } = litestream;\n    const sub = Subscription.create<BackedUpWatermark>();\n    const vfsEnv = getVfsEnv({\n      backupURL,\n      endpoint,\n      region,\n      logLevel,\n      logFile,\n      remotePollIntervalMs,\n    });\n    processes\n      .addWorker(\n        forkChildWorker(BACKUP_WATERMARK_POLLER_URL, {...env, ...vfsEnv}),\n        'supporting',\n        'backup-watermark-poller',\n      )\n      .onMessageType<BackupWatermarkUpdate>('backupWatermarkUpdate', msg =>\n        sub.push(msg),\n      );\n    stream = sub;\n  } else {\n    const {port: metricsPort} = litestream;\n    stream = new Litestream3PrometheusPoller(\n      lc,\n      replicaFile,\n      backupURL,\n      `http://localhost:${metricsPort}/metrics`,\n      verifyBackupState ??\n        (() => getLastBackupTime(lc, litestream, replica.file)),\n    ).start();\n  }\n\n  return new BackupMonitor(lc, stream, changeStreamer, replicaFile);\n}\n"],"mappings":";;;;;;;;;AA8BA,SAAgB,2BAA2B,EACzC,IACA,QACA,WACA,aACA,gBACA,mBACA,MAAM,QAAQ,OACsC;CACpD,MAAM,EAAC,YAAY,YAAW;CAC9B,MAAM,EAAC,cAAa;CAEpB,IAAI;CAEJ,IAAI,CAAC,WACH,SAAS,IAAI,cAAc,IAAI,WAAW,EAAE,MAAM;MAC7C,IAAI,OAAO,WAAW,eAAe;EAC1C,MAAM,EACJ,UACA,UACA,QACA,YAAY,SACZ,oBAAoB,yBAClB;EACJ,MAAM,MAAM,aAAa,OAA0B;EACnD,MAAM,SAAS,UAAU;GACvB;GACA;GACA;GACA;GACA;GACA;EACF,CAAC;EACD,UACG,UACC,gBAAgB,6BAA6B;GAAC,GAAG;GAAK,GAAG;EAAM,CAAC,GAChE,cACA,yBACF,EACC,cAAqC,0BAAyB,QAC7D,IAAI,KAAK,GAAG,CACd;EACF,SAAS;CACX,OAAO;EACL,MAAM,EAAC,MAAM,gBAAe;EAC5B,SAAS,IAAI,4BACX,IACA,aACA,WACA,oBAAoB,YAAY,WAChC,4BACS,kBAAkB,IAAI,YAAY,QAAQ,IAAI,EACzD,EAAE,MAAM;CACV;CAEA,OAAO,IAAI,cAAc,IAAI,QAAQ,gBAAgB,WAAW;AAClE"}

package/out/zero-cache/src/services/change-streamer/backup-monitor.d.ts

@@ -1,8 +1,18 @@
-import type { Subscription } from '../../types/subscription.ts';
+import type { LogContext } from '@rocicorp/logger';
+import type { Source } from '../../types/streams.ts';
 import type { SingletonService } from '../service.ts';
-import type { SnapshotMessage } from './snapshot.ts';
-export interface BackupMonitor extends SingletonService {
-    startSnapshotReservation(taskID: string): Subscription<SnapshotMessage>;
-    endReservation(taskID: string, updateCleanupDelay?: boolean | undefined): void;
+import type { ChangeStreamerService } from './change-streamer.ts';
+export type BackedUpWatermark = {
+    watermark: string;
+    writeTimeMs?: number | undefined;
+    backupTimeMs: number;
+};
+export declare class BackupMonitor implements SingletonService {
+    #private;
+    readonly id = "backup-monitor";
+    constructor(lc: LogContext, watermarks: Source<BackedUpWatermark>, changeStreamer: ChangeStreamerService, replicaFile: string);
+    firstBackupReceived(): Promise<void>;
+    run(): Promise<void>;
+    stop(): Promise<void>;
 }
 //# sourceMappingURL=backup-monitor.d.ts.map

package/out/zero-cache/src/services/change-streamer/backup-monitor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"backup-monitor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAC9D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,eAAe,CAAC;AACpD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,eAAe,CAAC;AAEnD,MAAM,WAAW,aAAc,SAAQ,gBAAgB;IACrD,wBAAwB,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,CAAC,eAAe,CAAC,CAAC;IACxE,cAAc,CACZ,MAAM,EAAE,MAAM,EACd,kBAAkB,CAAC,EAAE,OAAO,GAAG,SAAS,GACvC,IAAI,CAAC;CACT"}
+{"version":3,"file":"backup-monitor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,eAAe,CAAC;AACpD,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAEhE,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,qBAAa,aAAc,YAAW,gBAAgB;;IACpD,QAAQ,CAAC,EAAE,oBAAoB;gBAU7B,EAAE,EAAE,UAAU,EACd,UAAU,EAAE,MAAM,CAAC,iBAAiB,CAAC,EACrC,cAAc,EAAE,qBAAqB,EACrC,WAAW,EAAE,MAAM;IAQrB,mBAAmB;IAIb,GAAG;IAeT,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAkCtB"}

package/out/zero-cache/src/services/change-streamer/backup-monitor.js

@@ -0,0 +1,66 @@
+import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
+import { Database } from "../../../../zqlite/src/db.js";
+import { getOrCreateGauge } from "../../observability/metrics.js";
+import { resolver } from "@rocicorp/resolver";
+//#region ../zero-cache/src/services/change-streamer/backup-monitor.ts
+var BackupMonitor = class {
+	id = "backup-monitor";
+	#lc;
+	#watermarks;
+	#changeStreamer;
+	#replicaFile;
+	#firstBackupReceived = resolver();
+	#latestBackup;
+	constructor(lc, watermarks, changeStreamer, replicaFile) {
+		this.#lc = lc;
+		this.#watermarks = watermarks;
+		this.#changeStreamer = changeStreamer;
+		this.#replicaFile = replicaFile;
+	}
+	firstBackupReceived() {
+		return this.#firstBackupReceived.promise;
+	}
+	async run() {
+		this.#lc.info?.("starting backup monitor");
+		this.#initBackupLagMetric();
+		for await (const backedUp of this.#watermarks) {
+			if (!this.#latestBackup) {
+				this.#lc.info?.(`received first backup up watermark`, { backedUp });
+				this.#firstBackupReceived.resolve();
+			}
+			this.#latestBackup = backedUp;
+			this.#changeStreamer.trackBackupWatermark(backedUp.watermark);
+		}
+		this.#lc.info?.("watermark stream closed. BackupMonitor stopped.");
+	}
+	stop() {
+		this.#watermarks.cancel();
+		return promiseVoid;
+	}
+	#initBackupLagMetric() {
+		getOrCreateGauge("replica", "backup_lag", {
+			description: "Latency from when a change is written to the replica to when it is backed up to litestream. It is expected to create a saw pattern from 0 to the configured ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES.",
+			unit: "millisecond"
+		}).addCallback((o) => {
+			const latestBackup = this.#latestBackup;
+			if (!latestBackup) {
+				this.#lc.warn?.(`no backed up watermarks. unable to report replica.backup_lag`);
+				return;
+			}
+			const db = new Database(this.#lc, this.#replicaFile, { readonly: true });
+			try {
+				const { writeTimeMs } = db.prepare(`SELECT writeTimeMs FROM "_zero.replicationState"`).get();
+				const backupLag = Math.max(0, writeTimeMs - latestBackup.backupTimeMs);
+				o.observe(backupLag);
+			} catch (e) {
+				this.#lc.warn?.(`error measuring replica.backup_lag metric`, e);
+			} finally {
+				db.close();
+			}
+		});
+	}
+};
+//#endregion
+export { BackupMonitor };
+
+//# sourceMappingURL=backup-monitor.js.map

package/out/zero-cache/src/services/change-streamer/backup-monitor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"backup-monitor.js","names":["#lc","#watermarks","#changeStreamer","#replicaFile","#firstBackupReceived","#initBackupLagMetric","#latestBackup"],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {getOrCreateGauge} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {SingletonService} from '../service.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\n\nexport type BackedUpWatermark = {\n  watermark: string;\n  writeTimeMs?: number | undefined; // optional for debuggin; available on v5\n  backupTimeMs: number;\n};\n\nexport class BackupMonitor implements SingletonService {\n  readonly id = 'backup-monitor';\n  readonly #lc: LogContext;\n  readonly #watermarks: Source<BackedUpWatermark>;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #replicaFile: string;\n  readonly #firstBackupReceived = resolver();\n\n  #latestBackup: BackedUpWatermark | undefined;\n\n  constructor(\n    lc: LogContext,\n    watermarks: Source<BackedUpWatermark>,\n    changeStreamer: ChangeStreamerService,\n    replicaFile: string,\n  ) {\n    this.#lc = lc;\n    this.#watermarks = watermarks;\n    this.#changeStreamer = changeStreamer;\n    this.#replicaFile = replicaFile;\n  }\n\n  firstBackupReceived() {\n    return this.#firstBackupReceived.promise;\n  }\n\n  async run() {\n    this.#lc.info?.('starting backup monitor');\n    this.#initBackupLagMetric();\n\n    for await (const backedUp of this.#watermarks) {\n      if (!this.#latestBackup) {\n        this.#lc.info?.(`received first backup up watermark`, {backedUp});\n        this.#firstBackupReceived.resolve();\n      }\n      this.#latestBackup = backedUp;\n      this.#changeStreamer.trackBackupWatermark(backedUp.watermark);\n    }\n    this.#lc.info?.('watermark stream closed. BackupMonitor stopped.');\n  }\n\n  stop(): Promise<void> {\n    this.#watermarks.cancel();\n    return promiseVoid;\n  }\n\n  #initBackupLagMetric() {\n    getOrCreateGauge('replica', 'backup_lag', {\n      description:\n        'Latency from when a change is written to the replica ' +\n        'to when it is backed up to litestream. It is expected to create a saw ' +\n        'pattern from 0 to the configured ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES.',\n      unit: 'millisecond',\n    }).addCallback(o => {\n      const latestBackup = this.#latestBackup;\n      if (!latestBackup) {\n        this.#lc.warn?.(\n          `no backed up watermarks. unable to report replica.backup_lag`,\n        );\n        return;\n      }\n      const db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n      try {\n        const {writeTimeMs} = db\n          .prepare(/*sql*/ `SELECT writeTimeMs FROM \"_zero.replicationState\"`)\n          .get<{writeTimeMs: number}>();\n        const backupLag = Math.max(0, writeTimeMs - latestBackup.backupTimeMs);\n        o.observe(backupLag);\n      } catch (e) {\n        this.#lc.warn?.(`error measuring replica.backup_lag metric`, e);\n      } finally {\n        db.close();\n      }\n    });\n  }\n}\n"],"mappings":";;;;;AAeA,IAAa,gBAAb,MAAuD;CACrD,KAAc;CACd;CACA;CACA;CACA;CACA,uBAAgC,SAAS;CAEzC;CAEA,YACE,IACA,YACA,gBACA,aACA;EACA,KAAKA,MAAM;EACX,KAAKC,cAAc;EACnB,KAAKC,kBAAkB;EACvB,KAAKC,eAAe;CACtB;CAEA,sBAAsB;EACpB,OAAO,KAAKC,qBAAqB;CACnC;CAEA,MAAM,MAAM;EACV,KAAKJ,IAAI,OAAO,yBAAyB;EACzC,KAAKK,qBAAqB;EAE1B,WAAW,MAAM,YAAY,KAAKJ,aAAa;GAC7C,IAAI,CAAC,KAAKK,eAAe;IACvB,KAAKN,IAAI,OAAO,sCAAsC,EAAC,SAAQ,CAAC;IAChE,KAAKI,qBAAqB,QAAQ;GACpC;GACA,KAAKE,gBAAgB;GACrB,KAAKJ,gBAAgB,qBAAqB,SAAS,SAAS;EAC9D;EACA,KAAKF,IAAI,OAAO,iDAAiD;CACnE;CAEA,OAAsB;EACpB,KAAKC,YAAY,OAAO;EACxB,OAAO;CACT;CAEA,uBAAuB;EACrB,iBAAiB,WAAW,cAAc;GACxC,aACE;GAGF,MAAM;EACR,CAAC,EAAE,aAAY,MAAK;GAClB,MAAM,eAAe,KAAKK;GAC1B,IAAI,CAAC,cAAc;IACjB,KAAKN,IAAI,OACP,8DACF;IACA;GACF;GACA,MAAM,KAAK,IAAI,SAAS,KAAKA,KAAK,KAAKG,cAAc,EAAC,UAAU,KAAI,CAAC;GACrE,IAAI;IACF,MAAM,EAAC,gBAAe,GACnB,QAAgB,kDAAkD,EAClE,IAA2B;IAC9B,MAAM,YAAY,KAAK,IAAI,GAAG,cAAc,aAAa,YAAY;IACrE,EAAE,QAAQ,SAAS;GACrB,SAAS,GAAG;IACV,KAAKH,IAAI,OAAO,6CAA6C,CAAC;GAChE,UAAU;IACR,GAAG,MAAM;GACX;EACF,CAAC;CACH;AACF"}

package/out/zero-cache/src/services/change-streamer/broadcast.d.ts

@@ -1,17 +1,76 @@
 import type { LogContext } from '@rocicorp/logger';
-import type { WatermarkedChange } from './change-streamer-service.ts';
+import type { WatermarkedChange } from './change-streamer.ts';
 import type { Subscriber } from './subscriber.ts';
 export type BroadcastReleaseMode = 'all-subscribers' | 'consensus-timeout';
+/**
+ * Enables event-driven early release of a {@link Broadcast}. When provided and a
+ * majority has acked, the broadcast schedules its own consensus-timeout release
+ * proportional to how long the majority took to ack, instead of relying on the
+ * Forwarder's periodic progress tick.
+ */
+export type EarlyReleaseOptions = {
+    /** Time to wait, proportional to majority completion, before releasing. */
+    consensusTimeoutProportion: number;
+    /** Injectable timer, for deterministic testing. */
+    setTimeoutFn?: typeof setTimeout;
+    /** Injectable timer cancellation, for deterministic testing. */
+    clearTimeoutFn?: typeof clearTimeout;
+};
 /**
  * Initiates and tracks the progress of a change broadcasted to
- * a set of subscribers.
+ * a set of subscribers, implementing **consensus-based timeouts**
+ * to effect short-term graceful degradation when a minority of
+ * subscribers are behind.
+ *
+ * ### Background
+ *
+ * The purpose of flow control is to pull upstream replication changes
+ * no faster than the rate they are processed by downstream subscribers
+ * in the steady state. In the change-streamer, this is done by occasionally
+ * waiting for ACKs from subscribers before continuing; without doing so,
+ * I/O buffers fill up and cause the system to spend most of its time in GC.
+ *
+ * However, the naive algorithm of always waiting for all subscribers (e.g.
+ * `Promise.all()`) can behave poorly in scenarios where subscribers
+ * are imbalanced:
+ * * New subscribers may have a backlog of changes to catch up with.
+ *   Having all subscribers wait for the new subscriber to catch up results
+ *   in delaying the entire application.
+ * * Broken TCP connections similarly require all subscribers to wait until
+ *   connection liveness checks kick in and disconnect the subscriber.
  *
- * Creating a `Broadcast` automatically initiates the send.
+ * A simplistic approach is to add a limit to the amount of time waiting for
+ * subscribers, i.e. an ack timeout. However, deciding what this timeout
+ * should be is non-trivial because of the heterogeneous nature of changes;
+ * while most changes operate on single rows and are relatively predictable
+ * in terms of running time, some changes are table-wide operations and can
+ * legitimately take an arbitrary amount of time. In such scenarios, a
+ * timeout that is too short can stop progress on replication altogether.
  *
- * By default, {@link Broadcast.done} resolves when all subscribers
- * have acked the change. However, {@link Broadcast.checkProgress()}
- * can be called to resolve the broadcast earlier based on the flow
- * control policy.
+ * ### Consensus-based Timeout Algorithm
+ *
+ * To address these shortcomings, a "consensus-based timeout" algorithm is
+ * used:
+ * * Wait for more than half of the subscribers to finish. (In
+ *   case of a single node, or the case of one replication-manager
+ *   and one view-syncer, this reduces to waiting for all subscribers.)
+ * * Once more than half of the subscribers have finished, proceed after
+ *   the configured timeout, proportional to the time it took for the
+ *   majority of subscribers to ack, even if not all subscribers have
+ *   finished.
+ *
+ * In other words, the subscribers themselves are used to determine the
+ * timeout of each batch of changes; the majority determines this when
+ * they complete, upon which a timeout is logically started.
+ *
+ * In the common case, the remaining subscribers finish soon afterward and
+ * the timeout never elapses. However, in pathological cases where a minority
+ * of subscribers have a disproportionate amount of load, some will still
+ * be processing (or otherwise unresponsive). These subscribers are given
+ * a bounded amount of time to catch up at each flushed batch, up to the
+ * timeout interval. This guarantees eventual catchup because the
+ * subscribers with a backlog of changes necessarily have a higher
+ * processing rate than the subscribers that finished (and are made to wait).
  */
 export declare class Broadcast {
     #private;
@@ -24,79 +83,10 @@ export declare class Broadcast {
      * Broadcasts the `change` to the `subscribers` and tracks their
      * completion.
      */
-    constructor(subscribers: Iterable<Subscriber>, change: WatermarkedChange);
+    constructor(lc: LogContext, subscribers: Iterable<Subscriber>, change: WatermarkedChange, earlyRelease?: EarlyReleaseOptions);
     get isDone(): boolean;
     get done(): Promise<void>;
     get releaseMode(): BroadcastReleaseMode;
-    /**
-     * Checks for pathological situations in which flow should be reenabled
-     * before all subscribers have acked.
-     *
-     * ### Background
-     *
-     * The purpose of flow control is to pull upstream replication changes
-     * no faster than the rate as they are processed by downstream subscribers
-     * in the steady state. In the change-streamer, this is done by occasionally
-     * waiting for ACKs from subscribers before continuing; without doing so,
-     * I/O buffers fill up and cause the system to spend most of its time in GC.
-     *
-     * However, the naive algorithm of always waiting for all subscribers (e.g.
-     * `Promise.all()`) can behave poorly in scenarios where subscribers
-     * are imbalanced:
-     * * New subscribers may have a backlog of changes to catch up with.
-     *   Having all subscribers wait for the new subscriber to catch up results
-     *   in delaying the entire application.
-     * * Broken TCP connections similarly require all subscribers to wait until
-     *   connection liveness checks kick in and disconnect the subscriber.
-     *
-     * A simplistic approach is to add a limit to the amount of time waiting for
-     * subscribers, i.e. an ack timeout. However, deciding what this timeout
-     * should be is non-trivial because of the heterogeneous nature of changes;
-     * while most changes operate on single rows and are relatively predictable
-     * in terms of running time, some changes are table-wide operations and can
-     * legitimately take an arbitrary amount of time. In such scenarios, a
-     * timeout that is too short can stop progress on replication altogether.
-     *
-     * ### Consensus-based Timeout Algorithm
-     *
-     * To address these shortcomings, a "consensus-based timeout" algorithm is
-     * used:
-     * * Wait for more than half of the subscribers to finish. (In
-     *   case of a single node, or the case of one replication-manager
-     *   and one view-syncer, this reduces to waiting for all subscribers.)
-     * * Once more than half of the subscribers have finished, proceed after
-     *   a fixed timeout elapses (e.g. 1 second), even if not all subscribers
-     *   have finished.
-     *
-     * In other words, the subscribers themselves are used to determine the
-     * timeout of each batch of changes; the majority determines this when
-     * they complete, upon which a timeout is logically started.
-     *
-     * In the common case, the remaining subscribers finish soon afterward and
-     * the timeout never elapses. However, in pathological cases where a minority
-     * of subscribers have a disproportionate amount of load, some will still
-     * be processing (or otherwise unresponsive). These subscribers are given
-     * a bounded amount of time to catch up at each flushed batch, up to the
-     * timeout interval. This guarantees eventual catchup because the
-     * subscribers with a backlog of changes necessarily have a higher
-     * processing rate than the subscribers that finished (and are made to wait).
-     *
-     * ### Not implemented: Broken connection detection
-     *
-     * If a subscriber has not made progress for a certain interval, the
-     * algorithm could theoretically drop it preemptively, supplementing the
-     * existing websocket-level liveness checks.
-     *
-     * However, a more reliable approach would be to change the replicator
-     * to use non-blocking writes, and subsequently increase the frequency of
-     * connection-level liveness checks. The current synchronous replica writes
-     * can delay both ping responsiveness and change progress arbitrarily (e.g.
-     * a large index creation); an independently liveness check that is not
-     * delayed by synchronous writes on the subscriber would be a more failsafe
-     * solution.
-     *
-     * @returns `true` if the broadcast was already done or was marked done.
-     */
-    checkProgress(lc: LogContext, flowControlConsensusPaddingMs: number, now: number): boolean;
+    logSlowRequests(now: number): void;
 }
 //# sourceMappingURL=broadcast.d.ts.map

package/out/zero-cache/src/services/change-streamer/broadcast.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"broadcast.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/broadcast.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8BAA8B,CAAC;AACpE,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAEhD,MAAM,MAAM,oBAAoB,GAAG,iBAAiB,GAAG,mBAAmB,CAAC;AAE3E;;;;;;;;;;GAUG;AACH,qBAAa,SAAS;;IACpB;;;OAGG;IACH,MAAM,CAAC,eAAe,CACpB,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,EACjC,MAAM,EAAE,iBAAiB;IAmB3B;;;OAGG;gBACS,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,iBAAiB;IAsCxE,IAAI,MAAM,IAAI,OAAO,CAEpB;IAED,IAAI,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAExB;IAED,IAAI,WAAW,IAAI,oBAAoB,CAEtC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAoEG;IACH,aAAa,CACX,EAAE,EAAE,UAAU,EACd,6BAA6B,EAAE,MAAM,EACrC,GAAG,EAAE,MAAM;CAkDd"}
+{"version":3,"file":"broadcast.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/broadcast.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,sBAAsB,CAAC;AAC5D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAEhD,MAAM,MAAM,oBAAoB,GAAG,iBAAiB,GAAG,mBAAmB,CAAC;AAE3E;;;;;GAKG;AACH,MAAM,MAAM,mBAAmB,GAAG;IAChC,2EAA2E;IAC3E,0BAA0B,EAAE,MAAM,CAAC;IACnC,mDAAmD;IACnD,YAAY,CAAC,EAAE,OAAO,UAAU,CAAC;IACjC,gEAAgE;IAChE,cAAc,CAAC,EAAE,OAAO,YAAY,CAAC;CACtC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AACH,qBAAa,SAAS;;IACpB;;;OAGG;IACH,MAAM,CAAC,eAAe,CACpB,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,EACjC,MAAM,EAAE,iBAAiB;IA0B3B;;;OAGG;gBAED,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,EACjC,MAAM,EAAE,iBAAiB,EACzB,YAAY,CAAC,EAAE,mBAAmB;IA4EpC,IAAI,MAAM,IAAI,OAAO,CAEpB;IAED,IAAI,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAExB;IAED,IAAI,WAAW,IAAI,oBAAoB,CAEtC;IAED,eAAe,CAAC,GAAG,EAAE,MAAM;CA8B5B"}