@rocicorp/zero 1.11.0-canary.12 → 1.11.0-canary.14

package/out/zero-cache/src/observability/log-throttle.js

@@ -0,0 +1,76 @@
+import { assert } from "../../../shared/src/asserts.js";
+//#region ../zero-cache/src/observability/log-throttle.ts
+/**
+* Rate limits a log statement per key, for logs that would otherwise be
+* written for every client group, such as the slow hydration of a popular
+* query.
+*
+* Deduplication alone (logging a key once and never again) is not enough:
+* the one line is lost once the log rotates, and the set of logged keys
+* grows without bound. Instead a key is admitted at most once per
+* {@link LogThrottleOptions.windowMs}, and the occurrences suppressed in
+* between are counted and reported with the next admitted one.
+*/
+var LogThrottle = class {
+	#windowMs;
+	#maxKeys;
+	#maxPerWindow;
+	#now;
+	#entries = /* @__PURE__ */ new Map();
+	#windowStart = -Infinity;
+	#admittedInWindow = 0;
+	constructor({ windowMs, maxKeys = 1e3, maxPerWindow = 100, now = performance.now.bind(performance) }) {
+		assert(windowMs >= 0, "windowMs must be nonnegative");
+		assert(maxKeys > 0, "maxKeys must be positive");
+		assert(maxPerWindow > 0, "maxPerWindow must be positive");
+		this.#windowMs = windowMs;
+		this.#maxKeys = maxKeys;
+		this.#maxPerWindow = maxPerWindow;
+		this.#now = now;
+	}
+	/**
+	* Records an occurrence of `key`.
+	*
+	* @returns `undefined` if the occurrence should not be logged, or else the
+	*   number of occurrences of `key` suppressed since it was last logged.
+	*/
+	admit(key) {
+		const now = this.#now();
+		const entry = this.#entries.get(key);
+		if (entry && now - entry.admittedAt < this.#windowMs) {
+			entry.suppressed++;
+			return;
+		}
+		if (now - this.#windowStart >= this.#windowMs) {
+			this.#windowStart = now;
+			this.#admittedInWindow = 0;
+		}
+		if (this.#admittedInWindow >= this.#maxPerWindow) {
+			if (entry) entry.suppressed++;
+			else this.#set(key, {
+				admittedAt: -Infinity,
+				suppressed: 1
+			});
+			return;
+		}
+		this.#admittedInWindow++;
+		const suppressed = entry?.suppressed ?? 0;
+		this.#set(key, {
+			admittedAt: now,
+			suppressed: 0
+		});
+		return suppressed;
+	}
+	#set(key, entry) {
+		this.#entries.delete(key);
+		this.#entries.set(key, entry);
+		if (this.#entries.size > this.#maxKeys) {
+			const oldest = this.#entries.keys().next();
+			if (!oldest.done) this.#entries.delete(oldest.value);
+		}
+	}
+};
+//#endregion
+export { LogThrottle };
+
+//# sourceMappingURL=log-throttle.js.map

package/out/zero-cache/src/observability/log-throttle.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"log-throttle.js","names":["#windowMs","#maxKeys","#maxPerWindow","#now","#entries","#windowStart","#admittedInWindow","#set"],"sources":["../../../../../zero-cache/src/observability/log-throttle.ts"],"sourcesContent":["import {assert} from '../../../shared/src/asserts.ts';\n\nexport type LogThrottleOptions = {\n  /**\n   * An admitted key is suppressed for this long. A key that is still\n   * occurring after the window is admitted again, so a log that is still\n   * relevant keeps showing up (e.g. after the log has rotated) without\n   * repeating for every occurrence.\n   */\n  readonly windowMs: number;\n\n  /**\n   * The most keys tracked at once. The least recently admitted key is\n   * forgotten beyond this, which bounds memory when the key space is\n   * unbounded (e.g. dynamically built query shapes).\n   */\n  readonly maxKeys?: number | undefined;\n\n  /**\n   * The most occurrences admitted per window across all keys, which bounds\n   * log volume when many distinct keys occur at once.\n   */\n  readonly maxPerWindow?: number | undefined;\n\n  readonly now?: (() => number) | undefined;\n};\n\ntype Entry = {\n  admittedAt: number;\n  suppressed: number;\n};\n\n/**\n * Rate limits a log statement per key, for logs that would otherwise be\n * written for every client group, such as the slow hydration of a popular\n * query.\n *\n * Deduplication alone (logging a key once and never again) is not enough:\n * the one line is lost once the log rotates, and the set of logged keys\n * grows without bound. Instead a key is admitted at most once per\n * {@link LogThrottleOptions.windowMs}, and the occurrences suppressed in\n * between are counted and reported with the next admitted one.\n */\nexport class LogThrottle {\n  readonly #windowMs: number;\n  readonly #maxKeys: number;\n  readonly #maxPerWindow: number;\n  readonly #now: () => number;\n\n  // Insertion ordered by admission, so the first entry is the least recently\n  // admitted key.\n  readonly #entries = new Map<string, Entry>();\n  #windowStart = -Infinity;\n  #admittedInWindow = 0;\n\n  constructor({\n    windowMs,\n    maxKeys = 1000,\n    maxPerWindow = 100,\n    now = performance.now.bind(performance),\n  }: LogThrottleOptions) {\n    assert(windowMs >= 0, 'windowMs must be nonnegative');\n    assert(maxKeys > 0, 'maxKeys must be positive');\n    assert(maxPerWindow > 0, 'maxPerWindow must be positive');\n    this.#windowMs = windowMs;\n    this.#maxKeys = maxKeys;\n    this.#maxPerWindow = maxPerWindow;\n    this.#now = now;\n  }\n\n  /**\n   * Records an occurrence of `key`.\n   *\n   * @returns `undefined` if the occurrence should not be logged, or else the\n   *   number of occurrences of `key` suppressed since it was last logged.\n   */\n  admit(key: string): number | undefined {\n    const now = this.#now();\n    const entry = this.#entries.get(key);\n    if (entry && now - entry.admittedAt < this.#windowMs) {\n      entry.suppressed++;\n      return undefined;\n    }\n\n    if (now - this.#windowStart >= this.#windowMs) {\n      this.#windowStart = now;\n      this.#admittedInWindow = 0;\n    }\n    if (this.#admittedInWindow >= this.#maxPerWindow) {\n      // Over the global budget. The occurrence is counted against the key so\n      // that it is reported once the key is admitted.\n      if (entry) {\n        entry.suppressed++;\n      } else {\n        this.#set(key, {admittedAt: -Infinity, suppressed: 1});\n      }\n      return undefined;\n    }\n\n    this.#admittedInWindow++;\n    const suppressed = entry?.suppressed ?? 0;\n    this.#set(key, {admittedAt: now, suppressed: 0});\n    return suppressed;\n  }\n\n  #set(key: string, entry: Entry): void {\n    // Delete first so that the key moves to the end of the insertion order.\n    this.#entries.delete(key);\n    this.#entries.set(key, entry);\n    if (this.#entries.size > this.#maxKeys) {\n      const oldest = this.#entries.keys().next();\n      if (!oldest.done) {\n        this.#entries.delete(oldest.value);\n      }\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;AA2CA,IAAa,cAAb,MAAyB;CACvB;CACA;CACA;CACA;CAIA,2BAAoB,IAAI,IAAmB;CAC3C,eAAe;CACf,oBAAoB;CAEpB,YAAY,EACV,UACA,UAAU,KACV,eAAe,KACf,MAAM,YAAY,IAAI,KAAK,WAAW,KACjB;EACrB,OAAO,YAAY,GAAG,8BAA8B;EACpD,OAAO,UAAU,GAAG,0BAA0B;EAC9C,OAAO,eAAe,GAAG,+BAA+B;EACxD,KAAKA,YAAY;EACjB,KAAKC,WAAW;EAChB,KAAKC,gBAAgB;EACrB,KAAKC,OAAO;CACd;;;;;;;CAQA,MAAM,KAAiC;EACrC,MAAM,MAAM,KAAKA,KAAK;EACtB,MAAM,QAAQ,KAAKC,SAAS,IAAI,GAAG;EACnC,IAAI,SAAS,MAAM,MAAM,aAAa,KAAKJ,WAAW;GACpD,MAAM;GACN;EACF;EAEA,IAAI,MAAM,KAAKK,gBAAgB,KAAKL,WAAW;GAC7C,KAAKK,eAAe;GACpB,KAAKC,oBAAoB;EAC3B;EACA,IAAI,KAAKA,qBAAqB,KAAKJ,eAAe;GAGhD,IAAI,OACF,MAAM;QAEN,KAAKK,KAAK,KAAK;IAAC,YAAY;IAAW,YAAY;GAAC,CAAC;GAEvD;EACF;EAEA,KAAKD;EACL,MAAM,aAAa,OAAO,cAAc;EACxC,KAAKC,KAAK,KAAK;GAAC,YAAY;GAAK,YAAY;EAAC,CAAC;EAC/C,OAAO;CACT;CAEA,KAAK,KAAa,OAAoB;EAEpC,KAAKH,SAAS,OAAO,GAAG;EACxB,KAAKA,SAAS,IAAI,KAAK,KAAK;EAC5B,IAAI,KAAKA,SAAS,OAAO,KAAKH,UAAU;GACtC,MAAM,SAAS,KAAKG,SAAS,KAAK,EAAE,KAAK;GACzC,IAAI,CAAC,OAAO,MACV,KAAKA,SAAS,OAAO,OAAO,KAAK;EAErC;CACF;AACF"}

package/out/zero-cache/src/scripts/permissions.d.ts

@@ -51,6 +51,22 @@ export declare const deployPermissionsOptions: {
             type: v.Type<number>;
             desc: string[];
         };
+        slowAdvanceThreshold: {
+            type: v.Type<number>;
+            desc: string[];
+        };
+        planWarningRowThreshold: {
+            type: v.Type<number>;
+            desc: string[];
+        };
+        planWarningCostThreshold: {
+            type: v.Type<number>;
+            desc: string[];
+        };
+        queryStatsIntervalSeconds: {
+            type: v.Type<number>;
+            desc: string[];
+        };
         ivmSampling: {
             type: v.Type<number>;
             desc: string[];

package/out/zero-cache/src/scripts/permissions.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"permissions.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/scripts/permissions.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,CAAC,MAAM,+BAA+B,CAAC;AACnD,OAAO,EAEL,KAAK,iBAAiB,EACvB,MAAM,kDAAkD,CAAC;AAE1D,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,mCAAmC,CAAC;AAG9D,eAAO,MAAM,wBAAwB;IACnC,MAAM;QACJ,IAAI;YACF,IAAI;YACJ,IAAI;YACJ,KAAK;;;IAIT,QAAQ;QACN,EAAE;YACA,IAAI;YACJ,IAAI;;QAMN,IAAI;;;;;;IAGN,GAAG;QAAG,EAAE;;;;;IAER,KAAK;;;;;;;;;;;IAEL,GAAG;;;;;;;;;;;;;;;;;;;IAEH,MAAM;QACJ,IAAI;YACF,IAAI;YACJ,IAAI;;QAKN,MAAM;YACJ,IAAI;YACJ,IAAI;;;IAYR,KAAK;QACH,IAAI;QACJ,IAAI;QACJ,KAAK;;CAER,CAAC;AAEF,wBAAsB,wBAAwB,CAC5C,UAAU,EAAE,MAAM,EAClB,YAAY,EAAE,IAAI,GACjB,OAAO,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,iBAAiB,CAAA;CAAC,GAAG,SAAS,CAAC,CAAC;AACzE,wBAAsB,wBAAwB,CAC5C,UAAU,EAAE,MAAM,EAClB,YAAY,CAAC,EAAE,KAAK,GACnB,OAAO,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,iBAAiB,CAAA;CAAC,CAAC,CAAC"}
+{"version":3,"file":"permissions.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/scripts/permissions.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,CAAC,MAAM,+BAA+B,CAAC;AACnD,OAAO,EAEL,KAAK,iBAAiB,EACvB,MAAM,kDAAkD,CAAC;AAE1D,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,mCAAmC,CAAC;AAG9D,eAAO,MAAM,wBAAwB;IACnC,MAAM;QACJ,IAAI;YACF,IAAI;YACJ,IAAI;YACJ,KAAK;;;IAIT,QAAQ;QACN,EAAE;YACA,IAAI;YACJ,IAAI;;QAMN,IAAI;;;;;;IAGN,GAAG;QAAG,EAAE;;;;;IAER,KAAK;;;;;;;;;;;IAEL,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAEH,MAAM;QACJ,IAAI;YACF,IAAI;YACJ,IAAI;;QAKN,MAAM;YACJ,IAAI;YACJ,IAAI;;;IAYR,KAAK;QACH,IAAI;QACJ,IAAI;QACJ,KAAK;;CAER,CAAC;AAEF,wBAAsB,wBAAwB,CAC5C,UAAU,EAAE,MAAM,EAClB,YAAY,EAAE,IAAI,GACjB,OAAO,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,iBAAiB,CAAA;CAAC,GAAG,SAAS,CAAC,CAAC;AACzE,wBAAsB,wBAAwB,CAC5C,UAAU,EAAE,MAAM,EAClB,YAAY,CAAC,EAAE,KAAK,GACnB,OAAO,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,iBAAiB,CAAA;CAAC,CAAC,CAAC"}

package/out/zero-cache/src/server/runner/zero-dispatcher.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"zero-dispatcher.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/server/runner/zero-dispatcher.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AAEpE,OAAO,EAAC,WAAW,EAAE,KAAK,OAAO,EAAC,MAAM,gCAAgC,CAAC;AAOzE,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,0BAA0B,CAAC;AAMrD,qBAAa,cAAe,SAAQ,WAAW;;IAC7C,QAAQ,CAAC,EAAE,qBAAqB;IAIhC,YACE,MAAM,EAAE,oBAAoB,EAC5B,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,OAAO,EACb,SAAS,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,EAChC,OAAO,CAAC,EAAE,MAAM,IAAI,EAmBrB;IAED,UAAmB,QAAQ,SAE1B;CAYF"}
+{"version":3,"file":"zero-dispatcher.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/server/runner/zero-dispatcher.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AAEpE,OAAO,EAAC,WAAW,EAAE,KAAK,OAAO,EAAC,MAAM,gCAAgC,CAAC;AAQzE,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,0BAA0B,CAAC;AAMrD,qBAAa,cAAe,SAAQ,WAAW;;IAC7C,QAAQ,CAAC,EAAE,qBAAqB;IAIhC,YACE,MAAM,EAAE,oBAAoB,EAC5B,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,OAAO,EACb,SAAS,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,EAChC,OAAO,CAAC,EAAE,MAAM,IAAI,EAsBrB;IAED,UAAmB,QAAQ,SAE1B;CAYF"}

package/out/zero-cache/src/server/runner/zero-dispatcher.js

@@ -1,5 +1,6 @@
 import { handleHeapzRequest } from "../../services/heapz.js";
 import { HttpService } from "../../services/http-service.js";
+import { handlePlannerzRequest } from "../../services/plannerz.js";
 import { handleProfrmzRequest, handleProfzRequest } from "../../services/profz.js";
 import { handleStatzRequest } from "../../services/statz.js";
 import { installWebSocketHandoff } from "../../types/websocket-handoff.js";
@@ -11,6 +12,7 @@ var ZeroDispatcher = class extends HttpService {
 	constructor(config, lc, opts, getWorker, onStart) {
 		super(`zero-dispatcher`, lc, opts, (fastify) => {
 			fastify.get("/statz", (req, res) => handleStatzRequest(lc, config, req, res));
+			fastify.get("/plannerz", (req, res) => handlePlannerzRequest(lc, config, req, res));
 			fastify.get("/heapz", (req, res) => handleHeapzRequest(lc, config, req, res));
 			fastify.get("/profz", (req, res) => handleProfzRequest(lc, config, req, res, this.#getWorker));
 			fastify.get("/profrmz", (req, res) => handleProfrmzRequest(lc, config, req, res, this.#getWorker));

package/out/zero-cache/src/server/runner/zero-dispatcher.js.map

@@ -1 +1 @@
-{"version":3,"file":"zero-dispatcher.js","names":["#getWorker","#onStart","#handoff"],"sources":["../../../../../../zero-cache/src/server/runner/zero-dispatcher.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport {handleHeapzRequest} from '../../services/heapz.ts';\nimport {HttpService, type Options} from '../../services/http-service.ts';\nimport {\n  handleProfrmzRequest,\n  handleProfzRequest,\n} from '../../services/profz.ts';\nimport {handleStatzRequest} from '../../services/statz.ts';\nimport type {IncomingMessageSubset} from '../../types/http.ts';\nimport type {Worker} from '../../types/processes.ts';\nimport {\n  installWebSocketHandoff,\n  type HandoffSpec,\n} from '../../types/websocket-handoff.ts';\n\nexport class ZeroDispatcher extends HttpService {\n  readonly id = 'zero-dispatcher';\n  readonly #getWorker: () => Promise<Worker>;\n  readonly #onStart: (() => void) | undefined;\n\n  constructor(\n    config: NormalizedZeroConfig,\n    lc: LogContext,\n    opts: Options,\n    getWorker: () => Promise<Worker>,\n    onStart?: () => void,\n  ) {\n    super(`zero-dispatcher`, lc, opts, fastify => {\n      fastify.get('/statz', (req, res) =>\n        handleStatzRequest(lc, config, req, res),\n      );\n      fastify.get('/heapz', (req, res) =>\n        handleHeapzRequest(lc, config, req, res),\n      );\n      fastify.get('/profz', (req, res) =>\n        handleProfzRequest(lc, config, req, res, this.#getWorker),\n      );\n      fastify.get('/profrmz', (req, res) =>\n        handleProfrmzRequest(lc, config, req, res, this.#getWorker),\n      );\n      installWebSocketHandoff(lc, this.#handoff, fastify.server);\n    });\n    this.#getWorker = getWorker;\n    this.#onStart = onStart;\n  }\n\n  protected override _onStart() {\n    this.#onStart?.();\n  }\n\n  readonly #handoff = (\n    _req: IncomingMessageSubset,\n    dispatch: (h: HandoffSpec<string>) => void,\n    onError: (error: unknown) => void,\n  ) => {\n    void this.#getWorker().then(\n      sender => dispatch({payload: 'unused', sender}),\n      onError,\n    );\n  };\n}\n"],"mappings":";;;;;;AAgBA,IAAa,iBAAb,cAAoC,YAAY;CAC9C,KAAc;CACd;CACA;CAEA,YACE,QACA,IACA,MACA,WACA,SACA;EACA,MAAM,mBAAmB,IAAI,OAAM,YAAW;GAC5C,QAAQ,IAAI,WAAW,KAAK,QAC1B,mBAAmB,IAAI,QAAQ,KAAK,GAAG,CACzC;GACA,QAAQ,IAAI,WAAW,KAAK,QAC1B,mBAAmB,IAAI,QAAQ,KAAK,GAAG,CACzC;GACA,QAAQ,IAAI,WAAW,KAAK,QAC1B,mBAAmB,IAAI,QAAQ,KAAK,KAAK,KAAKA,UAAU,CAC1D;GACA,QAAQ,IAAI,aAAa,KAAK,QAC5B,qBAAqB,IAAI,QAAQ,KAAK,KAAK,KAAKA,UAAU,CAC5D;GACA,wBAAwB,IAAI,KAAKE,UAAU,QAAQ,MAAM;EAC3D,CAAC;EACD,KAAKF,aAAa;EAClB,KAAKC,WAAW;CAClB;CAEA,WAA8B;EAC5B,KAAKA,WAAW;CAClB;CAEA,YACE,MACA,UACA,YACG;EACH,KAAUD,WAAW,EAAE,MACrB,WAAU,SAAS;GAAC,SAAS;GAAU;EAAM,CAAC,GAC9C,OACF;CACF;AACF"}
+{"version":3,"file":"zero-dispatcher.js","names":["#getWorker","#onStart","#handoff"],"sources":["../../../../../../zero-cache/src/server/runner/zero-dispatcher.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport {handleHeapzRequest} from '../../services/heapz.ts';\nimport {HttpService, type Options} from '../../services/http-service.ts';\nimport {handlePlannerzRequest} from '../../services/plannerz.ts';\nimport {\n  handleProfrmzRequest,\n  handleProfzRequest,\n} from '../../services/profz.ts';\nimport {handleStatzRequest} from '../../services/statz.ts';\nimport type {IncomingMessageSubset} from '../../types/http.ts';\nimport type {Worker} from '../../types/processes.ts';\nimport {\n  installWebSocketHandoff,\n  type HandoffSpec,\n} from '../../types/websocket-handoff.ts';\n\nexport class ZeroDispatcher extends HttpService {\n  readonly id = 'zero-dispatcher';\n  readonly #getWorker: () => Promise<Worker>;\n  readonly #onStart: (() => void) | undefined;\n\n  constructor(\n    config: NormalizedZeroConfig,\n    lc: LogContext,\n    opts: Options,\n    getWorker: () => Promise<Worker>,\n    onStart?: () => void,\n  ) {\n    super(`zero-dispatcher`, lc, opts, fastify => {\n      fastify.get('/statz', (req, res) =>\n        handleStatzRequest(lc, config, req, res),\n      );\n      fastify.get('/plannerz', (req, res) =>\n        handlePlannerzRequest(lc, config, req, res),\n      );\n      fastify.get('/heapz', (req, res) =>\n        handleHeapzRequest(lc, config, req, res),\n      );\n      fastify.get('/profz', (req, res) =>\n        handleProfzRequest(lc, config, req, res, this.#getWorker),\n      );\n      fastify.get('/profrmz', (req, res) =>\n        handleProfrmzRequest(lc, config, req, res, this.#getWorker),\n      );\n      installWebSocketHandoff(lc, this.#handoff, fastify.server);\n    });\n    this.#getWorker = getWorker;\n    this.#onStart = onStart;\n  }\n\n  protected override _onStart() {\n    this.#onStart?.();\n  }\n\n  readonly #handoff = (\n    _req: IncomingMessageSubset,\n    dispatch: (h: HandoffSpec<string>) => void,\n    onError: (error: unknown) => void,\n  ) => {\n    void this.#getWorker().then(\n      sender => dispatch({payload: 'unused', sender}),\n      onError,\n    );\n  };\n}\n"],"mappings":";;;;;;;AAiBA,IAAa,iBAAb,cAAoC,YAAY;CAC9C,KAAc;CACd;CACA;CAEA,YACE,QACA,IACA,MACA,WACA,SACA;EACA,MAAM,mBAAmB,IAAI,OAAM,YAAW;GAC5C,QAAQ,IAAI,WAAW,KAAK,QAC1B,mBAAmB,IAAI,QAAQ,KAAK,GAAG,CACzC;GACA,QAAQ,IAAI,cAAc,KAAK,QAC7B,sBAAsB,IAAI,QAAQ,KAAK,GAAG,CAC5C;GACA,QAAQ,IAAI,WAAW,KAAK,QAC1B,mBAAmB,IAAI,QAAQ,KAAK,GAAG,CACzC;GACA,QAAQ,IAAI,WAAW,KAAK,QAC1B,mBAAmB,IAAI,QAAQ,KAAK,KAAK,KAAKA,UAAU,CAC1D;GACA,QAAQ,IAAI,aAAa,KAAK,QAC5B,qBAAqB,IAAI,QAAQ,KAAK,KAAK,KAAKA,UAAU,CAC5D;GACA,wBAAwB,IAAI,KAAKE,UAAU,QAAQ,MAAM;EAC3D,CAAC;EACD,KAAKF,aAAa;EAClB,KAAKC,WAAW;CAClB;CAEA,WAA8B;EAC5B,KAAKA,WAAW;CAClB;CAEA,YACE,MACA,UACA,YACG;EACH,KAAUD,WAAW,EAAE,MACrB,WAAU,SAAS;GAAC,SAAS;GAAU;EAAM,CAAC,GAC9C,OACF;CACF;AACF"}

package/out/zero-cache/src/server/syncer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"syncer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/syncer.ts"],"names":[],"mappings":"AAkCA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAqC/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CAgOf"}
+{"version":3,"file":"syncer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/syncer.ts"],"names":[],"mappings":"AAqCA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAqC/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CA6Qf"}

package/out/zero-cache/src/server/syncer.js

@@ -6,6 +6,7 @@ import { parentWorker, singleProcessMode } from "../types/processes.js";
 import { getShardID } from "../types/shards.js";
 import { getNormalizedZeroConfig } from "../config/zero-config.js";
 import { randInt } from "../../../shared/src/rand.js";
+import { getOrCreateGauge } from "../observability/metrics.js";
 import { Snapshotter } from "../services/view-syncer/snapshotter.js";
 import { PipelineDriver } from "../services/view-syncer/pipeline-driver.js";
 import { connectPgClient } from "../types/pg.js";
@@ -25,6 +26,8 @@ import { CustomQueryTransformer } from "../custom-queries/transform-query.js";
 import { MutagenService } from "../services/mutagen/mutagen.js";
 import { PusherService } from "../services/mutagen/pusher.js";
 import { ConnectionContextManagerImpl } from "../services/view-syncer/connection-context-manager.js";
+import { DeferredWritesBudget } from "../services/view-syncer/deferred-writes-budget.js";
+import { QueryStats } from "../services/view-syncer/query-stats.js";
 import { SnapshotRowCache } from "../services/view-syncer/snapshot-row-cache.js";
 import { ViewSyncerService } from "../services/view-syncer/view-syncer.js";
 import { Syncer } from "../workers/syncer.js";
@@ -102,6 +105,17 @@ async function runWorker(parent, env, ...args) {
 		};
 	};
 	const snapshotRowCache = config.snapshotRowCacheSize > 0 ? new SnapshotRowCache(config.snapshotRowCacheSize) : void 0;
+	const deferredWritesBudget = config.deferIvmWrites ? DeferredWritesBudget.forHeapProportion(config.deferIvmWritesHeapProportion) : void 0;
+	if (deferredWritesBudget) {
+		lc.info?.(`Deferred IVM writes may hold up to ${deferredWritesBudget.maxRows} rows (~${(deferredWritesBudget.maxBytes / 1024 ** 2).toFixed(2)} MB) across client groups`);
+		getOrCreateGauge("sync", "ivm.deferred-writes-reserved-rows", "Rows reserved by the client groups of a sync worker to hold IVM changes in memory (deferIvmWrites)").addCallback((o) => o.observe(deferredWritesBudget.reservedRows));
+		getOrCreateGauge("sync", "ivm.deferred-writes-held-bytes", {
+			description: "Estimated bytes of the IVM changes that the client groups of a sync worker hold in memory (deferIvmWrites)",
+			unit: "By"
+		}).addCallback((o) => o.observe(deferredWritesBudget.heldBytes));
+	}
+	const queryStats = config.log.queryStatsIntervalSeconds > 0 ? new QueryStats() : void 0;
+	const stopQueryStats = queryStats?.start(lc.withContext("taskID", config.taskID).withContext("component", "view-syncer"), config.log.queryStatsIntervalSeconds * 1e3);
 	const viewSyncerFactory = (id, sub, drainCoordinator) => {
 		const logger = lc.withContext("taskID", config.taskID).withContext("component", "view-syncer").withContext("appID", shard.appID).withContext("shardNum", shard.shardNum).withContext("clientGroupID", id).withContext("instance", randomID());
 		const customQueryTransformer = customQueryConfig && new CustomQueryTransformer(logger, shard);
@@ -110,12 +124,16 @@ async function runWorker(parent, env, ...args) {
 		const inspectorDelegate = new InspectorDelegate(customQueryTransformer);
 		const priorityOpRunningYieldThresholdMs = Math.max(config.yieldThresholdMs / 4, 2);
 		const normalYieldThresholdMs = Math.max(config.yieldThresholdMs, 2);
-		return new ViewSyncerService(config, logger, shard, config.taskID, id, cvrDB, new PipelineDriver(logger, config.log, new Snapshotter(logger, replicaFile, shard, void 0, snapshotRowCache), shard, operatorStorage.createClientGroupStorage(id), id, inspectorDelegate, () => isPriorityOpRunning() ? priorityOpRunningYieldThresholdMs : normalYieldThresholdMs, config.enableQueryPlanner, config), sub, drainCoordinator, config.log.slowHydrateThreshold, inspectorDelegate, connContextManager, customQueryTransformer, runPriorityOp);
+		return new ViewSyncerService(config, logger, shard, config.taskID, id, cvrDB, new PipelineDriver(logger, config.log, new Snapshotter(logger, replicaFile, shard, void 0, snapshotRowCache), shard, operatorStorage.createClientGroupStorage(id), id, inspectorDelegate, () => isPriorityOpRunning() ? priorityOpRunningYieldThresholdMs : normalYieldThresholdMs, config.enableQueryPlanner, config, deferredWritesBudget, queryStats), sub, drainCoordinator, config.log.slowHydrateThreshold, inspectorDelegate, connContextManager, customQueryTransformer, runPriorityOp);
 	};
 	const syncer = new Syncer(lc, config, viewSyncerFactory, upstreamDB ? (id) => new MutagenService(lc.withContext("component", "mutagen").withContext("clientGroupID", id), shard, id, upstreamDB, config, writeAuthzStorage) : void 0, pushConfig === void 0 ? void 0 : (id, connContextManager) => new PusherService(config, lc.withContext("clientGroupID", id), id, connContextManager), parent, validateLegacyJWT);
 	startAnonymousTelemetry(lc, config);
 	dbWarmup.then(() => parent.send(["ready", { ready: true }]));
-	return runUntilKilled(lc, parent, syncer);
+	try {
+		return await runUntilKilled(lc, parent, syncer);
+	} finally {
+		stopQueryStats?.();
+	}
 }
 if (!singleProcessMode()) exitAfter(() => lc, () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)));
 //#endregion

package/out/zero-cache/src/server/syncer.js.map

@@ -1 +1 @@
-{"version":3,"file":"syncer.js","names":[],"sources":["../../../../../zero-cache/src/server/syncer.ts"],"sourcesContent":["import {randomUUID} from 'node:crypto';\nimport {tmpdir} from 'node:os';\nimport path from 'node:path';\nimport {pid} from 'node:process';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {randInt} from '../../../shared/src/rand.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {DatabaseStorage} from '../../../zqlite/src/database-storage.ts';\nimport type {ValidateLegacyJWT} from '../auth/auth.ts';\nimport {tokenConfigOptions, verifyToken} from '../auth/jwt.ts';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {CustomQueryTransformer} from '../custom-queries/transform-query.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {MutagenService} from '../services/mutagen/mutagen.ts';\nimport {PusherService} from '../services/mutagen/pusher.ts';\nimport type {ReplicaState} from '../services/replicator/replicator.ts';\nimport {\n  type ConnectionContextManager,\n  ConnectionContextManagerImpl,\n} from '../services/view-syncer/connection-context-manager.ts';\nimport type {DrainCoordinator} from '../services/view-syncer/drain-coordinator.ts';\nimport {PipelineDriver} from '../services/view-syncer/pipeline-driver.ts';\nimport {SnapshotRowCache} from '../services/view-syncer/snapshot-row-cache.ts';\nimport {Snapshotter} from '../services/view-syncer/snapshotter.ts';\nimport {ViewSyncerService} from '../services/view-syncer/view-syncer.ts';\nimport {ProtocolErrorWithLevel} from '../types/error-with-level.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardID} from '../types/shards.ts';\nimport type {Subscription} from '../types/subscription.ts';\nimport {replicaFileModeSchema, replicaFileName} from '../workers/replicator.ts';\nimport {Syncer} from '../workers/syncer.ts';\nimport {startAnonymousTelemetry} from './anonymous-otel-start.ts';\nimport {InspectorDelegate} from './inspector-delegate.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {isPriorityOpRunning, runPriorityOp} from './priority-op.ts';\n\nfunction randomID() {\n  return randInt(1, Number.MAX_SAFE_INTEGER).toString(36);\n}\n\nfunction getCustomQueryConfig(\n  config: Pick<NormalizedZeroConfig, 'query' | 'getQueries'>,\n) {\n  const queryConfig = config.query?.url ? config.query : config.getQueries;\n\n  if (!queryConfig?.url) {\n    return undefined;\n  }\n\n  return {\n    url: queryConfig.url,\n    apiKey: queryConfig.apiKey,\n    allowedClientHeaders: queryConfig.allowedClientHeaders,\n    allowedRequestHeaders: queryConfig.allowedRequestHeaders,\n    forwardCookies: queryConfig.forwardCookies ?? false,\n  };\n}\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  assert(args.length >= 2, `expected [fileMode, workerIndex, ...flags]`);\n  const fileMode = v.parse(args[0], replicaFileModeSchema);\n  const workerIndex = Number(args[1]);\n  installProfileHandler(parent, `syncer-${workerIndex}`, workerIndex);\n  const config = getNormalizedZeroConfig({env, argv: args.slice(2)});\n\n  startOtelAuto(\n    createLogContext(config, 'syncer', workerIndex, false),\n    'syncer',\n    workerIndex,\n  );\n  lc = createLogContext(config, 'syncer', workerIndex);\n  initEventSink(lc, config);\n\n  const {cvr, upstream, enableCrudMutations} = config;\n\n  const replicaFile = replicaFileName(config.replica.file, fileMode);\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: 'syncer replica',\n      dbPath: replicaFile,\n    },\n    config.sqliteCorruptionChecks,\n  );\n  lc.debug?.(`running view-syncer on ${replicaFile}`);\n\n  const cvrDB = await connectPgClient(lc, cvr.db, `sync-worker-${pid}-cvr`, {\n    max: must(cvr.maxConnsPerWorker, 'cvr.maxConnsPerWorker must be set'),\n  });\n\n  const upstreamDB =\n    enableCrudMutations && upstream.type === 'pg'\n      ? await connectPgClient(lc, upstream.db, `sync-worker-${pid}-upstream`, {\n          max: must(\n            upstream.maxConnsPerWorker,\n            'upstream.maxConnsPerWorker must be set',\n          ),\n        })\n      : undefined;\n\n  const dbWarmup = Promise.allSettled([\n    warmupConnections(lc, cvrDB, 'cvr'),\n    upstreamDB ? warmupConnections(lc, upstreamDB, 'upstream') : promiseVoid,\n  ]);\n\n  const tmpDir = config.storageDBTmpDir ?? tmpdir();\n  const operatorStorage = DatabaseStorage.create(\n    lc,\n    path.join(tmpDir, `sync-worker-${randomUUID()}`),\n  );\n  const writeAuthzStorage = DatabaseStorage.create(\n    lc,\n    path.join(tmpDir, `mutagen-${randomUUID()}`),\n  );\n\n  const shard = getShardID(config);\n  const customQueryConfig = getCustomQueryConfig(config);\n  const pushConfig =\n    config.push.url === undefined && config.mutate.url === undefined\n      ? undefined\n      : {\n          ...config.push,\n          ...config.mutate,\n          url: must(\n            config.push.url ?? config.mutate.url,\n            'No push or mutate URL configured',\n          ),\n        };\n\n  /** @deprecated used in JWT validation */\n  let validateLegacyJWT: ValidateLegacyJWT | undefined = undefined;\n\n  const tokenOptions = tokenConfigOptions(config.auth ?? {});\n  if (tokenOptions.length === 1) {\n    validateLegacyJWT = async (token, {userID}) => {\n      if (!userID) {\n        throw new ProtocolErrorWithLevel(\n          {\n            kind: 'Unauthorized',\n            message: 'UserID is required for JWT validation.',\n            origin: 'zeroCache',\n          },\n          'warn',\n        );\n      }\n\n      const decoded = await verifyToken(config.auth, token, {\n        subject: userID,\n        ...(config.auth?.issuer && {issuer: config.auth.issuer}),\n        ...(config.auth?.audience && {\n          audience: config.auth.audience,\n        }),\n      });\n      return {\n        type: 'jwt',\n        raw: token,\n        decoded,\n      };\n    };\n  }\n\n  // Shared by all of the view-syncers on this worker so that the row reads\n  // performed when advancing their pipelines are done once per worker rather\n  // than once per client group.\n  const snapshotRowCache =\n    config.snapshotRowCacheSize > 0\n      ? new SnapshotRowCache(config.snapshotRowCacheSize)\n      : undefined;\n\n  const viewSyncerFactory = (\n    id: string,\n    sub: Subscription<ReplicaState>,\n    drainCoordinator: DrainCoordinator,\n  ) => {\n    const logger = lc\n      .withContext('taskID', config.taskID)\n      .withContext('component', 'view-syncer')\n      .withContext('appID', shard.appID)\n      .withContext('shardNum', shard.shardNum)\n      .withContext('clientGroupID', id)\n      .withContext('instance', randomID());\n\n    const customQueryTransformer =\n      customQueryConfig && new CustomQueryTransformer(logger, shard);\n    const connContextManager = new ConnectionContextManagerImpl(\n      logger,\n      config.auth.revalidateIntervalSeconds,\n      config.auth.retransformIntervalSeconds,\n      customQueryConfig,\n      pushConfig,\n      validateLegacyJWT,\n    );\n\n    lc.debug?.(\n      `creating view syncer. Query Planner Enabled: ${config.enableQueryPlanner}`,\n    );\n\n    const inspectorDelegate = new InspectorDelegate(customQueryTransformer);\n\n    const priorityOpRunningYieldThresholdMs = Math.max(\n      config.yieldThresholdMs / 4,\n      2,\n    );\n    const normalYieldThresholdMs = Math.max(config.yieldThresholdMs, 2);\n\n    return new ViewSyncerService(\n      config,\n      logger,\n      shard,\n      config.taskID,\n      id,\n      cvrDB,\n      new PipelineDriver(\n        logger,\n        config.log,\n        new Snapshotter(\n          logger,\n          replicaFile,\n          shard,\n          undefined,\n          snapshotRowCache,\n        ),\n        shard,\n        operatorStorage.createClientGroupStorage(id),\n        id,\n        inspectorDelegate,\n        () =>\n          isPriorityOpRunning()\n            ? priorityOpRunningYieldThresholdMs\n            : normalYieldThresholdMs,\n        config.enableQueryPlanner,\n        config,\n      ),\n      sub,\n      drainCoordinator,\n      config.log.slowHydrateThreshold,\n      inspectorDelegate,\n      connContextManager,\n      customQueryTransformer,\n      runPriorityOp,\n    );\n  };\n\n  const mutagenFactory = upstreamDB\n    ? (id: string) =>\n        new MutagenService(\n          lc\n            .withContext('component', 'mutagen')\n            .withContext('clientGroupID', id),\n          shard,\n          id,\n          upstreamDB,\n          config,\n          writeAuthzStorage,\n        )\n    : undefined;\n\n  const pusherFactory =\n    pushConfig === undefined\n      ? undefined\n      : (id: string, connContextManager: ConnectionContextManager) =>\n          new PusherService(\n            config,\n            lc.withContext('clientGroupID', id),\n            id,\n            connContextManager,\n          );\n\n  const syncer = new Syncer(\n    lc,\n    config,\n    viewSyncerFactory,\n    mutagenFactory,\n    pusherFactory,\n    parent,\n    validateLegacyJWT,\n  );\n\n  startAnonymousTelemetry(lc, config);\n\n  void dbWarmup.then(() => parent.send(['ready', {ready: true}]));\n\n  return runUntilKilled(lc, parent, syncer);\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkDA,SAAS,WAAW;CAClB,OAAO,QAAQ,GAAG,OAAO,gBAAgB,EAAE,SAAS,EAAE;AACxD;AAEA,SAAS,qBACP,QACA;CACA,MAAM,cAAc,OAAO,OAAO,MAAM,OAAO,QAAQ,OAAO;CAE9D,IAAI,CAAC,aAAa,KAChB;CAGF,OAAO;EACL,KAAK,YAAY;EACjB,QAAQ,YAAY;EACpB,sBAAsB,YAAY;EAClC,uBAAuB,YAAY;EACnC,gBAAgB,YAAY,kBAAkB;CAChD;AACF;AAGA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,OAAO,KAAK,UAAU,GAAG,4CAA4C;CACrE,MAAM,WAAW,MAAQ,KAAK,IAAI,qBAAqB;CACvD,MAAM,cAAc,OAAO,KAAK,EAAE;CAClC,sBAAsB,QAAQ,UAAU,eAAe,WAAW;CAClE,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CAEjE,cACE,iBAAiB,QAAQ,UAAU,aAAa,KAAK,GACrD,UACA,WACF;CACA,KAAK,iBAAiB,QAAQ,UAAU,WAAW;CACnD,cAAc,IAAI,MAAM;CAExB,MAAM,EAAC,KAAK,UAAU,wBAAuB;CAE7C,MAAM,cAAc,gBAAgB,OAAO,QAAQ,MAAM,QAAQ;CACjE,yCACE;EACE,WAAW;EACX,QAAQ;CACV,GACA,OAAO,sBACT;CACA,GAAG,QAAQ,0BAA0B,aAAa;CAElD,MAAM,QAAQ,MAAM,gBAAgB,IAAI,IAAI,IAAI,eAAe,IAAI,OAAO,EACxE,KAAK,KAAK,IAAI,mBAAmB,mCAAmC,EACtE,CAAC;CAED,MAAM,aACJ,uBAAuB,SAAS,SAAS,OACrC,MAAM,gBAAgB,IAAI,SAAS,IAAI,eAAe,IAAI,YAAY,EACpE,KAAK,KACH,SAAS,mBACT,wCACF,EACF,CAAC,IACD,KAAA;CAEN,MAAM,WAAW,QAAQ,WAAW,CAClC,kBAAkB,IAAI,OAAO,KAAK,GAClC,aAAa,kBAAkB,IAAI,YAAY,UAAU,IAAI,WAC/D,CAAC;CAED,MAAM,SAAS,OAAO,mBAAmB,OAAO;CAChD,MAAM,kBAAkB,gBAAgB,OACtC,IACA,KAAK,KAAK,QAAQ,eAAe,WAAW,GAAG,CACjD;CACA,MAAM,oBAAoB,gBAAgB,OACxC,IACA,KAAK,KAAK,QAAQ,WAAW,WAAW,GAAG,CAC7C;CAEA,MAAM,QAAQ,WAAW,MAAM;CAC/B,MAAM,oBAAoB,qBAAqB,MAAM;CACrD,MAAM,aACJ,OAAO,KAAK,QAAQ,KAAA,KAAa,OAAO,OAAO,QAAQ,KAAA,IACnD,KAAA,IACA;EACE,GAAG,OAAO;EACV,GAAG,OAAO;EACV,KAAK,KACH,OAAO,KAAK,OAAO,OAAO,OAAO,KACjC,kCACF;CACF;;CAGN,IAAI,oBAAmD,KAAA;CAGvD,IADqB,mBAAmB,OAAO,QAAQ,CAAC,CACpD,EAAa,WAAW,GAC1B,oBAAoB,OAAO,OAAO,EAAC,aAAY;EAC7C,IAAI,CAAC,QACH,MAAM,IAAI,uBACR;GACE,MAAM;GACN,SAAS;GACT,QAAQ;EACV,GACA,MACF;EAUF,OAAO;GACL,MAAM;GACN,KAAK;GACL,SAAA,MAVoB,YAAY,OAAO,MAAM,OAAO;IACpD,SAAS;IACT,GAAI,OAAO,MAAM,UAAU,EAAC,QAAQ,OAAO,KAAK,OAAM;IACtD,GAAI,OAAO,MAAM,YAAY,EAC3B,UAAU,OAAO,KAAK,SACxB;GACF,CAAC;EAKD;CACF;CAMF,MAAM,mBACJ,OAAO,uBAAuB,IAC1B,IAAI,iBAAiB,OAAO,oBAAoB,IAChD,KAAA;CAEN,MAAM,qBACJ,IACA,KACA,qBACG;EACH,MAAM,SAAS,GACZ,YAAY,UAAU,OAAO,MAAM,EACnC,YAAY,aAAa,aAAa,EACtC,YAAY,SAAS,MAAM,KAAK,EAChC,YAAY,YAAY,MAAM,QAAQ,EACtC,YAAY,iBAAiB,EAAE,EAC/B,YAAY,YAAY,SAAS,CAAC;EAErC,MAAM,yBACJ,qBAAqB,IAAI,uBAAuB,QAAQ,KAAK;EAC/D,MAAM,qBAAqB,IAAI,6BAC7B,QACA,OAAO,KAAK,2BACZ,OAAO,KAAK,4BACZ,mBACA,YACA,iBACF;EAEA,GAAG,QACD,gDAAgD,OAAO,oBACzD;EAEA,MAAM,oBAAoB,IAAI,kBAAkB,sBAAsB;EAEtE,MAAM,oCAAoC,KAAK,IAC7C,OAAO,mBAAmB,GAC1B,CACF;EACA,MAAM,yBAAyB,KAAK,IAAI,OAAO,kBAAkB,CAAC;EAElE,OAAO,IAAI,kBACT,QACA,QACA,OACA,OAAO,QACP,IACA,OACA,IAAI,eACF,QACA,OAAO,KACP,IAAI,YACF,QACA,aACA,OACA,KAAA,GACA,gBACF,GACA,OACA,gBAAgB,yBAAyB,EAAE,GAC3C,IACA,yBAEE,oBAAoB,IAChB,oCACA,wBACN,OAAO,oBACP,MACF,GACA,KACA,kBACA,OAAO,IAAI,sBACX,mBACA,oBACA,wBACA,aACF;CACF;CA2BA,MAAM,SAAS,IAAI,OACjB,IACA,QACA,mBA5BqB,cAClB,OACC,IAAI,eACF,GACG,YAAY,aAAa,SAAS,EAClC,YAAY,iBAAiB,EAAE,GAClC,OACA,IACA,YACA,QACA,iBACF,IACF,KAAA,GAGF,eAAe,KAAA,IACX,KAAA,KACC,IAAY,uBACX,IAAI,cACF,QACA,GAAG,YAAY,iBAAiB,EAAE,GAClC,IACA,kBACF,GAQN,QACA,iBACF;CAEA,wBAAwB,IAAI,MAAM;CAElC,SAAc,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC,CAAC;CAE9D,OAAO,eAAe,IAAI,QAAQ,MAAM;AAC1C;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}
+{"version":3,"file":"syncer.js","names":[],"sources":["../../../../../zero-cache/src/server/syncer.ts"],"sourcesContent":["import {randomUUID} from 'node:crypto';\nimport {tmpdir} from 'node:os';\nimport path from 'node:path';\nimport {pid} from 'node:process';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {randInt} from '../../../shared/src/rand.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {DatabaseStorage} from '../../../zqlite/src/database-storage.ts';\nimport type {ValidateLegacyJWT} from '../auth/auth.ts';\nimport {tokenConfigOptions, verifyToken} from '../auth/jwt.ts';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {CustomQueryTransformer} from '../custom-queries/transform-query.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {MutagenService} from '../services/mutagen/mutagen.ts';\nimport {PusherService} from '../services/mutagen/pusher.ts';\nimport type {ReplicaState} from '../services/replicator/replicator.ts';\nimport {\n  type ConnectionContextManager,\n  ConnectionContextManagerImpl,\n} from '../services/view-syncer/connection-context-manager.ts';\nimport {DeferredWritesBudget} from '../services/view-syncer/deferred-writes-budget.ts';\nimport type {DrainCoordinator} from '../services/view-syncer/drain-coordinator.ts';\nimport {PipelineDriver} from '../services/view-syncer/pipeline-driver.ts';\nimport {QueryStats} from '../services/view-syncer/query-stats.ts';\nimport {SnapshotRowCache} from '../services/view-syncer/snapshot-row-cache.ts';\nimport {Snapshotter} from '../services/view-syncer/snapshotter.ts';\nimport {ViewSyncerService} from '../services/view-syncer/view-syncer.ts';\nimport {ProtocolErrorWithLevel} from '../types/error-with-level.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardID} from '../types/shards.ts';\nimport type {Subscription} from '../types/subscription.ts';\nimport {replicaFileModeSchema, replicaFileName} from '../workers/replicator.ts';\nimport {Syncer} from '../workers/syncer.ts';\nimport {startAnonymousTelemetry} from './anonymous-otel-start.ts';\nimport {InspectorDelegate} from './inspector-delegate.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {isPriorityOpRunning, runPriorityOp} from './priority-op.ts';\n\nfunction randomID() {\n  return randInt(1, Number.MAX_SAFE_INTEGER).toString(36);\n}\n\nfunction getCustomQueryConfig(\n  config: Pick<NormalizedZeroConfig, 'query' | 'getQueries'>,\n) {\n  const queryConfig = config.query?.url ? config.query : config.getQueries;\n\n  if (!queryConfig?.url) {\n    return undefined;\n  }\n\n  return {\n    url: queryConfig.url,\n    apiKey: queryConfig.apiKey,\n    allowedClientHeaders: queryConfig.allowedClientHeaders,\n    allowedRequestHeaders: queryConfig.allowedRequestHeaders,\n    forwardCookies: queryConfig.forwardCookies ?? false,\n  };\n}\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  assert(args.length >= 2, `expected [fileMode, workerIndex, ...flags]`);\n  const fileMode = v.parse(args[0], replicaFileModeSchema);\n  const workerIndex = Number(args[1]);\n  installProfileHandler(parent, `syncer-${workerIndex}`, workerIndex);\n  const config = getNormalizedZeroConfig({env, argv: args.slice(2)});\n\n  startOtelAuto(\n    createLogContext(config, 'syncer', workerIndex, false),\n    'syncer',\n    workerIndex,\n  );\n  lc = createLogContext(config, 'syncer', workerIndex);\n  initEventSink(lc, config);\n\n  const {cvr, upstream, enableCrudMutations} = config;\n\n  const replicaFile = replicaFileName(config.replica.file, fileMode);\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: 'syncer replica',\n      dbPath: replicaFile,\n    },\n    config.sqliteCorruptionChecks,\n  );\n  lc.debug?.(`running view-syncer on ${replicaFile}`);\n\n  const cvrDB = await connectPgClient(lc, cvr.db, `sync-worker-${pid}-cvr`, {\n    max: must(cvr.maxConnsPerWorker, 'cvr.maxConnsPerWorker must be set'),\n  });\n\n  const upstreamDB =\n    enableCrudMutations && upstream.type === 'pg'\n      ? await connectPgClient(lc, upstream.db, `sync-worker-${pid}-upstream`, {\n          max: must(\n            upstream.maxConnsPerWorker,\n            'upstream.maxConnsPerWorker must be set',\n          ),\n        })\n      : undefined;\n\n  const dbWarmup = Promise.allSettled([\n    warmupConnections(lc, cvrDB, 'cvr'),\n    upstreamDB ? warmupConnections(lc, upstreamDB, 'upstream') : promiseVoid,\n  ]);\n\n  const tmpDir = config.storageDBTmpDir ?? tmpdir();\n  const operatorStorage = DatabaseStorage.create(\n    lc,\n    path.join(tmpDir, `sync-worker-${randomUUID()}`),\n  );\n  const writeAuthzStorage = DatabaseStorage.create(\n    lc,\n    path.join(tmpDir, `mutagen-${randomUUID()}`),\n  );\n\n  const shard = getShardID(config);\n  const customQueryConfig = getCustomQueryConfig(config);\n  const pushConfig =\n    config.push.url === undefined && config.mutate.url === undefined\n      ? undefined\n      : {\n          ...config.push,\n          ...config.mutate,\n          url: must(\n            config.push.url ?? config.mutate.url,\n            'No push or mutate URL configured',\n          ),\n        };\n\n  /** @deprecated used in JWT validation */\n  let validateLegacyJWT: ValidateLegacyJWT | undefined = undefined;\n\n  const tokenOptions = tokenConfigOptions(config.auth ?? {});\n  if (tokenOptions.length === 1) {\n    validateLegacyJWT = async (token, {userID}) => {\n      if (!userID) {\n        throw new ProtocolErrorWithLevel(\n          {\n            kind: 'Unauthorized',\n            message: 'UserID is required for JWT validation.',\n            origin: 'zeroCache',\n          },\n          'warn',\n        );\n      }\n\n      const decoded = await verifyToken(config.auth, token, {\n        subject: userID,\n        ...(config.auth?.issuer && {issuer: config.auth.issuer}),\n        ...(config.auth?.audience && {\n          audience: config.auth.audience,\n        }),\n      });\n      return {\n        type: 'jwt',\n        raw: token,\n        decoded,\n      };\n    };\n  }\n\n  // Shared by all of the view-syncers on this worker so that the row reads\n  // performed when advancing their pipelines are done once per worker rather\n  // than once per client group.\n  const snapshotRowCache =\n    config.snapshotRowCacheSize > 0\n      ? new SnapshotRowCache(config.snapshotRowCacheSize)\n      : undefined;\n\n  // Shared by all of the view-syncers on this worker, which each hold their\n  // own copy of the changes they are advancing through.\n  const deferredWritesBudget = config.deferIvmWrites\n    ? DeferredWritesBudget.forHeapProportion(\n        config.deferIvmWritesHeapProportion,\n      )\n    : undefined;\n  if (deferredWritesBudget) {\n    lc.info?.(\n      `Deferred IVM writes may hold up to ${deferredWritesBudget.maxRows} ` +\n        `rows (~${(deferredWritesBudget.maxBytes / 1024 ** 2).toFixed(2)} MB) ` +\n        `across client groups`,\n    );\n    getOrCreateGauge(\n      'sync',\n      'ivm.deferred-writes-reserved-rows',\n      'Rows reserved by the client groups of a sync worker to hold IVM ' +\n        'changes in memory (deferIvmWrites)',\n    ).addCallback(o => o.observe(deferredWritesBudget.reservedRows));\n    getOrCreateGauge('sync', 'ivm.deferred-writes-held-bytes', {\n      description:\n        'Estimated bytes of the IVM changes that the client groups of a sync ' +\n        'worker hold in memory (deferIvmWrites)',\n      unit: 'By',\n    }).addCallback(o => o.observe(deferredWritesBudget.heldBytes));\n  }\n\n  // Shared by all of the view-syncers on this worker, so that the work done\n  // for each query shape is logged once per worker per interval.\n  const queryStats =\n    config.log.queryStatsIntervalSeconds > 0 ? new QueryStats() : undefined;\n  const stopQueryStats = queryStats?.start(\n    lc\n      .withContext('taskID', config.taskID)\n      .withContext('component', 'view-syncer'),\n    config.log.queryStatsIntervalSeconds * 1000,\n  );\n\n  const viewSyncerFactory = (\n    id: string,\n    sub: Subscription<ReplicaState>,\n    drainCoordinator: DrainCoordinator,\n  ) => {\n    const logger = lc\n      .withContext('taskID', config.taskID)\n      .withContext('component', 'view-syncer')\n      .withContext('appID', shard.appID)\n      .withContext('shardNum', shard.shardNum)\n      .withContext('clientGroupID', id)\n      .withContext('instance', randomID());\n\n    const customQueryTransformer =\n      customQueryConfig && new CustomQueryTransformer(logger, shard);\n    const connContextManager = new ConnectionContextManagerImpl(\n      logger,\n      config.auth.revalidateIntervalSeconds,\n      config.auth.retransformIntervalSeconds,\n      customQueryConfig,\n      pushConfig,\n      validateLegacyJWT,\n    );\n\n    lc.debug?.(\n      `creating view syncer. Query Planner Enabled: ${config.enableQueryPlanner}`,\n    );\n\n    const inspectorDelegate = new InspectorDelegate(customQueryTransformer);\n\n    const priorityOpRunningYieldThresholdMs = Math.max(\n      config.yieldThresholdMs / 4,\n      2,\n    );\n    const normalYieldThresholdMs = Math.max(config.yieldThresholdMs, 2);\n\n    return new ViewSyncerService(\n      config,\n      logger,\n      shard,\n      config.taskID,\n      id,\n      cvrDB,\n      new PipelineDriver(\n        logger,\n        config.log,\n        new Snapshotter(\n          logger,\n          replicaFile,\n          shard,\n          undefined,\n          snapshotRowCache,\n        ),\n        shard,\n        operatorStorage.createClientGroupStorage(id),\n        id,\n        inspectorDelegate,\n        () =>\n          isPriorityOpRunning()\n            ? priorityOpRunningYieldThresholdMs\n            : normalYieldThresholdMs,\n        config.enableQueryPlanner,\n        config,\n        deferredWritesBudget,\n        queryStats,\n      ),\n      sub,\n      drainCoordinator,\n      config.log.slowHydrateThreshold,\n      inspectorDelegate,\n      connContextManager,\n      customQueryTransformer,\n      runPriorityOp,\n    );\n  };\n\n  const mutagenFactory = upstreamDB\n    ? (id: string) =>\n        new MutagenService(\n          lc\n            .withContext('component', 'mutagen')\n            .withContext('clientGroupID', id),\n          shard,\n          id,\n          upstreamDB,\n          config,\n          writeAuthzStorage,\n        )\n    : undefined;\n\n  const pusherFactory =\n    pushConfig === undefined\n      ? undefined\n      : (id: string, connContextManager: ConnectionContextManager) =>\n          new PusherService(\n            config,\n            lc.withContext('clientGroupID', id),\n            id,\n            connContextManager,\n          );\n\n  const syncer = new Syncer(\n    lc,\n    config,\n    viewSyncerFactory,\n    mutagenFactory,\n    pusherFactory,\n    parent,\n    validateLegacyJWT,\n  );\n\n  startAnonymousTelemetry(lc, config);\n\n  void dbWarmup.then(() => parent.send(['ready', {ready: true}]));\n\n  try {\n    return await runUntilKilled(lc, parent, syncer);\n  } finally {\n    // Logs the work done since the last interval.\n    stopQueryStats?.();\n  }\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqDA,SAAS,WAAW;CAClB,OAAO,QAAQ,GAAG,OAAO,gBAAgB,EAAE,SAAS,EAAE;AACxD;AAEA,SAAS,qBACP,QACA;CACA,MAAM,cAAc,OAAO,OAAO,MAAM,OAAO,QAAQ,OAAO;CAE9D,IAAI,CAAC,aAAa,KAChB;CAGF,OAAO;EACL,KAAK,YAAY;EACjB,QAAQ,YAAY;EACpB,sBAAsB,YAAY;EAClC,uBAAuB,YAAY;EACnC,gBAAgB,YAAY,kBAAkB;CAChD;AACF;AAGA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,OAAO,KAAK,UAAU,GAAG,4CAA4C;CACrE,MAAM,WAAW,MAAQ,KAAK,IAAI,qBAAqB;CACvD,MAAM,cAAc,OAAO,KAAK,EAAE;CAClC,sBAAsB,QAAQ,UAAU,eAAe,WAAW;CAClE,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CAEjE,cACE,iBAAiB,QAAQ,UAAU,aAAa,KAAK,GACrD,UACA,WACF;CACA,KAAK,iBAAiB,QAAQ,UAAU,WAAW;CACnD,cAAc,IAAI,MAAM;CAExB,MAAM,EAAC,KAAK,UAAU,wBAAuB;CAE7C,MAAM,cAAc,gBAAgB,OAAO,QAAQ,MAAM,QAAQ;CACjE,yCACE;EACE,WAAW;EACX,QAAQ;CACV,GACA,OAAO,sBACT;CACA,GAAG,QAAQ,0BAA0B,aAAa;CAElD,MAAM,QAAQ,MAAM,gBAAgB,IAAI,IAAI,IAAI,eAAe,IAAI,OAAO,EACxE,KAAK,KAAK,IAAI,mBAAmB,mCAAmC,EACtE,CAAC;CAED,MAAM,aACJ,uBAAuB,SAAS,SAAS,OACrC,MAAM,gBAAgB,IAAI,SAAS,IAAI,eAAe,IAAI,YAAY,EACpE,KAAK,KACH,SAAS,mBACT,wCACF,EACF,CAAC,IACD,KAAA;CAEN,MAAM,WAAW,QAAQ,WAAW,CAClC,kBAAkB,IAAI,OAAO,KAAK,GAClC,aAAa,kBAAkB,IAAI,YAAY,UAAU,IAAI,WAC/D,CAAC;CAED,MAAM,SAAS,OAAO,mBAAmB,OAAO;CAChD,MAAM,kBAAkB,gBAAgB,OACtC,IACA,KAAK,KAAK,QAAQ,eAAe,WAAW,GAAG,CACjD;CACA,MAAM,oBAAoB,gBAAgB,OACxC,IACA,KAAK,KAAK,QAAQ,WAAW,WAAW,GAAG,CAC7C;CAEA,MAAM,QAAQ,WAAW,MAAM;CAC/B,MAAM,oBAAoB,qBAAqB,MAAM;CACrD,MAAM,aACJ,OAAO,KAAK,QAAQ,KAAA,KAAa,OAAO,OAAO,QAAQ,KAAA,IACnD,KAAA,IACA;EACE,GAAG,OAAO;EACV,GAAG,OAAO;EACV,KAAK,KACH,OAAO,KAAK,OAAO,OAAO,OAAO,KACjC,kCACF;CACF;;CAGN,IAAI,oBAAmD,KAAA;CAGvD,IADqB,mBAAmB,OAAO,QAAQ,CAAC,CACpD,EAAa,WAAW,GAC1B,oBAAoB,OAAO,OAAO,EAAC,aAAY;EAC7C,IAAI,CAAC,QACH,MAAM,IAAI,uBACR;GACE,MAAM;GACN,SAAS;GACT,QAAQ;EACV,GACA,MACF;EAUF,OAAO;GACL,MAAM;GACN,KAAK;GACL,SAAA,MAVoB,YAAY,OAAO,MAAM,OAAO;IACpD,SAAS;IACT,GAAI,OAAO,MAAM,UAAU,EAAC,QAAQ,OAAO,KAAK,OAAM;IACtD,GAAI,OAAO,MAAM,YAAY,EAC3B,UAAU,OAAO,KAAK,SACxB;GACF,CAAC;EAKD;CACF;CAMF,MAAM,mBACJ,OAAO,uBAAuB,IAC1B,IAAI,iBAAiB,OAAO,oBAAoB,IAChD,KAAA;CAIN,MAAM,uBAAuB,OAAO,iBAChC,qBAAqB,kBACnB,OAAO,4BACT,IACA,KAAA;CACJ,IAAI,sBAAsB;EACxB,GAAG,OACD,sCAAsC,qBAAqB,QAAQ,WACtD,qBAAqB,WAAW,QAAQ,GAAG,QAAQ,CAAC,EAAE,0BAErE;EACA,iBACE,QACA,qCACA,oGAEF,EAAE,aAAY,MAAK,EAAE,QAAQ,qBAAqB,YAAY,CAAC;EAC/D,iBAAiB,QAAQ,kCAAkC;GACzD,aACE;GAEF,MAAM;EACR,CAAC,EAAE,aAAY,MAAK,EAAE,QAAQ,qBAAqB,SAAS,CAAC;CAC/D;CAIA,MAAM,aACJ,OAAO,IAAI,4BAA4B,IAAI,IAAI,WAAW,IAAI,KAAA;CAChE,MAAM,iBAAiB,YAAY,MACjC,GACG,YAAY,UAAU,OAAO,MAAM,EACnC,YAAY,aAAa,aAAa,GACzC,OAAO,IAAI,4BAA4B,GACzC;CAEA,MAAM,qBACJ,IACA,KACA,qBACG;EACH,MAAM,SAAS,GACZ,YAAY,UAAU,OAAO,MAAM,EACnC,YAAY,aAAa,aAAa,EACtC,YAAY,SAAS,MAAM,KAAK,EAChC,YAAY,YAAY,MAAM,QAAQ,EACtC,YAAY,iBAAiB,EAAE,EAC/B,YAAY,YAAY,SAAS,CAAC;EAErC,MAAM,yBACJ,qBAAqB,IAAI,uBAAuB,QAAQ,KAAK;EAC/D,MAAM,qBAAqB,IAAI,6BAC7B,QACA,OAAO,KAAK,2BACZ,OAAO,KAAK,4BACZ,mBACA,YACA,iBACF;EAEA,GAAG,QACD,gDAAgD,OAAO,oBACzD;EAEA,MAAM,oBAAoB,IAAI,kBAAkB,sBAAsB;EAEtE,MAAM,oCAAoC,KAAK,IAC7C,OAAO,mBAAmB,GAC1B,CACF;EACA,MAAM,yBAAyB,KAAK,IAAI,OAAO,kBAAkB,CAAC;EAElE,OAAO,IAAI,kBACT,QACA,QACA,OACA,OAAO,QACP,IACA,OACA,IAAI,eACF,QACA,OAAO,KACP,IAAI,YACF,QACA,aACA,OACA,KAAA,GACA,gBACF,GACA,OACA,gBAAgB,yBAAyB,EAAE,GAC3C,IACA,yBAEE,oBAAoB,IAChB,oCACA,wBACN,OAAO,oBACP,QACA,sBACA,UACF,GACA,KACA,kBACA,OAAO,IAAI,sBACX,mBACA,oBACA,wBACA,aACF;CACF;CA2BA,MAAM,SAAS,IAAI,OACjB,IACA,QACA,mBA5BqB,cAClB,OACC,IAAI,eACF,GACG,YAAY,aAAa,SAAS,EAClC,YAAY,iBAAiB,EAAE,GAClC,OACA,IACA,YACA,QACA,iBACF,IACF,KAAA,GAGF,eAAe,KAAA,IACX,KAAA,KACC,IAAY,uBACX,IAAI,cACF,QACA,GAAG,YAAY,iBAAiB,EAAE,GAClC,IACA,kBACF,GAQN,QACA,iBACF;CAEA,wBAAwB,IAAI,MAAM;CAElC,SAAc,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC,CAAC;CAE9D,IAAI;EACF,OAAO,MAAM,eAAe,IAAI,QAAQ,MAAM;CAChD,UAAU;EAER,iBAAiB;CACnB;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}

package/out/zero-cache/src/services/change-source/common/backfill-manager.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"backfill-manager.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/common/backfill-manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAOjD,OAAO,EAAmB,KAAK,UAAU,EAAC,MAAM,wBAAwB,CAAC;AAKzE,OAAO,KAAK,EACV,iBAAiB,EACjB,eAAe,EACf,mBAAmB,EAEnB,eAAe,EAChB,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EACV,UAAU,EACV,uBAAuB,EACvB,QAAQ,EACT,MAAM,gCAAgC,CAAC;AAMxC;;;;;GAKG;AACH,MAAM,MAAM,eAAe,GAAG;IAC5B,OAAO,EAAE,eAAe,GAAG,iBAAiB,CAAC;IAC7C,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,KAAK,gBAAgB,GAAG,CACtB,GAAG,EAAE,eAAe,KACjB,cAAc,CAAC,eAAe,CAAC,CAAC;AAuBrC;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,eAAgB,YAAW,UAAU,EAAE,QAAQ;;IAmC1D,YACE,EAAE,EAAE,UAAU,EACd,cAAc,EAAE,uBAAuB,EACvC,gBAAgB,EAAE,gBAAgB,EAClC,UAAU,GAAE,UAA6B,EACzC,YAAY,SAA0B,EACtC,YAAY,SAA0B,EACtC,oBAAoB,SAAyB,EAU9C;IAED,GAAG,CAAC,aAAa,EAAE,MAAM,EAAE,eAAe,EAAE,eAAe,EAAE,QAU5D;IAgQD;;;OAGG;IACH,QAAQ,CAAC,OAAO,EAAE,mBAAmB,GAAG,IAAI,CA2M3C;IAED,MAAM,IAAI,IAAI,CAab;CACF;AAED,uBAAe,mBAAoB,SAAQ,KAAK;IAC9C,YAAY,EAAE,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,EAM5D;CACF;AA6BD;;;;;GAKG;AACH,qBAAa,0BAA2B,SAAQ,mBAAmB;IACjE,QAAQ,CAAC,IAAI,gCAAgC;IAE7C,YAAY,EAAE,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,EAE5D;CACF"}
+{"version":3,"file":"backfill-manager.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/common/backfill-manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAUjD,OAAO,EAAmB,KAAK,UAAU,EAAC,MAAM,wBAAwB,CAAC;AAKzE,OAAO,KAAK,EACV,iBAAiB,EACjB,eAAe,EACf,mBAAmB,EAEnB,eAAe,EAChB,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EACV,UAAU,EACV,uBAAuB,EACvB,QAAQ,EACT,MAAM,gCAAgC,CAAC;AAMxC;;;;;GAKG;AACH,MAAM,MAAM,eAAe,GAAG;IAC5B,OAAO,EAAE,eAAe,GAAG,iBAAiB,CAAC;IAC7C,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,KAAK,gBAAgB,GAAG,CACtB,GAAG,EAAE,eAAe,KACjB,cAAc,CAAC,eAAe,CAAC,CAAC;AAkFrC;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,eAAgB,YAAW,UAAU,EAAE,QAAQ;;IA4C1D,YACE,EAAE,EAAE,UAAU,EACd,cAAc,EAAE,uBAAuB,EACvC,gBAAgB,EAAE,gBAAgB,EAClC,UAAU,GAAE,UAA6B,EACzC,YAAY,SAA0B,EACtC,YAAY,SAA0B,EACtC,oBAAoB,SAAyB,EAU9C;IAED,GAAG,CAAC,aAAa,EAAE,MAAM,EAAE,eAAe,EAAE,eAAe,EAAE,QAU5D;IA4bD;;;OAGG;IACH,QAAQ,CAAC,OAAO,EAAE,mBAAmB,GAAG,IAAI,CA6M3C;IAED,MAAM,IAAI,IAAI,CAab;CACF;AAED,uBAAe,mBAAoB,SAAQ,KAAK;IAC9C,YAAY,EAAE,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,EAM5D;CACF;AA6BD;;;;;GAKG;AACH,qBAAa,0BAA2B,SAAQ,mBAAmB;IACjE,QAAQ,CAAC,IAAI,gCAAgC;IAE7C,YAAY,EAAE,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,EAE5D;CACF"}

package/out/zero-cache/src/services/change-source/common/backfill-manager.js

@@ -1,9 +1,12 @@
 import { assert } from "../../../../../shared/src/asserts.js";
+import { AbortError } from "../../../../../shared/src/abort-error.js";
 import { must } from "../../../../../shared/src/must.js";
+import { promiseOrAbort } from "../../../../../shared/src/promise-race.js";
 import { stringify } from "../../../../../shared/src/bigint-json.js";
 import "../../../types/lite.js";
 import { randInt } from "../../../../../shared/src/rand.js";
 import { stateVersionFromString, stateVersionToString } from "../../../types/state-version.js";
+import { Queue } from "../../../../../shared/src/queue.js";
 import { CustomKeyMap } from "../../../../../shared/src/custom-key-map.js";
 import { resolver } from "@rocicorp/resolver";
 //#region ../zero-cache/src/services/change-source/common/backfill-manager.ts
@@ -16,6 +19,34 @@ function tableKey({ schema, name }) {
 * COMMIT/checkpoint so it can't monopolize the replica.
 */
 var COMMIT_THRESHOLD_BYTES = 8 * 1024 * 1024;
+/**
+* How many backfill messages the prefetch producer may read ahead of the
+* reservation-holding consumer loop. This bounds the in-memory buffer (~this
+* many {@link BackfillMessage}s, each up to the stream's flush threshold, e.g.
+* 64 KiB) while giving the consumer a supply of ready messages so it never
+* has to wait on the upstream COPY while holding the change-stream reservation.
+*/
+var BACKFILL_PREFETCH_CREDITS = 16;
+/**
+* While the backfill holds an open transaction but its prefetch queue is empty
+* (i.e. the upstream is momentarily behind), the consumer polls for the next
+* message at this interval so that a replication transaction that begins
+* waiting mid-fetch can preempt the backfill within this bound, rather than
+* being blocked for the entire (possibly slow) upstream read.
+*
+* If stats indicate that these stalls are highly correlative with replication
+* lag (indicating that replication changes are waiting on these polls), we
+* can address this by adding an API to the change stream multiplexer that
+* would allow backfill to relinquish an in-progress commit on demand (i.e.
+* only if replication asks for a reservation).
+*/
+var BACKFILL_WAITER_POLL_MS = 10;
+/** Interval at which in-progress backfill timing stats are logged. */
+var BACKFILL_STATS_LOG_INTERVAL_MS = 1e4;
+/** Sentinel enqueued by the prefetch producer when the backfill stream ends. */
+var DONE = Symbol("backfill-done");
+/** Sentinel returned by a polled dequeue that timed out (see {@link BACKFILL_WAITER_POLL_MS}). */
+var POLL = Symbol("backfill-poll");
 var MIN_BACKOFF_INTERVAL_MS = 2e3;
 var MAX_BACKOFF_INTERVAL_MS = 6e4;
 /**
@@ -57,8 +88,16 @@ var BackfillManager = class {
 	/** The watermark of the current transaction in the change stream. */
 	#currentTxWatermark = null;
 	#commitThresholdBytes;
-	/** Set when the change stream is canceled. No further backfills are run. */
-	#canceled = false;
+	/**
+	* Aborted when the change stream is canceled. No further backfills are run.
+	*
+	* TODO: Thread this into the backfillStream so that upstream I/O can be
+	* aborted.
+	*/
+	#controller = new AbortController();
+	get #canceled() {
+		return this.#controller.signal.aborted;
+	}
 	constructor(lc, changeStreamer, backfillStreamer, jsonFormat = "s", minBackoffMs = MIN_BACKOFF_INTERVAL_MS, maxBackoffMs = MAX_BACKOFF_INTERVAL_MS, commitThresholdBytes = COMMIT_THRESHOLD_BYTES) {
 		this.#lc = lc.withContext("component", "backfill-manager");
 		this.#changeStreamer = changeStreamer;
@@ -109,7 +148,8 @@ var BackfillManager = class {
 			const request = candidates[randInt(0, candidates.length - 1)];
 			const state = {
 				request,
-				minWatermark: ""
+				minWatermark: "",
+				stopped: new AbortController()
 			};
 			const lc = this.#lc.withContext("table", request.table.name);
 			this.#runningBackfill = state;
@@ -138,14 +178,27 @@ var BackfillManager = class {
 		const changeStream = this.#changeStreamer;
 		let backfillTx = null;
 		let uncommittedBytes = 0;
+		const stats = {
+			start: performance.now(),
+			lastLogged: performance.now(),
+			messages: 0,
+			bytes: 0,
+			commits: 0,
+			stalls: 0,
+			produceMs: 0,
+			pushMs: 0,
+			reserveMs: 0,
+			stallMs: 0,
+			waitMs: 0
+		};
 		/**
 		* @returns the new tx watermark, or null if backfill was cancelled
 		*/
 		const beginTxFor = async (msg) => {
 			assert(backfillTx === null, "Expected no active backfill transaction");
 			const lastWatermark = await changeStream.reserve("backfill");
-			if (state.canceledReason || msg.tag === "backfill" && msg.watermark < state.minWatermark) {
-				if (state.canceledReason === void 0) {
+			if (state.stopped.signal.aborted || msg.tag === "backfill" && msg.watermark < state.minWatermark) {
+				if (!state.stopped.signal.aborted) {
 					assert(msg.tag === "backfill", "Expected backfill message tag");
 					this.#stopRunningBackfill(`row key change at ${state.minWatermark} postdates backfill watermark at ${msg.watermark}`, state);
 				}
@@ -157,7 +210,7 @@ var BackfillManager = class {
 				major,
 				minor: BigInt(minor) + 1n
 			});
-			if (msg.tag === "backfill-completed" && tx < msg.watermark) tx = msg.watermark;
+			if (tx < msg.watermark) tx = msg.watermark;
 			changeStream.push([
 				"begin",
 				{
@@ -177,29 +230,138 @@ var BackfillManager = class {
 					{ watermark: backfillTx }
 				]);
 				changeStream.release(backfillTx);
+				stats.commits++;
 			}
 			backfillTx = null;
 			uncommittedBytes = 0;
 		};
-		for await (const { message: msg, byteSize } of this.#backfillStreamer(state.request)) {
-			if (this.#canceled) {
-				lc.info?.(`backfill stream canceled: change stream canceled`);
-				return;
+		const ready = new Queue();
+		const credits = new Queue();
+		for (let i = 0; i < BACKFILL_PREFETCH_CREDITS; i++) credits.enqueue(1);
+		let stopProducer = false;
+		const onCancel = () => ready.enqueueRejection(new AbortError("backfill canceled"));
+		if (!this.#canceled) this.#controller.signal.addEventListener("abort", onCancel);
+		const producer = (async () => {
+			try {
+				let t0 = performance.now();
+				for await (const value of this.#backfillStreamer(state.request)) {
+					stats.produceMs += performance.now() - t0;
+					await credits.dequeue();
+					if (stopProducer || this.#canceled) {
+						ready.enqueueRejection(new AbortError("backfill canceled"));
+						return;
+					}
+					ready.enqueue(value);
+					t0 = performance.now();
+				}
+				ready.enqueue(DONE);
+			} catch (e) {
+				ready.enqueueRejection(e);
 			}
-			const mustWaitBeforeFlush = msg.tag === "backfill-completed" && (this.#changeStreamReached(lc, msg.watermark) || backfillTx !== null && backfillTx < msg.watermark);
-			if (backfillTx && (changeStream.waiterDelay() > 0 || mustWaitBeforeFlush || uncommittedBytes >= this.#commitThresholdBytes)) commitTx();
-			mustWaitBeforeFlush && await mustWaitBeforeFlush;
-			if (msg.tag === "backfill" && msg.rowValues.length > 0 && msg.relation.rowKey.columns.length === 0) throw new MissingRowKeyError(state.request);
-			if ((backfillTx ??= await beginTxFor(msg)) === null) {
-				lc.info?.(`backfill stream canceled: ${state.canceledReason}`, state.request);
-				this.#checkAndStartBackfill();
-				return;
+		})();
+		producer.catch(() => {});
+		const nextItem = async () => {
+			if (backfillTx === null || ready.size() > 0) {
+				const n = ready.dequeue();
+				return n instanceof Promise ? await n : n;
 			}
-			await changeStream.push(["data", msg]);
-			uncommittedBytes += byteSize;
+			for (;;) {
+				if (backfillTx !== null && changeStream.waiterDelay() > 0) commitTx();
+				const n = backfillTx !== null ? ready.dequeue(POLL, BACKFILL_WAITER_POLL_MS) : ready.dequeue();
+				const t0 = performance.now();
+				const v = n instanceof Promise ? await n : n;
+				if (v === POLL) {
+					stats.stallMs += performance.now() - t0;
+					continue;
+				}
+				if (n instanceof Promise) {
+					stats.stallMs += performance.now() - t0;
+					stats.stalls++;
+				}
+				return v;
+			}
+		};
+		try {
+			for (;;) {
+				const item = await nextItem();
+				if (item === DONE) break;
+				credits.enqueue(1);
+				if (this.#canceled) {
+					lc.info?.(`backfill stream canceled: change stream canceled`);
+					return;
+				}
+				const { message: msg, byteSize } = item;
+				const waitForChangeStream = this.#changeStreamReached(lc, msg.watermark);
+				if (backfillTx && (changeStream.waiterDelay() > 0 || waitForChangeStream || uncommittedBytes >= this.#commitThresholdBytes)) commitTx();
+				if (waitForChangeStream) {
+					const t0 = performance.now();
+					await promiseOrAbort(waitForChangeStream, state.stopped.signal);
+					const elapsed = performance.now() - t0;
+					stats.waitMs += elapsed;
+					lc.info?.(`change stream reached ${msg.watermark} after waiting ${Math.round(elapsed)} ms`);
+				}
+				if (msg.tag === "backfill" && msg.rowValues.length > 0 && msg.relation.rowKey.columns.length === 0) throw new MissingRowKeyError(state.request);
+				if (backfillTx === null) {
+					const t0 = performance.now();
+					backfillTx = await beginTxFor(msg);
+					stats.reserveMs += performance.now() - t0;
+					if (backfillTx === null) {
+						lc.info?.(`backfill stream canceled: ${state.stopped.signal.reason}`, state.request);
+						this.#checkAndStartBackfill();
+						return;
+					}
+				}
+				const t0 = performance.now();
+				await changeStream.push(["data", msg]);
+				stats.pushMs += performance.now() - t0;
+				uncommittedBytes += byteSize;
+				stats.messages++;
+				stats.bytes += byteSize;
+				this.#maybeLogBackfillStats(lc, stats);
+			}
+			if (backfillTx) commitTx();
+			lc.debug?.(`backfill stream exited`);
+		} finally {
+			stopProducer = true;
+			credits.enqueue(1);
+			if (!this.#canceled) await producer;
+			this.#controller.signal.removeEventListener("abort", onCancel);
+			this.#logBackfillStats(lc, stats);
+		}
+	}
+	#maybeLogBackfillStats(lc, stats) {
+		const now = performance.now();
+		if (now - stats.lastLogged >= BACKFILL_STATS_LOG_INTERVAL_MS) {
+			this.#logBackfillStats(lc, stats);
+			stats.lastLogged = now;
 		}
-		backfillTx && commitTx();
-		lc.debug?.(`backfill stream exited`, state.canceledReason ?? "");
+	}
+	/**
+	* Logs where the backfill spent its time. `produceMs` (upstream read +
+	* parse/decode) accrues on the prefetch producer concurrently with the
+	* consumer's `pushMs` (downstream backpressure), so the two can overlap and
+	* their sum can exceed `elapsedMs`. `reserveMs` is time spent (re)acquiring
+	* the change-stream reservation, and `stallMs` is time the consumer waited on
+	* an empty prefetch queue (i.e. blocked on the upstream) — a large `stallMs`
+	* indicates an upstream-bound backfill. `waitMs` is time spent waiting
+	* (without the reservation) for the change stream to reach the backfill
+	* watermark, during which the upstream snapshot is held open.
+	*/
+	#logBackfillStats(lc, stats) {
+		const elapsed = performance.now() - stats.start;
+		lc.info?.(`backfill timing`, {
+			messages: stats.messages,
+			bytes: stats.bytes,
+			commits: stats.commits,
+			stalls: stats.stalls,
+			elapsedMs: Math.round(elapsed),
+			produceMs: Math.round(stats.produceMs),
+			pushMs: Math.round(stats.pushMs),
+			reserveMs: Math.round(stats.reserveMs),
+			stallMs: Math.round(stats.stallMs),
+			waitMs: Math.round(stats.waitMs),
+			throughputKBps: elapsed > 0 ? Math.round(stats.bytes / 1024 / (elapsed / 1e3)) : 0
+		});
 	}
 	#backfillRunningFor(table) {
 		const state = this.#runningBackfill;
@@ -214,7 +376,7 @@ var BackfillManager = class {
 	#stopRunningBackfill(reason, instance) {
 		const backfill = this.#runningBackfill;
 		if (backfill && backfill === (instance ?? backfill)) {
-			backfill.canceledReason = reason;
+			backfill.stopped.abort(reason);
 			this.#runningBackfill = null;
 			reason && this.#lc.info?.(`canceling backfill:`, reason);
 		}
@@ -278,7 +440,7 @@ var BackfillManager = class {
 						name,
 						metadata
 					},
-					columns: backfill
+					columns: Object.fromEntries(Object.entries(backfill).map(([col, id]) => [col, { id }]))
 				});
 				break;
 			}
@@ -317,7 +479,7 @@ var BackfillManager = class {
 							...table,
 							metadata
 						},
-						columns: { [column.name]: backfill }
+						columns: { [column.name]: { id: backfill } }
 					});
 					else this.#setRequiredBackfill(tag, {
 						...backfillRequest,
@@ -327,7 +489,7 @@ var BackfillManager = class {
 						},
 						columns: {
 							...backfillRequest.columns,
-							[column.name]: backfill
+							[column.name]: { id: backfill }
 						}
 					});
 				}
@@ -396,7 +558,7 @@ var BackfillManager = class {
 		}
 	}
 	cancel() {
-		this.#canceled = true;
+		this.#controller.abort("canceled");
 		this.#stopRunningBackfill(`change stream canceled`);
 		clearTimeout(this.#backfillRetryTimer);
 		this.#backfillRetryTimer = void 0;