@rocicorp/zero 1.11.0-canary.2 → 1.11.0-canary.20

package/out/zero-cache/src/server/syncer-assigner.js

@@ -0,0 +1,72 @@
+import { h32 } from "../../../shared/src/hash.js";
+//#region ../zero-cache/src/server/syncer-assigner.ts
+var SyncerAssigner = class {
+	#taskID;
+	#numWorkers;
+	#assignments = /* @__PURE__ */ new Map();
+	#workerLoads;
+	#nextGeneration = 0;
+	constructor(taskID, numWorkers) {
+		this.#taskID = taskID;
+		this.#numWorkers = numWorkers;
+		this.#workerLoads = new Array(numWorkers).fill(0);
+	}
+	assign(clientGroupID) {
+		if (this.#numWorkers <= 0) return {
+			worker: 0,
+			generation: 0
+		};
+		const generation = ++this.#nextGeneration;
+		const existing = this.#assignments.get(clientGroupID);
+		if (existing !== void 0) {
+			const updated = {
+				worker: existing.worker,
+				generation
+			};
+			this.#assignments.set(clientGroupID, updated);
+			return updated;
+		}
+		let minLoad = Infinity;
+		const tied = [];
+		for (let i = 0; i < this.#numWorkers; i++) {
+			const load = this.#workerLoads[i];
+			if (load < minLoad) {
+				minLoad = load;
+				tied.length = 0;
+				tied.push(i);
+			} else if (load === minLoad) tied.push(i);
+		}
+		const chosen = tied.length === 1 ? tied[0] : tied[h32(this.#taskID + "/" + clientGroupID) % tied.length];
+		this.#workerLoads[chosen]++;
+		const assignment = {
+			worker: chosen,
+			generation
+		};
+		this.#assignments.set(clientGroupID, assignment);
+		return assignment;
+	}
+	release(clientGroupID, workerIndex, generation) {
+		const existing = this.#assignments.get(clientGroupID);
+		if (existing === void 0 || existing.worker !== workerIndex) return;
+		if (generation !== void 0 && existing.generation !== generation) return;
+		this.#decrementLoad(workerIndex);
+		this.#assignments.delete(clientGroupID);
+	}
+	getWorkerLoad(workerIndex) {
+		return this.#workerLoads[workerIndex] ?? 0;
+	}
+	getAssignment(clientGroupID) {
+		return this.#assignments.get(clientGroupID)?.worker;
+	}
+	#decrementLoad(worker) {
+		if (this.#workerLoads[worker] > 0) this.#workerLoads[worker]--;
+	}
+	destroy() {
+		this.#assignments.clear();
+		this.#workerLoads.fill(0);
+	}
+};
+//#endregion
+export { SyncerAssigner };
+
+//# sourceMappingURL=syncer-assigner.js.map

package/out/zero-cache/src/server/syncer-assigner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"syncer-assigner.js","names":["#taskID","#numWorkers","#assignments","#workerLoads","#nextGeneration","#decrementLoad"],"sources":["../../../../../zero-cache/src/server/syncer-assigner.ts"],"sourcesContent":["import {h32} from '../../../shared/src/hash.ts';\n\nexport type Assignment = {\n  readonly worker: number;\n  readonly generation: number;\n};\n\nexport class SyncerAssigner {\n  readonly #taskID: string;\n  readonly #numWorkers: number;\n  readonly #assignments = new Map<string, Assignment>();\n  readonly #workerLoads: number[];\n  #nextGeneration = 0;\n\n  constructor(taskID: string, numWorkers: number) {\n    this.#taskID = taskID;\n    this.#numWorkers = numWorkers;\n    this.#workerLoads = new Array<number>(numWorkers).fill(0);\n  }\n\n  assign(clientGroupID: string): Assignment {\n    if (this.#numWorkers <= 0) {\n      return {worker: 0, generation: 0};\n    }\n\n    const generation = ++this.#nextGeneration;\n    const existing = this.#assignments.get(clientGroupID);\n    if (existing !== undefined) {\n      const updated: Assignment = {worker: existing.worker, generation};\n      this.#assignments.set(clientGroupID, updated);\n      return updated;\n    }\n\n    // Find worker(s) with minimum active load\n    let minLoad = Infinity;\n    const tied: number[] = [];\n    for (let i = 0; i < this.#numWorkers; i++) {\n      const load = this.#workerLoads[i];\n      if (load < minLoad) {\n        minLoad = load;\n        tied.length = 0;\n        tied.push(i);\n      } else if (load === minLoad) {\n        tied.push(i);\n      }\n    }\n\n    const chosen =\n      tied.length === 1\n        ? tied[0]\n        : tied[h32(this.#taskID + '/' + clientGroupID) % tied.length];\n\n    this.#workerLoads[chosen]++;\n    const assignment: Assignment = {worker: chosen, generation};\n    this.#assignments.set(clientGroupID, assignment);\n    return assignment;\n  }\n\n  release(\n    clientGroupID: string,\n    workerIndex: number,\n    generation?: number,\n  ): void {\n    const existing = this.#assignments.get(clientGroupID);\n    if (existing === undefined || existing.worker !== workerIndex) {\n      return;\n    }\n    if (generation !== undefined && existing.generation !== generation) {\n      return;\n    }\n    this.#decrementLoad(workerIndex);\n    this.#assignments.delete(clientGroupID);\n  }\n\n  getWorkerLoad(workerIndex: number): number {\n    return this.#workerLoads[workerIndex] ?? 0;\n  }\n\n  getAssignment(clientGroupID: string): number | undefined {\n    return this.#assignments.get(clientGroupID)?.worker;\n  }\n\n  #decrementLoad(worker: number): void {\n    if (this.#workerLoads[worker] > 0) {\n      this.#workerLoads[worker]--;\n    }\n  }\n\n  destroy(): void {\n    this.#assignments.clear();\n    this.#workerLoads.fill(0);\n  }\n}\n"],"mappings":";;AAOA,IAAa,iBAAb,MAA4B;CAC1B;CACA;CACA,+BAAwB,IAAI,IAAwB;CACpD;CACA,kBAAkB;CAElB,YAAY,QAAgB,YAAoB;EAC9C,KAAKA,UAAU;EACf,KAAKC,cAAc;EACnB,KAAKE,eAAe,IAAI,MAAc,UAAU,EAAE,KAAK,CAAC;CAC1D;CAEA,OAAO,eAAmC;EACxC,IAAI,KAAKF,eAAe,GACtB,OAAO;GAAC,QAAQ;GAAG,YAAY;EAAC;EAGlC,MAAM,aAAa,EAAE,KAAKG;EAC1B,MAAM,WAAW,KAAKF,aAAa,IAAI,aAAa;EACpD,IAAI,aAAa,KAAA,GAAW;GAC1B,MAAM,UAAsB;IAAC,QAAQ,SAAS;IAAQ;GAAU;GAChE,KAAKA,aAAa,IAAI,eAAe,OAAO;GAC5C,OAAO;EACT;EAGA,IAAI,UAAU;EACd,MAAM,OAAiB,CAAC;EACxB,KAAK,IAAI,IAAI,GAAG,IAAI,KAAKD,aAAa,KAAK;GACzC,MAAM,OAAO,KAAKE,aAAa;GAC/B,IAAI,OAAO,SAAS;IAClB,UAAU;IACV,KAAK,SAAS;IACd,KAAK,KAAK,CAAC;GACb,OAAO,IAAI,SAAS,SAClB,KAAK,KAAK,CAAC;EAEf;EAEA,MAAM,SACJ,KAAK,WAAW,IACZ,KAAK,KACL,KAAK,IAAI,KAAKH,UAAU,MAAM,aAAa,IAAI,KAAK;EAE1D,KAAKG,aAAa;EAClB,MAAM,aAAyB;GAAC,QAAQ;GAAQ;EAAU;EAC1D,KAAKD,aAAa,IAAI,eAAe,UAAU;EAC/C,OAAO;CACT;CAEA,QACE,eACA,aACA,YACM;EACN,MAAM,WAAW,KAAKA,aAAa,IAAI,aAAa;EACpD,IAAI,aAAa,KAAA,KAAa,SAAS,WAAW,aAChD;EAEF,IAAI,eAAe,KAAA,KAAa,SAAS,eAAe,YACtD;EAEF,KAAKG,eAAe,WAAW;EAC/B,KAAKH,aAAa,OAAO,aAAa;CACxC;CAEA,cAAc,aAA6B;EACzC,OAAO,KAAKC,aAAa,gBAAgB;CAC3C;CAEA,cAAc,eAA2C;EACvD,OAAO,KAAKD,aAAa,IAAI,aAAa,GAAG;CAC/C;CAEA,eAAe,QAAsB;EACnC,IAAI,KAAKC,aAAa,UAAU,GAC9B,KAAKA,aAAa;CAEtB;CAEA,UAAgB;EACd,KAAKD,aAAa,MAAM;EACxB,KAAKC,aAAa,KAAK,CAAC;CAC1B;AACF"}

package/out/zero-cache/src/server/syncer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"syncer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/syncer.ts"],"names":[],"mappings":"AAiCA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAoC/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CAiNf"}
+{"version":3,"file":"syncer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/syncer.ts"],"names":[],"mappings":"AAqCA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAqC/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CA6Qf"}

package/out/zero-cache/src/server/syncer.js

@@ -6,12 +6,14 @@ import { parentWorker, singleProcessMode } from "../types/processes.js";
 import { getShardID } from "../types/shards.js";
 import { getNormalizedZeroConfig } from "../config/zero-config.js";
 import { randInt } from "../../../shared/src/rand.js";
+import { getOrCreateGauge } from "../observability/metrics.js";
 import { Snapshotter } from "../services/view-syncer/snapshotter.js";
 import { PipelineDriver } from "../services/view-syncer/pipeline-driver.js";
 import { connectPgClient } from "../types/pg.js";
 import { registerSQLiteCorruptionDiagnosticTarget } from "../db/sqlite-corruption.js";
 import { exitAfter, runUntilKilled } from "../services/life-cycle.js";
 import { createLogContext } from "./logging.js";
+import { installProfileHandler } from "../types/profiler.js";
 import { warmupConnections } from "../db/warmup.js";
 import { initEventSink } from "../observability/events.js";
 import { startOtelAuto } from "./otel-start.js";
@@ -24,6 +26,9 @@ import { CustomQueryTransformer } from "../custom-queries/transform-query.js";
 import { MutagenService } from "../services/mutagen/mutagen.js";
 import { PusherService } from "../services/mutagen/pusher.js";
 import { ConnectionContextManagerImpl } from "../services/view-syncer/connection-context-manager.js";
+import { DeferredWritesBudget } from "../services/view-syncer/deferred-writes-budget.js";
+import { QueryStats } from "../services/view-syncer/query-stats.js";
+import { SnapshotRowCache } from "../services/view-syncer/snapshot-row-cache.js";
 import { ViewSyncerService } from "../services/view-syncer/view-syncer.js";
 import { Syncer } from "../workers/syncer.js";
 import { InspectorDelegate } from "./inspector-delegate.js";
@@ -53,6 +58,7 @@ async function runWorker(parent, env, ...args) {
 	assert(args.length >= 2, `expected [fileMode, workerIndex, ...flags]`);
 	const fileMode = parse(args[0], replicaFileModeSchema);
 	const workerIndex = Number(args[1]);
+	installProfileHandler(parent, `syncer-${workerIndex}`, workerIndex);
 	const config = getNormalizedZeroConfig({
 		env,
 		argv: args.slice(2)
@@ -98,6 +104,18 @@ async function runWorker(parent, env, ...args) {
 			})
 		};
 	};
+	const snapshotRowCache = config.snapshotRowCacheSize > 0 ? new SnapshotRowCache(config.snapshotRowCacheSize) : void 0;
+	const deferredWritesBudget = config.deferIvmWrites ? DeferredWritesBudget.forHeapProportion(config.deferIvmWritesHeapProportion) : void 0;
+	if (deferredWritesBudget) {
+		lc.info?.(`Deferred IVM writes may hold up to ${deferredWritesBudget.maxRows} rows (~${(deferredWritesBudget.maxBytes / 1024 ** 2).toFixed(2)} MB) across client groups`);
+		getOrCreateGauge("sync", "ivm.deferred-writes-reserved-rows", "Rows reserved by the client groups of a sync worker to hold IVM changes in memory (deferIvmWrites)").addCallback((o) => o.observe(deferredWritesBudget.reservedRows));
+		getOrCreateGauge("sync", "ivm.deferred-writes-held-bytes", {
+			description: "Estimated bytes of the IVM changes that the client groups of a sync worker hold in memory (deferIvmWrites)",
+			unit: "By"
+		}).addCallback((o) => o.observe(deferredWritesBudget.heldBytes));
+	}
+	const queryStats = config.log.queryStatsIntervalSeconds > 0 ? new QueryStats() : void 0;
+	const stopQueryStats = queryStats?.start(lc.withContext("taskID", config.taskID).withContext("component", "view-syncer"), config.log.queryStatsIntervalSeconds * 1e3);
 	const viewSyncerFactory = (id, sub, drainCoordinator) => {
 		const logger = lc.withContext("taskID", config.taskID).withContext("component", "view-syncer").withContext("appID", shard.appID).withContext("shardNum", shard.shardNum).withContext("clientGroupID", id).withContext("instance", randomID());
 		const customQueryTransformer = customQueryConfig && new CustomQueryTransformer(logger, shard);
@@ -106,12 +124,16 @@ async function runWorker(parent, env, ...args) {
 		const inspectorDelegate = new InspectorDelegate(customQueryTransformer);
 		const priorityOpRunningYieldThresholdMs = Math.max(config.yieldThresholdMs / 4, 2);
 		const normalYieldThresholdMs = Math.max(config.yieldThresholdMs, 2);
-		return new ViewSyncerService(config, logger, shard, config.taskID, id, cvrDB, new PipelineDriver(logger, config.log, new Snapshotter(logger, replicaFile, shard), shard, operatorStorage.createClientGroupStorage(id), id, inspectorDelegate, () => isPriorityOpRunning() ? priorityOpRunningYieldThresholdMs : normalYieldThresholdMs, config.enableQueryPlanner, config), sub, drainCoordinator, config.log.slowHydrateThreshold, inspectorDelegate, connContextManager, customQueryTransformer, runPriorityOp);
+		return new ViewSyncerService(config, logger, shard, config.taskID, id, cvrDB, new PipelineDriver(logger, config.log, new Snapshotter(logger, replicaFile, shard, void 0, snapshotRowCache), shard, operatorStorage.createClientGroupStorage(id), id, inspectorDelegate, () => isPriorityOpRunning() ? priorityOpRunningYieldThresholdMs : normalYieldThresholdMs, config.enableQueryPlanner, config, deferredWritesBudget, queryStats), sub, drainCoordinator, config.log.slowHydrateThreshold, inspectorDelegate, connContextManager, customQueryTransformer, runPriorityOp);
 	};
 	const syncer = new Syncer(lc, config, viewSyncerFactory, upstreamDB ? (id) => new MutagenService(lc.withContext("component", "mutagen").withContext("clientGroupID", id), shard, id, upstreamDB, config, writeAuthzStorage) : void 0, pushConfig === void 0 ? void 0 : (id, connContextManager) => new PusherService(config, lc.withContext("clientGroupID", id), id, connContextManager), parent, validateLegacyJWT);
 	startAnonymousTelemetry(lc, config);
 	dbWarmup.then(() => parent.send(["ready", { ready: true }]));
-	return runUntilKilled(lc, parent, syncer);
+	try {
+		return await runUntilKilled(lc, parent, syncer);
+	} finally {
+		stopQueryStats?.();
+	}
 }
 if (!singleProcessMode()) exitAfter(() => lc, () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)));
 //#endregion

package/out/zero-cache/src/server/syncer.js.map

@@ -1 +1 @@
-{"version":3,"file":"syncer.js","names":[],"sources":["../../../../../zero-cache/src/server/syncer.ts"],"sourcesContent":["import {randomUUID} from 'node:crypto';\nimport {tmpdir} from 'node:os';\nimport path from 'node:path';\nimport {pid} from 'node:process';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {randInt} from '../../../shared/src/rand.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {DatabaseStorage} from '../../../zqlite/src/database-storage.ts';\nimport type {ValidateLegacyJWT} from '../auth/auth.ts';\nimport {tokenConfigOptions, verifyToken} from '../auth/jwt.ts';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {CustomQueryTransformer} from '../custom-queries/transform-query.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {MutagenService} from '../services/mutagen/mutagen.ts';\nimport {PusherService} from '../services/mutagen/pusher.ts';\nimport type {ReplicaState} from '../services/replicator/replicator.ts';\nimport {\n  type ConnectionContextManager,\n  ConnectionContextManagerImpl,\n} from '../services/view-syncer/connection-context-manager.ts';\nimport type {DrainCoordinator} from '../services/view-syncer/drain-coordinator.ts';\nimport {PipelineDriver} from '../services/view-syncer/pipeline-driver.ts';\nimport {Snapshotter} from '../services/view-syncer/snapshotter.ts';\nimport {ViewSyncerService} from '../services/view-syncer/view-syncer.ts';\nimport {ProtocolErrorWithLevel} from '../types/error-with-level.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {getShardID} from '../types/shards.ts';\nimport type {Subscription} from '../types/subscription.ts';\nimport {replicaFileModeSchema, replicaFileName} from '../workers/replicator.ts';\nimport {Syncer} from '../workers/syncer.ts';\nimport {startAnonymousTelemetry} from './anonymous-otel-start.ts';\nimport {InspectorDelegate} from './inspector-delegate.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {isPriorityOpRunning, runPriorityOp} from './priority-op.ts';\n\nfunction randomID() {\n  return randInt(1, Number.MAX_SAFE_INTEGER).toString(36);\n}\n\nfunction getCustomQueryConfig(\n  config: Pick<NormalizedZeroConfig, 'query' | 'getQueries'>,\n) {\n  const queryConfig = config.query?.url ? config.query : config.getQueries;\n\n  if (!queryConfig?.url) {\n    return undefined;\n  }\n\n  return {\n    url: queryConfig.url,\n    apiKey: queryConfig.apiKey,\n    allowedClientHeaders: queryConfig.allowedClientHeaders,\n    allowedRequestHeaders: queryConfig.allowedRequestHeaders,\n    forwardCookies: queryConfig.forwardCookies ?? false,\n  };\n}\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  assert(args.length >= 2, `expected [fileMode, workerIndex, ...flags]`);\n  const fileMode = v.parse(args[0], replicaFileModeSchema);\n  const workerIndex = Number(args[1]);\n  const config = getNormalizedZeroConfig({env, argv: args.slice(2)});\n\n  startOtelAuto(\n    createLogContext(config, 'syncer', workerIndex, false),\n    'syncer',\n    workerIndex,\n  );\n  lc = createLogContext(config, 'syncer', workerIndex);\n  initEventSink(lc, config);\n\n  const {cvr, upstream, enableCrudMutations} = config;\n\n  const replicaFile = replicaFileName(config.replica.file, fileMode);\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: 'syncer replica',\n      dbPath: replicaFile,\n    },\n    config.sqliteCorruptionChecks,\n  );\n  lc.debug?.(`running view-syncer on ${replicaFile}`);\n\n  const cvrDB = await connectPgClient(lc, cvr.db, `sync-worker-${pid}-cvr`, {\n    max: must(cvr.maxConnsPerWorker, 'cvr.maxConnsPerWorker must be set'),\n  });\n\n  const upstreamDB =\n    enableCrudMutations && upstream.type === 'pg'\n      ? await connectPgClient(lc, upstream.db, `sync-worker-${pid}-upstream`, {\n          max: must(\n            upstream.maxConnsPerWorker,\n            'upstream.maxConnsPerWorker must be set',\n          ),\n        })\n      : undefined;\n\n  const dbWarmup = Promise.allSettled([\n    warmupConnections(lc, cvrDB, 'cvr'),\n    upstreamDB ? warmupConnections(lc, upstreamDB, 'upstream') : promiseVoid,\n  ]);\n\n  const tmpDir = config.storageDBTmpDir ?? tmpdir();\n  const operatorStorage = DatabaseStorage.create(\n    lc,\n    path.join(tmpDir, `sync-worker-${randomUUID()}`),\n  );\n  const writeAuthzStorage = DatabaseStorage.create(\n    lc,\n    path.join(tmpDir, `mutagen-${randomUUID()}`),\n  );\n\n  const shard = getShardID(config);\n  const customQueryConfig = getCustomQueryConfig(config);\n  const pushConfig =\n    config.push.url === undefined && config.mutate.url === undefined\n      ? undefined\n      : {\n          ...config.push,\n          ...config.mutate,\n          url: must(\n            config.push.url ?? config.mutate.url,\n            'No push or mutate URL configured',\n          ),\n        };\n\n  /** @deprecated used in JWT validation */\n  let validateLegacyJWT: ValidateLegacyJWT | undefined = undefined;\n\n  const tokenOptions = tokenConfigOptions(config.auth ?? {});\n  if (tokenOptions.length === 1) {\n    validateLegacyJWT = async (token, {userID}) => {\n      if (!userID) {\n        throw new ProtocolErrorWithLevel(\n          {\n            kind: 'Unauthorized',\n            message: 'UserID is required for JWT validation.',\n            origin: 'zeroCache',\n          },\n          'warn',\n        );\n      }\n\n      const decoded = await verifyToken(config.auth, token, {\n        subject: userID,\n        ...(config.auth?.issuer && {issuer: config.auth.issuer}),\n        ...(config.auth?.audience && {\n          audience: config.auth.audience,\n        }),\n      });\n      return {\n        type: 'jwt',\n        raw: token,\n        decoded,\n      };\n    };\n  }\n\n  const viewSyncerFactory = (\n    id: string,\n    sub: Subscription<ReplicaState>,\n    drainCoordinator: DrainCoordinator,\n  ) => {\n    const logger = lc\n      .withContext('taskID', config.taskID)\n      .withContext('component', 'view-syncer')\n      .withContext('appID', shard.appID)\n      .withContext('shardNum', shard.shardNum)\n      .withContext('clientGroupID', id)\n      .withContext('instance', randomID());\n\n    const customQueryTransformer =\n      customQueryConfig && new CustomQueryTransformer(logger, shard);\n    const connContextManager = new ConnectionContextManagerImpl(\n      logger,\n      config.auth.revalidateIntervalSeconds,\n      config.auth.retransformIntervalSeconds,\n      customQueryConfig,\n      pushConfig,\n      validateLegacyJWT,\n    );\n\n    lc.debug?.(\n      `creating view syncer. Query Planner Enabled: ${config.enableQueryPlanner}`,\n    );\n\n    const inspectorDelegate = new InspectorDelegate(customQueryTransformer);\n\n    const priorityOpRunningYieldThresholdMs = Math.max(\n      config.yieldThresholdMs / 4,\n      2,\n    );\n    const normalYieldThresholdMs = Math.max(config.yieldThresholdMs, 2);\n\n    return new ViewSyncerService(\n      config,\n      logger,\n      shard,\n      config.taskID,\n      id,\n      cvrDB,\n      new PipelineDriver(\n        logger,\n        config.log,\n        new Snapshotter(logger, replicaFile, shard),\n        shard,\n        operatorStorage.createClientGroupStorage(id),\n        id,\n        inspectorDelegate,\n        () =>\n          isPriorityOpRunning()\n            ? priorityOpRunningYieldThresholdMs\n            : normalYieldThresholdMs,\n        config.enableQueryPlanner,\n        config,\n      ),\n      sub,\n      drainCoordinator,\n      config.log.slowHydrateThreshold,\n      inspectorDelegate,\n      connContextManager,\n      customQueryTransformer,\n      runPriorityOp,\n    );\n  };\n\n  const mutagenFactory = upstreamDB\n    ? (id: string) =>\n        new MutagenService(\n          lc\n            .withContext('component', 'mutagen')\n            .withContext('clientGroupID', id),\n          shard,\n          id,\n          upstreamDB,\n          config,\n          writeAuthzStorage,\n        )\n    : undefined;\n\n  const pusherFactory =\n    pushConfig === undefined\n      ? undefined\n      : (id: string, connContextManager: ConnectionContextManager) =>\n          new PusherService(\n            config,\n            lc.withContext('clientGroupID', id),\n            id,\n            connContextManager,\n          );\n\n  const syncer = new Syncer(\n    lc,\n    config,\n    viewSyncerFactory,\n    mutagenFactory,\n    pusherFactory,\n    parent,\n    validateLegacyJWT,\n  );\n\n  startAnonymousTelemetry(lc, config);\n\n  void dbWarmup.then(() => parent.send(['ready', {ready: true}]));\n\n  return runUntilKilled(lc, parent, syncer);\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAgDA,SAAS,WAAW;CAClB,OAAO,QAAQ,GAAG,OAAO,gBAAgB,EAAE,SAAS,EAAE;AACxD;AAEA,SAAS,qBACP,QACA;CACA,MAAM,cAAc,OAAO,OAAO,MAAM,OAAO,QAAQ,OAAO;CAE9D,IAAI,CAAC,aAAa,KAChB;CAGF,OAAO;EACL,KAAK,YAAY;EACjB,QAAQ,YAAY;EACpB,sBAAsB,YAAY;EAClC,uBAAuB,YAAY;EACnC,gBAAgB,YAAY,kBAAkB;CAChD;AACF;AAGA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,OAAO,KAAK,UAAU,GAAG,4CAA4C;CACrE,MAAM,WAAW,MAAQ,KAAK,IAAI,qBAAqB;CACvD,MAAM,cAAc,OAAO,KAAK,EAAE;CAClC,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CAEjE,cACE,iBAAiB,QAAQ,UAAU,aAAa,KAAK,GACrD,UACA,WACF;CACA,KAAK,iBAAiB,QAAQ,UAAU,WAAW;CACnD,cAAc,IAAI,MAAM;CAExB,MAAM,EAAC,KAAK,UAAU,wBAAuB;CAE7C,MAAM,cAAc,gBAAgB,OAAO,QAAQ,MAAM,QAAQ;CACjE,yCACE;EACE,WAAW;EACX,QAAQ;CACV,GACA,OAAO,sBACT;CACA,GAAG,QAAQ,0BAA0B,aAAa;CAElD,MAAM,QAAQ,MAAM,gBAAgB,IAAI,IAAI,IAAI,eAAe,IAAI,OAAO,EACxE,KAAK,KAAK,IAAI,mBAAmB,mCAAmC,EACtE,CAAC;CAED,MAAM,aACJ,uBAAuB,SAAS,SAAS,OACrC,MAAM,gBAAgB,IAAI,SAAS,IAAI,eAAe,IAAI,YAAY,EACpE,KAAK,KACH,SAAS,mBACT,wCACF,EACF,CAAC,IACD,KAAA;CAEN,MAAM,WAAW,QAAQ,WAAW,CAClC,kBAAkB,IAAI,OAAO,KAAK,GAClC,aAAa,kBAAkB,IAAI,YAAY,UAAU,IAAI,WAC/D,CAAC;CAED,MAAM,SAAS,OAAO,mBAAmB,OAAO;CAChD,MAAM,kBAAkB,gBAAgB,OACtC,IACA,KAAK,KAAK,QAAQ,eAAe,WAAW,GAAG,CACjD;CACA,MAAM,oBAAoB,gBAAgB,OACxC,IACA,KAAK,KAAK,QAAQ,WAAW,WAAW,GAAG,CAC7C;CAEA,MAAM,QAAQ,WAAW,MAAM;CAC/B,MAAM,oBAAoB,qBAAqB,MAAM;CACrD,MAAM,aACJ,OAAO,KAAK,QAAQ,KAAA,KAAa,OAAO,OAAO,QAAQ,KAAA,IACnD,KAAA,IACA;EACE,GAAG,OAAO;EACV,GAAG,OAAO;EACV,KAAK,KACH,OAAO,KAAK,OAAO,OAAO,OAAO,KACjC,kCACF;CACF;;CAGN,IAAI,oBAAmD,KAAA;CAGvD,IADqB,mBAAmB,OAAO,QAAQ,CAAC,CACpD,EAAa,WAAW,GAC1B,oBAAoB,OAAO,OAAO,EAAC,aAAY;EAC7C,IAAI,CAAC,QACH,MAAM,IAAI,uBACR;GACE,MAAM;GACN,SAAS;GACT,QAAQ;EACV,GACA,MACF;EAUF,OAAO;GACL,MAAM;GACN,KAAK;GACL,SAAA,MAVoB,YAAY,OAAO,MAAM,OAAO;IACpD,SAAS;IACT,GAAI,OAAO,MAAM,UAAU,EAAC,QAAQ,OAAO,KAAK,OAAM;IACtD,GAAI,OAAO,MAAM,YAAY,EAC3B,UAAU,OAAO,KAAK,SACxB;GACF,CAAC;EAKD;CACF;CAGF,MAAM,qBACJ,IACA,KACA,qBACG;EACH,MAAM,SAAS,GACZ,YAAY,UAAU,OAAO,MAAM,EACnC,YAAY,aAAa,aAAa,EACtC,YAAY,SAAS,MAAM,KAAK,EAChC,YAAY,YAAY,MAAM,QAAQ,EACtC,YAAY,iBAAiB,EAAE,EAC/B,YAAY,YAAY,SAAS,CAAC;EAErC,MAAM,yBACJ,qBAAqB,IAAI,uBAAuB,QAAQ,KAAK;EAC/D,MAAM,qBAAqB,IAAI,6BAC7B,QACA,OAAO,KAAK,2BACZ,OAAO,KAAK,4BACZ,mBACA,YACA,iBACF;EAEA,GAAG,QACD,gDAAgD,OAAO,oBACzD;EAEA,MAAM,oBAAoB,IAAI,kBAAkB,sBAAsB;EAEtE,MAAM,oCAAoC,KAAK,IAC7C,OAAO,mBAAmB,GAC1B,CACF;EACA,MAAM,yBAAyB,KAAK,IAAI,OAAO,kBAAkB,CAAC;EAElE,OAAO,IAAI,kBACT,QACA,QACA,OACA,OAAO,QACP,IACA,OACA,IAAI,eACF,QACA,OAAO,KACP,IAAI,YAAY,QAAQ,aAAa,KAAK,GAC1C,OACA,gBAAgB,yBAAyB,EAAE,GAC3C,IACA,yBAEE,oBAAoB,IAChB,oCACA,wBACN,OAAO,oBACP,MACF,GACA,KACA,kBACA,OAAO,IAAI,sBACX,mBACA,oBACA,wBACA,aACF;CACF;CA2BA,MAAM,SAAS,IAAI,OACjB,IACA,QACA,mBA5BqB,cAClB,OACC,IAAI,eACF,GACG,YAAY,aAAa,SAAS,EAClC,YAAY,iBAAiB,EAAE,GAClC,OACA,IACA,YACA,QACA,iBACF,IACF,KAAA,GAGF,eAAe,KAAA,IACX,KAAA,KACC,IAAY,uBACX,IAAI,cACF,QACA,GAAG,YAAY,iBAAiB,EAAE,GAClC,IACA,kBACF,GAQN,QACA,iBACF;CAEA,wBAAwB,IAAI,MAAM;CAElC,SAAc,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC,CAAC;CAE9D,OAAO,eAAe,IAAI,QAAQ,MAAM;AAC1C;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}
+{"version":3,"file":"syncer.js","names":[],"sources":["../../../../../zero-cache/src/server/syncer.ts"],"sourcesContent":["import {randomUUID} from 'node:crypto';\nimport {tmpdir} from 'node:os';\nimport path from 'node:path';\nimport {pid} from 'node:process';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {randInt} from '../../../shared/src/rand.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {DatabaseStorage} from '../../../zqlite/src/database-storage.ts';\nimport type {ValidateLegacyJWT} from '../auth/auth.ts';\nimport {tokenConfigOptions, verifyToken} from '../auth/jwt.ts';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {CustomQueryTransformer} from '../custom-queries/transform-query.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {MutagenService} from '../services/mutagen/mutagen.ts';\nimport {PusherService} from '../services/mutagen/pusher.ts';\nimport type {ReplicaState} from '../services/replicator/replicator.ts';\nimport {\n  type ConnectionContextManager,\n  ConnectionContextManagerImpl,\n} from '../services/view-syncer/connection-context-manager.ts';\nimport {DeferredWritesBudget} from '../services/view-syncer/deferred-writes-budget.ts';\nimport type {DrainCoordinator} from '../services/view-syncer/drain-coordinator.ts';\nimport {PipelineDriver} from '../services/view-syncer/pipeline-driver.ts';\nimport {QueryStats} from '../services/view-syncer/query-stats.ts';\nimport {SnapshotRowCache} from '../services/view-syncer/snapshot-row-cache.ts';\nimport {Snapshotter} from '../services/view-syncer/snapshotter.ts';\nimport {ViewSyncerService} from '../services/view-syncer/view-syncer.ts';\nimport {ProtocolErrorWithLevel} from '../types/error-with-level.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardID} from '../types/shards.ts';\nimport type {Subscription} from '../types/subscription.ts';\nimport {replicaFileModeSchema, replicaFileName} from '../workers/replicator.ts';\nimport {Syncer} from '../workers/syncer.ts';\nimport {startAnonymousTelemetry} from './anonymous-otel-start.ts';\nimport {InspectorDelegate} from './inspector-delegate.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {isPriorityOpRunning, runPriorityOp} from './priority-op.ts';\n\nfunction randomID() {\n  return randInt(1, Number.MAX_SAFE_INTEGER).toString(36);\n}\n\nfunction getCustomQueryConfig(\n  config: Pick<NormalizedZeroConfig, 'query' | 'getQueries'>,\n) {\n  const queryConfig = config.query?.url ? config.query : config.getQueries;\n\n  if (!queryConfig?.url) {\n    return undefined;\n  }\n\n  return {\n    url: queryConfig.url,\n    apiKey: queryConfig.apiKey,\n    allowedClientHeaders: queryConfig.allowedClientHeaders,\n    allowedRequestHeaders: queryConfig.allowedRequestHeaders,\n    forwardCookies: queryConfig.forwardCookies ?? false,\n  };\n}\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  assert(args.length >= 2, `expected [fileMode, workerIndex, ...flags]`);\n  const fileMode = v.parse(args[0], replicaFileModeSchema);\n  const workerIndex = Number(args[1]);\n  installProfileHandler(parent, `syncer-${workerIndex}`, workerIndex);\n  const config = getNormalizedZeroConfig({env, argv: args.slice(2)});\n\n  startOtelAuto(\n    createLogContext(config, 'syncer', workerIndex, false),\n    'syncer',\n    workerIndex,\n  );\n  lc = createLogContext(config, 'syncer', workerIndex);\n  initEventSink(lc, config);\n\n  const {cvr, upstream, enableCrudMutations} = config;\n\n  const replicaFile = replicaFileName(config.replica.file, fileMode);\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: 'syncer replica',\n      dbPath: replicaFile,\n    },\n    config.sqliteCorruptionChecks,\n  );\n  lc.debug?.(`running view-syncer on ${replicaFile}`);\n\n  const cvrDB = await connectPgClient(lc, cvr.db, `sync-worker-${pid}-cvr`, {\n    max: must(cvr.maxConnsPerWorker, 'cvr.maxConnsPerWorker must be set'),\n  });\n\n  const upstreamDB =\n    enableCrudMutations && upstream.type === 'pg'\n      ? await connectPgClient(lc, upstream.db, `sync-worker-${pid}-upstream`, {\n          max: must(\n            upstream.maxConnsPerWorker,\n            'upstream.maxConnsPerWorker must be set',\n          ),\n        })\n      : undefined;\n\n  const dbWarmup = Promise.allSettled([\n    warmupConnections(lc, cvrDB, 'cvr'),\n    upstreamDB ? warmupConnections(lc, upstreamDB, 'upstream') : promiseVoid,\n  ]);\n\n  const tmpDir = config.storageDBTmpDir ?? tmpdir();\n  const operatorStorage = DatabaseStorage.create(\n    lc,\n    path.join(tmpDir, `sync-worker-${randomUUID()}`),\n  );\n  const writeAuthzStorage = DatabaseStorage.create(\n    lc,\n    path.join(tmpDir, `mutagen-${randomUUID()}`),\n  );\n\n  const shard = getShardID(config);\n  const customQueryConfig = getCustomQueryConfig(config);\n  const pushConfig =\n    config.push.url === undefined && config.mutate.url === undefined\n      ? undefined\n      : {\n          ...config.push,\n          ...config.mutate,\n          url: must(\n            config.push.url ?? config.mutate.url,\n            'No push or mutate URL configured',\n          ),\n        };\n\n  /** @deprecated used in JWT validation */\n  let validateLegacyJWT: ValidateLegacyJWT | undefined = undefined;\n\n  const tokenOptions = tokenConfigOptions(config.auth ?? {});\n  if (tokenOptions.length === 1) {\n    validateLegacyJWT = async (token, {userID}) => {\n      if (!userID) {\n        throw new ProtocolErrorWithLevel(\n          {\n            kind: 'Unauthorized',\n            message: 'UserID is required for JWT validation.',\n            origin: 'zeroCache',\n          },\n          'warn',\n        );\n      }\n\n      const decoded = await verifyToken(config.auth, token, {\n        subject: userID,\n        ...(config.auth?.issuer && {issuer: config.auth.issuer}),\n        ...(config.auth?.audience && {\n          audience: config.auth.audience,\n        }),\n      });\n      return {\n        type: 'jwt',\n        raw: token,\n        decoded,\n      };\n    };\n  }\n\n  // Shared by all of the view-syncers on this worker so that the row reads\n  // performed when advancing their pipelines are done once per worker rather\n  // than once per client group.\n  const snapshotRowCache =\n    config.snapshotRowCacheSize > 0\n      ? new SnapshotRowCache(config.snapshotRowCacheSize)\n      : undefined;\n\n  // Shared by all of the view-syncers on this worker, which each hold their\n  // own copy of the changes they are advancing through.\n  const deferredWritesBudget = config.deferIvmWrites\n    ? DeferredWritesBudget.forHeapProportion(\n        config.deferIvmWritesHeapProportion,\n      )\n    : undefined;\n  if (deferredWritesBudget) {\n    lc.info?.(\n      `Deferred IVM writes may hold up to ${deferredWritesBudget.maxRows} ` +\n        `rows (~${(deferredWritesBudget.maxBytes / 1024 ** 2).toFixed(2)} MB) ` +\n        `across client groups`,\n    );\n    getOrCreateGauge(\n      'sync',\n      'ivm.deferred-writes-reserved-rows',\n      'Rows reserved by the client groups of a sync worker to hold IVM ' +\n        'changes in memory (deferIvmWrites)',\n    ).addCallback(o => o.observe(deferredWritesBudget.reservedRows));\n    getOrCreateGauge('sync', 'ivm.deferred-writes-held-bytes', {\n      description:\n        'Estimated bytes of the IVM changes that the client groups of a sync ' +\n        'worker hold in memory (deferIvmWrites)',\n      unit: 'By',\n    }).addCallback(o => o.observe(deferredWritesBudget.heldBytes));\n  }\n\n  // Shared by all of the view-syncers on this worker, so that the work done\n  // for each query shape is logged once per worker per interval.\n  const queryStats =\n    config.log.queryStatsIntervalSeconds > 0 ? new QueryStats() : undefined;\n  const stopQueryStats = queryStats?.start(\n    lc\n      .withContext('taskID', config.taskID)\n      .withContext('component', 'view-syncer'),\n    config.log.queryStatsIntervalSeconds * 1000,\n  );\n\n  const viewSyncerFactory = (\n    id: string,\n    sub: Subscription<ReplicaState>,\n    drainCoordinator: DrainCoordinator,\n  ) => {\n    const logger = lc\n      .withContext('taskID', config.taskID)\n      .withContext('component', 'view-syncer')\n      .withContext('appID', shard.appID)\n      .withContext('shardNum', shard.shardNum)\n      .withContext('clientGroupID', id)\n      .withContext('instance', randomID());\n\n    const customQueryTransformer =\n      customQueryConfig && new CustomQueryTransformer(logger, shard);\n    const connContextManager = new ConnectionContextManagerImpl(\n      logger,\n      config.auth.revalidateIntervalSeconds,\n      config.auth.retransformIntervalSeconds,\n      customQueryConfig,\n      pushConfig,\n      validateLegacyJWT,\n    );\n\n    lc.debug?.(\n      `creating view syncer. Query Planner Enabled: ${config.enableQueryPlanner}`,\n    );\n\n    const inspectorDelegate = new InspectorDelegate(customQueryTransformer);\n\n    const priorityOpRunningYieldThresholdMs = Math.max(\n      config.yieldThresholdMs / 4,\n      2,\n    );\n    const normalYieldThresholdMs = Math.max(config.yieldThresholdMs, 2);\n\n    return new ViewSyncerService(\n      config,\n      logger,\n      shard,\n      config.taskID,\n      id,\n      cvrDB,\n      new PipelineDriver(\n        logger,\n        config.log,\n        new Snapshotter(\n          logger,\n          replicaFile,\n          shard,\n          undefined,\n          snapshotRowCache,\n        ),\n        shard,\n        operatorStorage.createClientGroupStorage(id),\n        id,\n        inspectorDelegate,\n        () =>\n          isPriorityOpRunning()\n            ? priorityOpRunningYieldThresholdMs\n            : normalYieldThresholdMs,\n        config.enableQueryPlanner,\n        config,\n        deferredWritesBudget,\n        queryStats,\n      ),\n      sub,\n      drainCoordinator,\n      config.log.slowHydrateThreshold,\n      inspectorDelegate,\n      connContextManager,\n      customQueryTransformer,\n      runPriorityOp,\n    );\n  };\n\n  const mutagenFactory = upstreamDB\n    ? (id: string) =>\n        new MutagenService(\n          lc\n            .withContext('component', 'mutagen')\n            .withContext('clientGroupID', id),\n          shard,\n          id,\n          upstreamDB,\n          config,\n          writeAuthzStorage,\n        )\n    : undefined;\n\n  const pusherFactory =\n    pushConfig === undefined\n      ? undefined\n      : (id: string, connContextManager: ConnectionContextManager) =>\n          new PusherService(\n            config,\n            lc.withContext('clientGroupID', id),\n            id,\n            connContextManager,\n          );\n\n  const syncer = new Syncer(\n    lc,\n    config,\n    viewSyncerFactory,\n    mutagenFactory,\n    pusherFactory,\n    parent,\n    validateLegacyJWT,\n  );\n\n  startAnonymousTelemetry(lc, config);\n\n  void dbWarmup.then(() => parent.send(['ready', {ready: true}]));\n\n  try {\n    return await runUntilKilled(lc, parent, syncer);\n  } finally {\n    // Logs the work done since the last interval.\n    stopQueryStats?.();\n  }\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqDA,SAAS,WAAW;CAClB,OAAO,QAAQ,GAAG,OAAO,gBAAgB,EAAE,SAAS,EAAE;AACxD;AAEA,SAAS,qBACP,QACA;CACA,MAAM,cAAc,OAAO,OAAO,MAAM,OAAO,QAAQ,OAAO;CAE9D,IAAI,CAAC,aAAa,KAChB;CAGF,OAAO;EACL,KAAK,YAAY;EACjB,QAAQ,YAAY;EACpB,sBAAsB,YAAY;EAClC,uBAAuB,YAAY;EACnC,gBAAgB,YAAY,kBAAkB;CAChD;AACF;AAGA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,OAAO,KAAK,UAAU,GAAG,4CAA4C;CACrE,MAAM,WAAW,MAAQ,KAAK,IAAI,qBAAqB;CACvD,MAAM,cAAc,OAAO,KAAK,EAAE;CAClC,sBAAsB,QAAQ,UAAU,eAAe,WAAW;CAClE,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CAEjE,cACE,iBAAiB,QAAQ,UAAU,aAAa,KAAK,GACrD,UACA,WACF;CACA,KAAK,iBAAiB,QAAQ,UAAU,WAAW;CACnD,cAAc,IAAI,MAAM;CAExB,MAAM,EAAC,KAAK,UAAU,wBAAuB;CAE7C,MAAM,cAAc,gBAAgB,OAAO,QAAQ,MAAM,QAAQ;CACjE,yCACE;EACE,WAAW;EACX,QAAQ;CACV,GACA,OAAO,sBACT;CACA,GAAG,QAAQ,0BAA0B,aAAa;CAElD,MAAM,QAAQ,MAAM,gBAAgB,IAAI,IAAI,IAAI,eAAe,IAAI,OAAO,EACxE,KAAK,KAAK,IAAI,mBAAmB,mCAAmC,EACtE,CAAC;CAED,MAAM,aACJ,uBAAuB,SAAS,SAAS,OACrC,MAAM,gBAAgB,IAAI,SAAS,IAAI,eAAe,IAAI,YAAY,EACpE,KAAK,KACH,SAAS,mBACT,wCACF,EACF,CAAC,IACD,KAAA;CAEN,MAAM,WAAW,QAAQ,WAAW,CAClC,kBAAkB,IAAI,OAAO,KAAK,GAClC,aAAa,kBAAkB,IAAI,YAAY,UAAU,IAAI,WAC/D,CAAC;CAED,MAAM,SAAS,OAAO,mBAAmB,OAAO;CAChD,MAAM,kBAAkB,gBAAgB,OACtC,IACA,KAAK,KAAK,QAAQ,eAAe,WAAW,GAAG,CACjD;CACA,MAAM,oBAAoB,gBAAgB,OACxC,IACA,KAAK,KAAK,QAAQ,WAAW,WAAW,GAAG,CAC7C;CAEA,MAAM,QAAQ,WAAW,MAAM;CAC/B,MAAM,oBAAoB,qBAAqB,MAAM;CACrD,MAAM,aACJ,OAAO,KAAK,QAAQ,KAAA,KAAa,OAAO,OAAO,QAAQ,KAAA,IACnD,KAAA,IACA;EACE,GAAG,OAAO;EACV,GAAG,OAAO;EACV,KAAK,KACH,OAAO,KAAK,OAAO,OAAO,OAAO,KACjC,kCACF;CACF;;CAGN,IAAI,oBAAmD,KAAA;CAGvD,IADqB,mBAAmB,OAAO,QAAQ,CAAC,CACpD,EAAa,WAAW,GAC1B,oBAAoB,OAAO,OAAO,EAAC,aAAY;EAC7C,IAAI,CAAC,QACH,MAAM,IAAI,uBACR;GACE,MAAM;GACN,SAAS;GACT,QAAQ;EACV,GACA,MACF;EAUF,OAAO;GACL,MAAM;GACN,KAAK;GACL,SAAA,MAVoB,YAAY,OAAO,MAAM,OAAO;IACpD,SAAS;IACT,GAAI,OAAO,MAAM,UAAU,EAAC,QAAQ,OAAO,KAAK,OAAM;IACtD,GAAI,OAAO,MAAM,YAAY,EAC3B,UAAU,OAAO,KAAK,SACxB;GACF,CAAC;EAKD;CACF;CAMF,MAAM,mBACJ,OAAO,uBAAuB,IAC1B,IAAI,iBAAiB,OAAO,oBAAoB,IAChD,KAAA;CAIN,MAAM,uBAAuB,OAAO,iBAChC,qBAAqB,kBACnB,OAAO,4BACT,IACA,KAAA;CACJ,IAAI,sBAAsB;EACxB,GAAG,OACD,sCAAsC,qBAAqB,QAAQ,WACtD,qBAAqB,WAAW,QAAQ,GAAG,QAAQ,CAAC,EAAE,0BAErE;EACA,iBACE,QACA,qCACA,oGAEF,EAAE,aAAY,MAAK,EAAE,QAAQ,qBAAqB,YAAY,CAAC;EAC/D,iBAAiB,QAAQ,kCAAkC;GACzD,aACE;GAEF,MAAM;EACR,CAAC,EAAE,aAAY,MAAK,EAAE,QAAQ,qBAAqB,SAAS,CAAC;CAC/D;CAIA,MAAM,aACJ,OAAO,IAAI,4BAA4B,IAAI,IAAI,WAAW,IAAI,KAAA;CAChE,MAAM,iBAAiB,YAAY,MACjC,GACG,YAAY,UAAU,OAAO,MAAM,EACnC,YAAY,aAAa,aAAa,GACzC,OAAO,IAAI,4BAA4B,GACzC;CAEA,MAAM,qBACJ,IACA,KACA,qBACG;EACH,MAAM,SAAS,GACZ,YAAY,UAAU,OAAO,MAAM,EACnC,YAAY,aAAa,aAAa,EACtC,YAAY,SAAS,MAAM,KAAK,EAChC,YAAY,YAAY,MAAM,QAAQ,EACtC,YAAY,iBAAiB,EAAE,EAC/B,YAAY,YAAY,SAAS,CAAC;EAErC,MAAM,yBACJ,qBAAqB,IAAI,uBAAuB,QAAQ,KAAK;EAC/D,MAAM,qBAAqB,IAAI,6BAC7B,QACA,OAAO,KAAK,2BACZ,OAAO,KAAK,4BACZ,mBACA,YACA,iBACF;EAEA,GAAG,QACD,gDAAgD,OAAO,oBACzD;EAEA,MAAM,oBAAoB,IAAI,kBAAkB,sBAAsB;EAEtE,MAAM,oCAAoC,KAAK,IAC7C,OAAO,mBAAmB,GAC1B,CACF;EACA,MAAM,yBAAyB,KAAK,IAAI,OAAO,kBAAkB,CAAC;EAElE,OAAO,IAAI,kBACT,QACA,QACA,OACA,OAAO,QACP,IACA,OACA,IAAI,eACF,QACA,OAAO,KACP,IAAI,YACF,QACA,aACA,OACA,KAAA,GACA,gBACF,GACA,OACA,gBAAgB,yBAAyB,EAAE,GAC3C,IACA,yBAEE,oBAAoB,IAChB,oCACA,wBACN,OAAO,oBACP,QACA,sBACA,UACF,GACA,KACA,kBACA,OAAO,IAAI,sBACX,mBACA,oBACA,wBACA,aACF;CACF;CA2BA,MAAM,SAAS,IAAI,OACjB,IACA,QACA,mBA5BqB,cAClB,OACC,IAAI,eACF,GACG,YAAY,aAAa,SAAS,EAClC,YAAY,iBAAiB,EAAE,GAClC,OACA,IACA,YACA,QACA,iBACF,IACF,KAAA,GAGF,eAAe,KAAA,IACX,KAAA,KACC,IAAY,uBACX,IAAI,cACF,QACA,GAAG,YAAY,iBAAiB,EAAE,GAClC,IACA,kBACF,GAQN,QACA,iBACF;CAEA,wBAAwB,IAAI,MAAM;CAElC,SAAc,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC,CAAC;CAE9D,IAAI;EACF,OAAO,MAAM,eAAe,IAAI,QAAQ,MAAM;CAChD,UAAU;EAER,iBAAiB;CACnB;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}

package/out/zero-cache/src/server/worker-dispatcher.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"worker-dispatcher.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/worker-dispatcher.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAMjD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,wBAAwB,CAAC;AAEpD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,uBAAuB,CAAC;AAIlD,qBAAa,gBAAiB,YAAW,OAAO;;IAC9C,QAAQ,CAAC,EAAE,uBAAuB;IAKlC,YACE,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,EAAE,EACjB,OAAO,EAAE,MAAM,GAAG,SAAS,EAC3B,cAAc,EAAE,MAAM,GAAG,SAAS,EA4GnC;IAED,GAAG,kBAQF;IAED,IAAI,kBAGH;CACF;AAID,wBAAgB,SAAS,CAAC,GAAG,EAAE,GAAG,GAC9B;IACE,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,GAAG,QAAQ,GAAG,aAAa,CAAC;IAC1C,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAChB,GACD,SAAS,CAGZ"}
+{"version":3,"file":"worker-dispatcher.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/worker-dispatcher.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,wBAAwB,CAAC;AAEpD,OAAO,KAAK,EAA2B,MAAM,EAAC,MAAM,uBAAuB,CAAC;AAK5E,qBAAa,gBAAiB,YAAW,OAAO;;IAC9C,QAAQ,CAAC,EAAE,uBAAuB;IAKlC,YACE,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,EAAE,EACjB,OAAO,EAAE,MAAM,GAAG,SAAS,EAC3B,cAAc,EAAE,MAAM,GAAG,SAAS,EAsInC;IAED,GAAG,kBAQF;IAED,IAAI,kBAIH;CACF;AAID,wBAAgB,SAAS,CAAC,GAAG,EAAE,GAAG,GAC9B;IACE,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,GAAG,QAAQ,GAAG,aAAa,CAAC;IAC1C,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAChB,GACD,SAAS,CAGZ"}

package/out/zero-cache/src/server/worker-dispatcher.js

@@ -1,17 +1,29 @@
 import { assert } from "../../../shared/src/asserts.js";
-import { h32 } from "../../../shared/src/hash.js";
-import { getOrCreateGauge } from "../observability/metrics.js";
+import { getOrCreateCounter, getOrCreateGauge } from "../observability/metrics.js";
 import { RunningState } from "../services/running-state.js";
 import { installWebSocketHandoff } from "../types/websocket-handoff.js";
 import { getConnectParams } from "../workers/connect-params.js";
+import { SyncerAssigner } from "./syncer-assigner.js";
 import UrlPattern from "url-pattern";
 //#region ../zero-cache/src/server/worker-dispatcher.ts
 var WorkerDispatcher = class {
 	id = "worker-dispatcher";
 	#lc;
 	#state = new RunningState(this.id);
+	#assigner;
 	constructor(lc, taskID, parent, syncers, mutator, changeStreamer) {
 		this.#lc = lc;
+		const assigner = new SyncerAssigner(taskID, syncers.length);
+		this.#assigner = assigner;
+		const workerDispatchesCounter = getOrCreateCounter("sync", "worker-dispatches", "Dispatched client group connections to worker");
+		getOrCreateGauge("sync", "worker-client-groups", "Number of active client groups assigned to worker").addCallback((result) => {
+			for (let i = 0; i < syncers.length; i++) result.observe(assigner.getWorkerLoad(i), { worker: String(i) });
+		});
+		syncers.forEach((syncer, index) => {
+			syncer.onMessageType("clientGroupStatus", ({ clientGroupID, active, generation }) => {
+				if (!active) assigner.release(clientGroupID, index, generation);
+			});
+		});
 		function connectParams(req) {
 			const { headers, url: u } = req;
 			const url = new URL(u ?? "", "http://unused/");
@@ -39,10 +51,14 @@ var WorkerDispatcher = class {
 			const params = connectParams(req);
 			const { clientGroupID, protocolVersion } = params;
 			maxProtocolVersion = Math.max(maxProtocolVersion, protocolVersion);
-			const syncer = h32(taskID + "/" + clientGroupID) % syncers.length;
+			const { worker: syncer, generation } = assigner.assign(clientGroupID);
+			workerDispatchesCounter.add(1, { worker: String(syncer) });
 			lc.debug?.(`connecting ${clientGroupID} to syncer ${syncer}`);
 			return {
-				payload: params,
+				payload: {
+					...params,
+					generation
+				},
 				sender: syncers[syncer]
 			};
 		};
@@ -77,6 +93,7 @@ var WorkerDispatcher = class {
 		return this.#state.stopped();
 	}
 	stop() {
+		this.#assigner.destroy();
 		this.#state.stop(this.#lc);
 		return this.#state.stopped();
 	}

package/out/zero-cache/src/server/worker-dispatcher.js.map

@@ -1 +1 @@
-{"version":3,"file":"worker-dispatcher.js","names":["#lc","#state"],"sources":["../../../../../zero-cache/src/server/worker-dispatcher.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport UrlPattern from 'url-pattern';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {h32} from '../../../shared/src/hash.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {RunningState} from '../services/running-state.ts';\nimport type {Service} from '../services/service.ts';\nimport type {IncomingMessageSubset} from '../types/http.ts';\nimport type {Worker} from '../types/processes.ts';\nimport {installWebSocketHandoff} from '../types/websocket-handoff.ts';\nimport {getConnectParams} from '../workers/connect-params.ts';\n\nexport class WorkerDispatcher implements Service {\n  readonly id = 'worker-dispatcher';\n  readonly #lc: LogContext;\n\n  readonly #state = new RunningState(this.id);\n\n  constructor(\n    lc: LogContext,\n    taskID: string,\n    parent: Worker,\n    syncers: Worker[],\n    mutator: Worker | undefined,\n    changeStreamer: Worker | undefined,\n  ) {\n    this.#lc = lc;\n\n    function connectParams(req: IncomingMessageSubset) {\n      const {headers, url: u} = req;\n      const url = new URL(u ?? '', 'http://unused/');\n      const path = parsePath(url);\n      if (!path) {\n        throw new Error(`Invalid URL: ${u}`);\n      }\n      const version = Number(path.version);\n      if (Number.isNaN(version)) {\n        throw new Error(`Invalid version: ${u}`);\n      }\n      const {params, error} = getConnectParams(version, url, headers);\n      if (error !== null) {\n        throw new Error(error);\n      }\n      return params;\n    }\n\n    const handlePush = (req: IncomingMessageSubset) => {\n      assert(\n        mutator !== undefined,\n        'Received a push for a custom mutation but no `push.url` was configured.',\n      );\n      return {payload: connectParams(req), sender: mutator};\n    };\n\n    let maxProtocolVersion = 0;\n    getOrCreateGauge(\n      'sync',\n      'max-protocol-version',\n      'Latest sync protocol version from a connecting client',\n    ).addCallback(result => {\n      if (maxProtocolVersion) {\n        result.observe(maxProtocolVersion);\n      }\n    });\n\n    const handleSync = (req: IncomingMessageSubset) => {\n      assert(syncers.length, 'Received a sync request with no sync workers.');\n      const params = connectParams(req);\n      const {clientGroupID, protocolVersion} = params;\n      maxProtocolVersion = Math.max(maxProtocolVersion, protocolVersion);\n\n      // Include the TaskID when hash-bucketting the client group to the sync\n      // worker. This diversifies the distribution of client groups (across\n      // workers) for different tasks, so that if one task sheds connections\n      // from its most heavily loaded sync worker(s), those client groups will\n      // be distributed uniformly across workers on the receiving task(s).\n      const syncer = h32(taskID + '/' + clientGroupID) % syncers.length;\n\n      lc.debug?.(`connecting ${clientGroupID} to syncer ${syncer}`);\n      return {payload: params, sender: syncers[syncer]};\n    };\n\n    const handleChangeStream = (req: IncomingMessageSubset) => {\n      // Note: The change-streamer is generally not dispatched via the main\n      //       port, and in particular, should *not* be accessible via that\n      //       port in single-node mode. However, this plumbing is maintained\n      //       for the purpose of allowing --lazy-startup of the\n      //       replication-manager as a possible future feature.\n      assert(\n        syncers.length === 0 && mutator === undefined,\n        'Dispatch to the change-streamer via the main port ' +\n          'is only allowed in multi-node mode',\n      );\n      assert(\n        changeStreamer,\n        'Received a change-streamer request without a change-streamer worker',\n      );\n      const url = new URL(req.url ?? '', 'http://unused/');\n      const path = parsePath(url);\n      if (!path) {\n        throw new Error(`Invalid URL: ${req.url}`);\n      }\n\n      return {\n        payload: path.action,\n        sender: changeStreamer,\n      };\n    };\n\n    // handoff messages from this ZeroDispatcher to the appropriate worker (pool).\n    installWebSocketHandoff<unknown>(\n      lc,\n      request => {\n        const {url: u} = request;\n        const url = new URL(u ?? '', 'http://unused/');\n        const path = parsePath(url);\n        if (!path) {\n          throw new Error(`Invalid URL: ${u}`);\n        }\n        switch (path.worker) {\n          case 'sync':\n            return handleSync(request);\n          case 'replication':\n            return handleChangeStream(request);\n          case 'mutate':\n            return handlePush(request);\n          default:\n            throw new Error(`Invalid URL: ${u}`);\n        }\n      },\n      parent,\n    );\n  }\n\n  run() {\n    const readyStart = Date.now();\n    getOrCreateGauge('server', 'uptime', {\n      description: 'Cumulative uptime, starting from when requests are served',\n      unit: 's',\n    }).addCallback(result => result.observe((Date.now() - readyStart) / 1000));\n\n    return this.#state.stopped();\n  }\n\n  stop() {\n    this.#state.stop(this.#lc);\n    return this.#state.stopped();\n  }\n}\n\nconst URL_PATTERN = new UrlPattern('(/:base)/:worker/v:version/:action');\n\nexport function parsePath(url: URL):\n  | {\n      base?: string;\n      worker: 'sync' | 'mutate' | 'replication';\n      version: string;\n      action: string;\n    }\n  | undefined {\n  // The match() returns both null and undefined.\n  return URL_PATTERN.match(url.pathname) || undefined;\n} // The server allows the client to use any /:base/ path to facilitate\n// servicing requests on the same domain as the application.\n"],"mappings":";;;;;;;;AAYA,IAAa,mBAAb,MAAiD;CAC/C,KAAc;CACd;CAEA,SAAkB,IAAI,aAAa,KAAK,EAAE;CAE1C,YACE,IACA,QACA,QACA,SACA,SACA,gBACA;EACA,KAAKA,MAAM;EAEX,SAAS,cAAc,KAA4B;GACjD,MAAM,EAAC,SAAS,KAAK,MAAK;GAC1B,MAAM,MAAM,IAAI,IAAI,KAAK,IAAI,gBAAgB;GAC7C,MAAM,OAAO,UAAU,GAAG;GAC1B,IAAI,CAAC,MACH,MAAM,IAAI,MAAM,gBAAgB,GAAG;GAErC,MAAM,UAAU,OAAO,KAAK,OAAO;GACnC,IAAI,OAAO,MAAM,OAAO,GACtB,MAAM,IAAI,MAAM,oBAAoB,GAAG;GAEzC,MAAM,EAAC,QAAQ,UAAS,iBAAiB,SAAS,KAAK,OAAO;GAC9D,IAAI,UAAU,MACZ,MAAM,IAAI,MAAM,KAAK;GAEvB,OAAO;EACT;EAEA,MAAM,cAAc,QAA+B;GACjD,OACE,YAAY,KAAA,GACZ,yEACF;GACA,OAAO;IAAC,SAAS,cAAc,GAAG;IAAG,QAAQ;GAAO;EACtD;EAEA,IAAI,qBAAqB;EACzB,iBACE,QACA,wBACA,uDACF,EAAE,aAAY,WAAU;GACtB,IAAI,oBACF,OAAO,QAAQ,kBAAkB;EAErC,CAAC;EAED,MAAM,cAAc,QAA+B;GACjD,OAAO,QAAQ,QAAQ,+CAA+C;GACtE,MAAM,SAAS,cAAc,GAAG;GAChC,MAAM,EAAC,eAAe,oBAAmB;GACzC,qBAAqB,KAAK,IAAI,oBAAoB,eAAe;GAOjE,MAAM,SAAS,IAAI,SAAS,MAAM,aAAa,IAAI,QAAQ;GAE3D,GAAG,QAAQ,cAAc,cAAc,aAAa,QAAQ;GAC5D,OAAO;IAAC,SAAS;IAAQ,QAAQ,QAAQ;GAAO;EAClD;EAEA,MAAM,sBAAsB,QAA+B;GAMzD,OACE,QAAQ,WAAW,KAAK,YAAY,KAAA,GACpC,sFAEF;GACA,OACE,gBACA,qEACF;GAEA,MAAM,OAAO,UAAU,IADP,IAAI,IAAI,OAAO,IAAI,gBACZ,CAAG;GAC1B,IAAI,CAAC,MACH,MAAM,IAAI,MAAM,gBAAgB,IAAI,KAAK;GAG3C,OAAO;IACL,SAAS,KAAK;IACd,QAAQ;GACV;EACF;EAGA,wBACE,KACA,YAAW;GACT,MAAM,EAAC,KAAK,MAAK;GAEjB,MAAM,OAAO,UAAU,IADP,IAAI,KAAK,IAAI,gBACN,CAAG;GAC1B,IAAI,CAAC,MACH,MAAM,IAAI,MAAM,gBAAgB,GAAG;GAErC,QAAQ,KAAK,QAAb;IACE,KAAK,QACH,OAAO,WAAW,OAAO;IAC3B,KAAK,eACH,OAAO,mBAAmB,OAAO;IACnC,KAAK,UACH,OAAO,WAAW,OAAO;IAC3B,SACE,MAAM,IAAI,MAAM,gBAAgB,GAAG;GACvC;EACF,GACA,MACF;CACF;CAEA,MAAM;EACJ,MAAM,aAAa,KAAK,IAAI;EAC5B,iBAAiB,UAAU,UAAU;GACnC,aAAa;GACb,MAAM;EACR,CAAC,EAAE,aAAY,WAAU,OAAO,SAAS,KAAK,IAAI,IAAI,cAAc,GAAI,CAAC;EAEzE,OAAO,KAAKC,OAAO,QAAQ;CAC7B;CAEA,OAAO;EACL,KAAKA,OAAO,KAAK,KAAKD,GAAG;EACzB,OAAO,KAAKC,OAAO,QAAQ;CAC7B;AACF;AAEA,IAAM,cAAc,IAAI,WAAW,oCAAoC;AAEvE,SAAgB,UAAU,KAOZ;CAEZ,OAAO,YAAY,MAAM,IAAI,QAAQ,KAAK,KAAA;AAC5C"}
+{"version":3,"file":"worker-dispatcher.js","names":["#lc","#state","#assigner"],"sources":["../../../../../zero-cache/src/server/worker-dispatcher.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport UrlPattern from 'url-pattern';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateGauge,\n} from '../observability/metrics.ts';\nimport {RunningState} from '../services/running-state.ts';\nimport type {Service} from '../services/service.ts';\nimport type {IncomingMessageSubset} from '../types/http.ts';\nimport type {ClientGroupStatusMessage, Worker} from '../types/processes.ts';\nimport {installWebSocketHandoff} from '../types/websocket-handoff.ts';\nimport {getConnectParams} from '../workers/connect-params.ts';\nimport {SyncerAssigner} from './syncer-assigner.ts';\n\nexport class WorkerDispatcher implements Service {\n  readonly id = 'worker-dispatcher';\n  readonly #lc: LogContext;\n  readonly #state = new RunningState(this.id);\n  readonly #assigner: SyncerAssigner;\n\n  constructor(\n    lc: LogContext,\n    taskID: string,\n    parent: Worker,\n    syncers: Worker[],\n    mutator: Worker | undefined,\n    changeStreamer: Worker | undefined,\n  ) {\n    this.#lc = lc;\n\n    const assigner = new SyncerAssigner(taskID, syncers.length);\n    this.#assigner = assigner;\n\n    const workerDispatchesCounter = getOrCreateCounter(\n      'sync',\n      'worker-dispatches',\n      'Dispatched client group connections to worker',\n    );\n\n    getOrCreateGauge(\n      'sync',\n      'worker-client-groups',\n      'Number of active client groups assigned to worker',\n    ).addCallback(result => {\n      for (let i = 0; i < syncers.length; i++) {\n        result.observe(assigner.getWorkerLoad(i), {worker: String(i)});\n      }\n    });\n\n    syncers.forEach((syncer, index) => {\n      syncer.onMessageType<ClientGroupStatusMessage>(\n        'clientGroupStatus',\n        ({clientGroupID, active, generation}) => {\n          if (!active) {\n            assigner.release(clientGroupID, index, generation);\n          }\n        },\n      );\n    });\n\n    function connectParams(req: IncomingMessageSubset) {\n      const {headers, url: u} = req;\n      const url = new URL(u ?? '', 'http://unused/');\n      const path = parsePath(url);\n      if (!path) {\n        throw new Error(`Invalid URL: ${u}`);\n      }\n      const version = Number(path.version);\n      if (Number.isNaN(version)) {\n        throw new Error(`Invalid version: ${u}`);\n      }\n      const {params, error} = getConnectParams(version, url, headers);\n      if (error !== null) {\n        throw new Error(error);\n      }\n      return params;\n    }\n\n    const handlePush = (req: IncomingMessageSubset) => {\n      assert(\n        mutator !== undefined,\n        'Received a push for a custom mutation but no `push.url` was configured.',\n      );\n      return {payload: connectParams(req), sender: mutator};\n    };\n\n    let maxProtocolVersion = 0;\n    getOrCreateGauge(\n      'sync',\n      'max-protocol-version',\n      'Latest sync protocol version from a connecting client',\n    ).addCallback(result => {\n      if (maxProtocolVersion) {\n        result.observe(maxProtocolVersion);\n      }\n    });\n\n    const handleSync = (req: IncomingMessageSubset) => {\n      assert(syncers.length, 'Received a sync request with no sync workers.');\n      const params = connectParams(req);\n      const {clientGroupID, protocolVersion} = params;\n      maxProtocolVersion = Math.max(maxProtocolVersion, protocolVersion);\n\n      const {worker: syncer, generation} = assigner.assign(clientGroupID);\n      workerDispatchesCounter.add(1, {worker: String(syncer)});\n\n      lc.debug?.(`connecting ${clientGroupID} to syncer ${syncer}`);\n      return {payload: {...params, generation}, sender: syncers[syncer]};\n    };\n\n    const handleChangeStream = (req: IncomingMessageSubset) => {\n      // Note: The change-streamer is generally not dispatched via the main\n      //       port, and in particular, should *not* be accessible via that\n      //       port in single-node mode. However, this plumbing is maintained\n      //       for the purpose of allowing --lazy-startup of the\n      //       replication-manager as a possible future feature.\n      assert(\n        syncers.length === 0 && mutator === undefined,\n        'Dispatch to the change-streamer via the main port ' +\n          'is only allowed in multi-node mode',\n      );\n      assert(\n        changeStreamer,\n        'Received a change-streamer request without a change-streamer worker',\n      );\n      const url = new URL(req.url ?? '', 'http://unused/');\n      const path = parsePath(url);\n      if (!path) {\n        throw new Error(`Invalid URL: ${req.url}`);\n      }\n\n      return {\n        payload: path.action,\n        sender: changeStreamer,\n      };\n    };\n\n    // handoff messages from this ZeroDispatcher to the appropriate worker (pool).\n    installWebSocketHandoff<unknown>(\n      lc,\n      request => {\n        const {url: u} = request;\n        const url = new URL(u ?? '', 'http://unused/');\n        const path = parsePath(url);\n        if (!path) {\n          throw new Error(`Invalid URL: ${u}`);\n        }\n        switch (path.worker) {\n          case 'sync':\n            return handleSync(request);\n          case 'replication':\n            return handleChangeStream(request);\n          case 'mutate':\n            return handlePush(request);\n          default:\n            throw new Error(`Invalid URL: ${u}`);\n        }\n      },\n      parent,\n    );\n  }\n\n  run() {\n    const readyStart = Date.now();\n    getOrCreateGauge('server', 'uptime', {\n      description: 'Cumulative uptime, starting from when requests are served',\n      unit: 's',\n    }).addCallback(result => result.observe((Date.now() - readyStart) / 1000));\n\n    return this.#state.stopped();\n  }\n\n  stop() {\n    this.#assigner.destroy();\n    this.#state.stop(this.#lc);\n    return this.#state.stopped();\n  }\n}\n\nconst URL_PATTERN = new UrlPattern('(/:base)/:worker/v:version/:action');\n\nexport function parsePath(url: URL):\n  | {\n      base?: string;\n      worker: 'sync' | 'mutate' | 'replication';\n      version: string;\n      action: string;\n    }\n  | undefined {\n  // The match() returns both null and undefined.\n  return URL_PATTERN.match(url.pathname) || undefined;\n} // The server allows the client to use any /:base/ path to facilitate\n// servicing requests on the same domain as the application.\n"],"mappings":";;;;;;;;AAeA,IAAa,mBAAb,MAAiD;CAC/C,KAAc;CACd;CACA,SAAkB,IAAI,aAAa,KAAK,EAAE;CAC1C;CAEA,YACE,IACA,QACA,QACA,SACA,SACA,gBACA;EACA,KAAKA,MAAM;EAEX,MAAM,WAAW,IAAI,eAAe,QAAQ,QAAQ,MAAM;EAC1D,KAAKE,YAAY;EAEjB,MAAM,0BAA0B,mBAC9B,QACA,qBACA,+CACF;EAEA,iBACE,QACA,wBACA,mDACF,EAAE,aAAY,WAAU;GACtB,KAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAClC,OAAO,QAAQ,SAAS,cAAc,CAAC,GAAG,EAAC,QAAQ,OAAO,CAAC,EAAC,CAAC;EAEjE,CAAC;EAED,QAAQ,SAAS,QAAQ,UAAU;GACjC,OAAO,cACL,sBACC,EAAC,eAAe,QAAQ,iBAAgB;IACvC,IAAI,CAAC,QACH,SAAS,QAAQ,eAAe,OAAO,UAAU;GAErD,CACF;EACF,CAAC;EAED,SAAS,cAAc,KAA4B;GACjD,MAAM,EAAC,SAAS,KAAK,MAAK;GAC1B,MAAM,MAAM,IAAI,IAAI,KAAK,IAAI,gBAAgB;GAC7C,MAAM,OAAO,UAAU,GAAG;GAC1B,IAAI,CAAC,MACH,MAAM,IAAI,MAAM,gBAAgB,GAAG;GAErC,MAAM,UAAU,OAAO,KAAK,OAAO;GACnC,IAAI,OAAO,MAAM,OAAO,GACtB,MAAM,IAAI,MAAM,oBAAoB,GAAG;GAEzC,MAAM,EAAC,QAAQ,UAAS,iBAAiB,SAAS,KAAK,OAAO;GAC9D,IAAI,UAAU,MACZ,MAAM,IAAI,MAAM,KAAK;GAEvB,OAAO;EACT;EAEA,MAAM,cAAc,QAA+B;GACjD,OACE,YAAY,KAAA,GACZ,yEACF;GACA,OAAO;IAAC,SAAS,cAAc,GAAG;IAAG,QAAQ;GAAO;EACtD;EAEA,IAAI,qBAAqB;EACzB,iBACE,QACA,wBACA,uDACF,EAAE,aAAY,WAAU;GACtB,IAAI,oBACF,OAAO,QAAQ,kBAAkB;EAErC,CAAC;EAED,MAAM,cAAc,QAA+B;GACjD,OAAO,QAAQ,QAAQ,+CAA+C;GACtE,MAAM,SAAS,cAAc,GAAG;GAChC,MAAM,EAAC,eAAe,oBAAmB;GACzC,qBAAqB,KAAK,IAAI,oBAAoB,eAAe;GAEjE,MAAM,EAAC,QAAQ,QAAQ,eAAc,SAAS,OAAO,aAAa;GAClE,wBAAwB,IAAI,GAAG,EAAC,QAAQ,OAAO,MAAM,EAAC,CAAC;GAEvD,GAAG,QAAQ,cAAc,cAAc,aAAa,QAAQ;GAC5D,OAAO;IAAC,SAAS;KAAC,GAAG;KAAQ;IAAU;IAAG,QAAQ,QAAQ;GAAO;EACnE;EAEA,MAAM,sBAAsB,QAA+B;GAMzD,OACE,QAAQ,WAAW,KAAK,YAAY,KAAA,GACpC,sFAEF;GACA,OACE,gBACA,qEACF;GAEA,MAAM,OAAO,UAAU,IADP,IAAI,IAAI,OAAO,IAAI,gBACZ,CAAG;GAC1B,IAAI,CAAC,MACH,MAAM,IAAI,MAAM,gBAAgB,IAAI,KAAK;GAG3C,OAAO;IACL,SAAS,KAAK;IACd,QAAQ;GACV;EACF;EAGA,wBACE,KACA,YAAW;GACT,MAAM,EAAC,KAAK,MAAK;GAEjB,MAAM,OAAO,UAAU,IADP,IAAI,KAAK,IAAI,gBACN,CAAG;GAC1B,IAAI,CAAC,MACH,MAAM,IAAI,MAAM,gBAAgB,GAAG;GAErC,QAAQ,KAAK,QAAb;IACE,KAAK,QACH,OAAO,WAAW,OAAO;IAC3B,KAAK,eACH,OAAO,mBAAmB,OAAO;IACnC,KAAK,UACH,OAAO,WAAW,OAAO;IAC3B,SACE,MAAM,IAAI,MAAM,gBAAgB,GAAG;GACvC;EACF,GACA,MACF;CACF;CAEA,MAAM;EACJ,MAAM,aAAa,KAAK,IAAI;EAC5B,iBAAiB,UAAU,UAAU;GACnC,aAAa;GACb,MAAM;EACR,CAAC,EAAE,aAAY,WAAU,OAAO,SAAS,KAAK,IAAI,IAAI,cAAc,GAAI,CAAC;EAEzE,OAAO,KAAKD,OAAO,QAAQ;CAC7B;CAEA,OAAO;EACL,KAAKC,UAAU,QAAQ;EACvB,KAAKD,OAAO,KAAK,KAAKD,GAAG;EACzB,OAAO,KAAKC,OAAO,QAAQ;CAC7B;AACF;AAEA,IAAM,cAAc,IAAI,WAAW,oCAAoC;AAEvE,SAAgB,UAAU,KAOZ;CAEZ,OAAO,YAAY,MAAM,IAAI,QAAQ,KAAK,KAAA;AAC5C"}

package/out/zero-cache/src/services/analyze.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"analyze.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/analyze.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,oDAAoD,CAAC;AAC3F,OAAO,KAAK,EAAC,GAAG,EAAC,MAAM,mCAAmC,CAAC;AAC3D,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6CAA6C,CAAC;AAC9E,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,kDAAkD,CAAC;AAWxF,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,iBAAiB,CAAC;AAC7C,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,wBAAwB,CAAC;AAQjE,wBAAsB,YAAY,CAChC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,oBAAoB,EAC5B,YAAY,EAAE,YAAY,EAC1B,GAAG,EAAE,GAAG,EACR,UAAU,UAAO,EACjB,UAAU,UAAQ,EAClB,WAAW,CAAC,EAAE,iBAAiB,EAC/B,IAAI,CAAC,EAAE,OAAO,EACd,SAAS,UAAQ,GAChB,OAAO,CAAC,kBAAkB,CAAC,CA4F7B"}
+{"version":3,"file":"analyze.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/analyze.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,oDAAoD,CAAC;AAC3F,OAAO,KAAK,EAAC,GAAG,EAAC,MAAM,mCAAmC,CAAC;AAC3D,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6CAA6C,CAAC;AAC9E,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,kDAAkD,CAAC;AAWxF,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,iBAAiB,CAAC;AAC7C,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,wBAAwB,CAAC;AAQjE,wBAAsB,YAAY,CAChC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,oBAAoB,EAC5B,YAAY,EAAE,YAAY,EAC1B,GAAG,EAAE,GAAG,EACR,UAAU,UAAO,EACjB,UAAU,UAAQ,EAClB,WAAW,CAAC,EAAE,iBAAiB,EAC/B,IAAI,CAAC,EAAE,OAAO,EACd,SAAS,UAAQ,GAChB,OAAO,CAAC,kBAAkB,CAAC,CA4F7B"}

package/out/zero-cache/src/services/analyze.js

@@ -1,3 +1,4 @@
+import { getOrInsertComputed } from "../../../shared/src/map.js";
 import { MemoryStorage } from "../../../zql/src/ivm/memory-storage.js";
 import { AccumulatorDebugger, serializePlanDebugEvents } from "../../../zql/src/planner/planner-debug.js";
 import { Debug } from "../../../zql/src/builder/debug-delegate.js";
@@ -5,7 +6,7 @@ import { Database } from "../../../zqlite/src/db.js";
 import { TableSource } from "../../../zqlite/src/table-source.js";
 import { computeZqlSpecs, mustGetTableSpec } from "../db/lite-tables.js";
 import { createSQLiteCostModel } from "../../../zqlite/src/sqlite-cost-model.js";
-import { runAst } from "./run-ast.js";
+import { MAX_ANALYZE_ROWS, runAst } from "./run-ast.js";
 import { explainQueries } from "../../../zqlite/src/explain-queries.js";
 import { _usingCtx } from "../../../_virtual/_@oxc-project_runtime@0.130.0/helpers/usingCtx.js";
 import { TimeSliceTimer } from "./view-syncer/view-syncer.js";
@@ -34,17 +35,18 @@ async function analyzeQuery(lc, config, clientSchema, ast, syncedRows = true, ve
 			permissions,
 			costModel,
 			planDebugger,
+			maxSyncedRowsPerTable: MAX_ANALYZE_ROWS,
 			host: {
-				debug: new Debug(),
+				debug: new Debug(vendedRows, MAX_ANALYZE_ROWS),
 				enableNotExists: true,
+				disableCorrelatedPredicatePushdown: config.enableCorrelatedPredicatePushdown === false,
+				enablePlannerAwarePushdown: config.enablePlannerAwarePushdown !== false,
 				getSource(tableName) {
-					let source = tables.get(tableName);
-					if (source) return source;
-					const tableSpec = mustGetTableSpec(tableSpecs, tableName);
-					const { primaryKey } = tableSpec.tableSpec;
-					source = new TableSource(lc, config.log, db, tableName, tableSpec.zqlSpec, primaryKey, shouldYield);
-					tables.set(tableName, source);
-					return source;
+					return getOrInsertComputed(tables, tableName, (tableName) => {
+						const tableSpec = mustGetTableSpec(tableSpecs, tableName);
+						const { primaryKey } = tableSpec.tableSpec;
+						return new TableSource(lc, config.log, db, tableName, tableSpec.zqlSpec, primaryKey, shouldYield);
+					});
 				},
 				createStorage() {
 					return new MemoryStorage();

package/out/zero-cache/src/services/analyze.js.map

@@ -1 +1 @@
-{"version":3,"file":"analyze.js","names":[],"sources":["../../../../../zero-cache/src/services/analyze.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {AnalyzeQueryResult} from '../../../zero-protocol/src/analyze-query-result.ts';\nimport type {AST} from '../../../zero-protocol/src/ast.ts';\nimport type {ClientSchema} from '../../../zero-protocol/src/client-schema.ts';\nimport type {PermissionsConfig} from '../../../zero-schema/src/compiled-permissions.ts';\nimport {Debug} from '../../../zql/src/builder/debug-delegate.ts';\nimport {MemoryStorage} from '../../../zql/src/ivm/memory-storage.ts';\nimport {\n  AccumulatorDebugger,\n  serializePlanDebugEvents,\n} from '../../../zql/src/planner/planner-debug.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport {explainQueries} from '../../../zqlite/src/explain-queries.ts';\nimport {createSQLiteCostModel} from '../../../zqlite/src/sqlite-cost-model.ts';\nimport {TableSource} from '../../../zqlite/src/table-source.ts';\nimport type {JWTAuth} from '../auth/auth.ts';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {computeZqlSpecs, mustGetTableSpec} from '../db/lite-tables.ts';\nimport type {LiteAndZqlSpec, LiteTableSpec} from '../db/specs.ts';\nimport {runAst} from './run-ast.ts';\nimport {TimeSliceTimer} from './view-syncer/view-syncer.ts';\n\nconst TIME_SLICE_LAP_THRESHOLD_MS = 200;\n\nexport async function analyzeQuery(\n  lc: LogContext,\n  config: NormalizedZeroConfig,\n  clientSchema: ClientSchema,\n  ast: AST,\n  syncedRows = true,\n  vendedRows = false,\n  permissions?: PermissionsConfig,\n  auth?: JWTAuth,\n  joinPlans = false,\n): Promise<AnalyzeQueryResult> {\n  using db = new Database(lc, config.replica.file);\n  const fullTables = new Map<string, LiteTableSpec>();\n  const tableSpecs = new Map<string, LiteAndZqlSpec>();\n  const tables = new Map<string, TableSource>();\n\n  computeZqlSpecs(\n    lc,\n    db,\n    {includeBackfillingColumns: false},\n    tableSpecs,\n    fullTables,\n  );\n\n  // Mirror production: the planner runs iff ZERO_ENABLE_QUERY_PLANNER is set\n  // on the server, so the analysis reflects what actually executes. Diagnostic\n  // event collection is orthogonal and opt-in via `joinPlans`.\n  const costModel = config.enableQueryPlanner\n    ? createSQLiteCostModel(db, tableSpecs)\n    : undefined;\n  const planDebugger = joinPlans ? new AccumulatorDebugger() : undefined;\n  const timer = await new TimeSliceTimer(lc).start();\n  const shouldYield = () => timer.elapsedLap() > TIME_SLICE_LAP_THRESHOLD_MS;\n  const yieldProcess = () => timer.yieldProcess();\n  const result = await runAst(\n    lc,\n    clientSchema,\n    ast,\n    true,\n    {\n      applyPermissions: permissions !== undefined,\n      syncedRows,\n      vendedRows,\n      auth,\n      db,\n      tableSpecs,\n      permissions,\n      costModel,\n      planDebugger,\n      host: {\n        debug: new Debug(),\n        enableNotExists: true,\n        getSource(tableName: string) {\n          let source = tables.get(tableName);\n          if (source) {\n            return source;\n          }\n\n          const tableSpec = mustGetTableSpec(tableSpecs, tableName);\n          const {primaryKey} = tableSpec.tableSpec;\n\n          source = new TableSource(\n            lc,\n            config.log,\n            db,\n            tableName,\n            tableSpec.zqlSpec,\n            primaryKey,\n            shouldYield,\n          );\n          tables.set(tableName, source);\n          return source;\n        },\n        createStorage() {\n          return new MemoryStorage();\n        },\n        decorateSourceInput: input => input,\n        decorateInput: input => input,\n        addEdge() {},\n        decorateFilterInput: input => input,\n      },\n    },\n    yieldProcess,\n  );\n\n  // Fill in plans for any queries SQLite did not actually execute (and thus\n  // did not populate scanStatus EXPLAIN for) using the substituted-binding\n  // fallback. Plans captured at execution time use the real bindings and win.\n  const fallback = explainQueries(result.readRowCountsByQuery ?? {}, db);\n  const captured = result.sqlitePlans ?? {};\n  for (const [query, plan] of Object.entries(fallback)) {\n    if (!captured[query]) {\n      captured[query] = plan;\n    }\n  }\n  result.sqlitePlans = captured;\n\n  if (planDebugger) {\n    result.joinPlans = serializePlanDebugEvents(planDebugger.events);\n  }\n\n  return result;\n}\n"],"mappings":";;;;;;;;;;;;AAsBA,IAAM,8BAA8B;AAEpC,eAAsB,aACpB,IACA,QACA,cACA,KACA,aAAa,MACb,aAAa,OACb,aACA,MACA,YAAY,OACiB;;;EAC7B,MAAM,KAAA,YAAA,EAAK,IAAI,SAAS,IAAI,OAAO,QAAQ,IAAI,CAAA;EAC/C,MAAM,6BAAa,IAAI,IAA2B;EAClD,MAAM,6BAAa,IAAI,IAA4B;EACnD,MAAM,yBAAS,IAAI,IAAyB;EAE5C,gBACE,IACA,IACA,EAAC,2BAA2B,MAAK,GACjC,YACA,UACF;EAKA,MAAM,YAAY,OAAO,qBACrB,sBAAsB,IAAI,UAAU,IACpC,KAAA;EACJ,MAAM,eAAe,YAAY,IAAI,oBAAoB,IAAI,KAAA;EAC7D,MAAM,QAAQ,MAAM,IAAI,eAAe,EAAE,EAAE,MAAM;EACjD,MAAM,oBAAoB,MAAM,WAAW,IAAI;EAC/C,MAAM,qBAAqB,MAAM,aAAa;EAC9C,MAAM,SAAS,MAAM,OACnB,IACA,cACA,KACA,MACA;GACE,kBAAkB,gBAAgB,KAAA;GAClC;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA,MAAM;IACJ,OAAO,IAAI,MAAM;IACjB,iBAAiB;IACjB,UAAU,WAAmB;KAC3B,IAAI,SAAS,OAAO,IAAI,SAAS;KACjC,IAAI,QACF,OAAO;KAGT,MAAM,YAAY,iBAAiB,YAAY,SAAS;KACxD,MAAM,EAAC,eAAc,UAAU;KAE/B,SAAS,IAAI,YACX,IACA,OAAO,KACP,IACA,WACA,UAAU,SACV,YACA,WACF;KACA,OAAO,IAAI,WAAW,MAAM;KAC5B,OAAO;IACT;IACA,gBAAgB;KACd,OAAO,IAAI,cAAc;IAC3B;IACA,sBAAqB,UAAS;IAC9B,gBAAe,UAAS;IACxB,UAAU,CAAC;IACX,sBAAqB,UAAS;GAChC;EACF,GACA,YACF;EAKA,MAAM,WAAW,eAAe,OAAO,wBAAwB,CAAC,GAAG,EAAE;EACrE,MAAM,WAAW,OAAO,eAAe,CAAC;EACxC,KAAK,MAAM,CAAC,OAAO,SAAS,OAAO,QAAQ,QAAQ,GACjD,IAAI,CAAC,SAAS,QACZ,SAAS,SAAS;EAGtB,OAAO,cAAc;EAErB,IAAI,cACF,OAAO,YAAY,yBAAyB,aAAa,MAAM;EAGjE,OAAO;;;;;;AACT"}
+{"version":3,"file":"analyze.js","names":[],"sources":["../../../../../zero-cache/src/services/analyze.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {getOrInsertComputed} from '../../../shared/src/map.ts';\nimport type {AnalyzeQueryResult} from '../../../zero-protocol/src/analyze-query-result.ts';\nimport type {AST} from '../../../zero-protocol/src/ast.ts';\nimport type {ClientSchema} from '../../../zero-protocol/src/client-schema.ts';\nimport type {PermissionsConfig} from '../../../zero-schema/src/compiled-permissions.ts';\nimport {Debug} from '../../../zql/src/builder/debug-delegate.ts';\nimport {MemoryStorage} from '../../../zql/src/ivm/memory-storage.ts';\nimport {\n  AccumulatorDebugger,\n  serializePlanDebugEvents,\n} from '../../../zql/src/planner/planner-debug.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport {explainQueries} from '../../../zqlite/src/explain-queries.ts';\nimport {createSQLiteCostModel} from '../../../zqlite/src/sqlite-cost-model.ts';\nimport {TableSource} from '../../../zqlite/src/table-source.ts';\nimport type {JWTAuth} from '../auth/auth.ts';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {computeZqlSpecs, mustGetTableSpec} from '../db/lite-tables.ts';\nimport type {LiteAndZqlSpec, LiteTableSpec} from '../db/specs.ts';\nimport {MAX_ANALYZE_ROWS, runAst} from './run-ast.ts';\nimport {TimeSliceTimer} from './view-syncer/view-syncer.ts';\n\nconst TIME_SLICE_LAP_THRESHOLD_MS = 200;\n\nexport async function analyzeQuery(\n  lc: LogContext,\n  config: NormalizedZeroConfig,\n  clientSchema: ClientSchema,\n  ast: AST,\n  syncedRows = true,\n  vendedRows = false,\n  permissions?: PermissionsConfig,\n  auth?: JWTAuth,\n  joinPlans = false,\n): Promise<AnalyzeQueryResult> {\n  using db = new Database(lc, config.replica.file);\n  const fullTables = new Map<string, LiteTableSpec>();\n  const tableSpecs = new Map<string, LiteAndZqlSpec>();\n  const tables = new Map<string, TableSource>();\n\n  computeZqlSpecs(\n    lc,\n    db,\n    {includeBackfillingColumns: false},\n    tableSpecs,\n    fullTables,\n  );\n\n  // Mirror production: the planner runs iff ZERO_ENABLE_QUERY_PLANNER is set\n  // on the server, so the analysis reflects what actually executes. Diagnostic\n  // event collection is orthogonal and opt-in via `joinPlans`.\n  const costModel = config.enableQueryPlanner\n    ? createSQLiteCostModel(db, tableSpecs)\n    : undefined;\n  const planDebugger = joinPlans ? new AccumulatorDebugger() : undefined;\n  const timer = await new TimeSliceTimer(lc).start();\n  const shouldYield = () => timer.elapsedLap() > TIME_SLICE_LAP_THRESHOLD_MS;\n  const yieldProcess = () => timer.yieldProcess();\n  const result = await runAst(\n    lc,\n    clientSchema,\n    ast,\n    true,\n    {\n      applyPermissions: permissions !== undefined,\n      syncedRows,\n      vendedRows,\n      auth,\n      db,\n      tableSpecs,\n      permissions,\n      costModel,\n      planDebugger,\n      maxSyncedRowsPerTable: MAX_ANALYZE_ROWS,\n      host: {\n        debug: new Debug(vendedRows, MAX_ANALYZE_ROWS),\n        enableNotExists: true,\n        // Mirror production, as with the planner above.\n        disableCorrelatedPredicatePushdown:\n          config.enableCorrelatedPredicatePushdown === false,\n        enablePlannerAwarePushdown: config.enablePlannerAwarePushdown !== false,\n        getSource(tableName: string) {\n          return getOrInsertComputed(tables, tableName, tableName => {\n            const tableSpec = mustGetTableSpec(tableSpecs, tableName);\n            const {primaryKey} = tableSpec.tableSpec;\n\n            return new TableSource(\n              lc,\n              config.log,\n              db,\n              tableName,\n              tableSpec.zqlSpec,\n              primaryKey,\n              shouldYield,\n            );\n          });\n        },\n        createStorage() {\n          return new MemoryStorage();\n        },\n        decorateSourceInput: input => input,\n        decorateInput: input => input,\n        addEdge() {},\n        decorateFilterInput: input => input,\n      },\n    },\n    yieldProcess,\n  );\n\n  // Fill in plans for any queries SQLite did not actually execute (and thus\n  // did not populate scanStatus EXPLAIN for) using the substituted-binding\n  // fallback. Plans captured at execution time use the real bindings and win.\n  const fallback = explainQueries(result.readRowCountsByQuery ?? {}, db);\n  const captured = result.sqlitePlans ?? {};\n  for (const [query, plan] of Object.entries(fallback)) {\n    if (!captured[query]) {\n      captured[query] = plan;\n    }\n  }\n  result.sqlitePlans = captured;\n\n  if (planDebugger) {\n    result.joinPlans = serializePlanDebugEvents(planDebugger.events);\n  }\n\n  return result;\n}\n"],"mappings":";;;;;;;;;;;;;AAuBA,IAAM,8BAA8B;AAEpC,eAAsB,aACpB,IACA,QACA,cACA,KACA,aAAa,MACb,aAAa,OACb,aACA,MACA,YAAY,OACiB;;;EAC7B,MAAM,KAAA,YAAA,EAAK,IAAI,SAAS,IAAI,OAAO,QAAQ,IAAI,CAAA;EAC/C,MAAM,6BAAa,IAAI,IAA2B;EAClD,MAAM,6BAAa,IAAI,IAA4B;EACnD,MAAM,yBAAS,IAAI,IAAyB;EAE5C,gBACE,IACA,IACA,EAAC,2BAA2B,MAAK,GACjC,YACA,UACF;EAKA,MAAM,YAAY,OAAO,qBACrB,sBAAsB,IAAI,UAAU,IACpC,KAAA;EACJ,MAAM,eAAe,YAAY,IAAI,oBAAoB,IAAI,KAAA;EAC7D,MAAM,QAAQ,MAAM,IAAI,eAAe,EAAE,EAAE,MAAM;EACjD,MAAM,oBAAoB,MAAM,WAAW,IAAI;EAC/C,MAAM,qBAAqB,MAAM,aAAa;EAC9C,MAAM,SAAS,MAAM,OACnB,IACA,cACA,KACA,MACA;GACE,kBAAkB,gBAAgB,KAAA;GAClC;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA,uBAAuB;GACvB,MAAM;IACJ,OAAO,IAAI,MAAM,YAAY,gBAAgB;IAC7C,iBAAiB;IAEjB,oCACE,OAAO,sCAAsC;IAC/C,4BAA4B,OAAO,+BAA+B;IAClE,UAAU,WAAmB;KAC3B,OAAO,oBAAoB,QAAQ,YAAW,cAAa;MACzD,MAAM,YAAY,iBAAiB,YAAY,SAAS;MACxD,MAAM,EAAC,eAAc,UAAU;MAE/B,OAAO,IAAI,YACT,IACA,OAAO,KACP,IACA,WACA,UAAU,SACV,YACA,WACF;KACF,CAAC;IACH;IACA,gBAAgB;KACd,OAAO,IAAI,cAAc;IAC3B;IACA,sBAAqB,UAAS;IAC9B,gBAAe,UAAS;IACxB,UAAU,CAAC;IACX,sBAAqB,UAAS;GAChC;EACF,GACA,YACF;EAKA,MAAM,WAAW,eAAe,OAAO,wBAAwB,CAAC,GAAG,EAAE;EACrE,MAAM,WAAW,OAAO,eAAe,CAAC;EACxC,KAAK,MAAM,CAAC,OAAO,SAAS,OAAO,QAAQ,QAAQ,GACjD,IAAI,CAAC,SAAS,QACZ,SAAS,SAAS;EAGtB,OAAO,cAAc;EAErB,IAAI,cACF,OAAO,YAAY,yBAAyB,aAAa,MAAM;EAGjE,OAAO;;;;;;AACT"}

package/out/zero-cache/src/services/change-source/common/backfill-manager.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"backfill-manager.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/common/backfill-manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAOjD,OAAO,EAAmB,KAAK,UAAU,EAAC,MAAM,wBAAwB,CAAC;AAKzE,OAAO,KAAK,EACV,iBAAiB,EACjB,eAAe,EACf,mBAAmB,EAEnB,eAAe,EAChB,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EACV,UAAU,EACV,uBAAuB,EACvB,QAAQ,EACT,MAAM,gCAAgC,CAAC;AAMxC;;;;;GAKG;AACH,MAAM,MAAM,eAAe,GAAG;IAC5B,OAAO,EAAE,eAAe,GAAG,iBAAiB,CAAC;IAC7C,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,KAAK,gBAAgB,GAAG,CACtB,GAAG,EAAE,eAAe,KACjB,cAAc,CAAC,eAAe,CAAC,CAAC;AAuBrC;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,eAAgB,YAAW,UAAU,EAAE,QAAQ;;IAgC1D,YACE,EAAE,EAAE,UAAU,EACd,cAAc,EAAE,uBAAuB,EACvC,gBAAgB,EAAE,gBAAgB,EAClC,UAAU,GAAE,UAA6B,EACzC,YAAY,SAA0B,EACtC,YAAY,SAA0B,EACtC,oBAAoB,SAAyB,EAU9C;IAED,GAAG,CAAC,aAAa,EAAE,MAAM,EAAE,eAAe,EAAE,eAAe,EAAE,QAU5D;IAqPD;;;OAGG;IACH,QAAQ,CAAC,OAAO,EAAE,mBAAmB,GAAG,IAAI,CA2M3C;IAED,MAAM,IAAI,IAAI,CAGb;CACF;AAED,uBAAe,mBAAoB,SAAQ,KAAK;IAC9C,YAAY,EAAE,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,EAM5D;CACF;AA6BD;;;;;GAKG;AACH,qBAAa,0BAA2B,SAAQ,mBAAmB;IACjE,QAAQ,CAAC,IAAI,gCAAgC;IAE7C,YAAY,EAAE,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,EAE5D;CACF"}
+{"version":3,"file":"backfill-manager.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/common/backfill-manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAUjD,OAAO,EAAmB,KAAK,UAAU,EAAC,MAAM,wBAAwB,CAAC;AAKzE,OAAO,KAAK,EACV,iBAAiB,EACjB,eAAe,EACf,mBAAmB,EAEnB,eAAe,EAChB,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EACV,UAAU,EACV,uBAAuB,EACvB,QAAQ,EACT,MAAM,gCAAgC,CAAC;AAMxC;;;;;GAKG;AACH,MAAM,MAAM,eAAe,GAAG;IAC5B,OAAO,EAAE,eAAe,GAAG,iBAAiB,CAAC;IAC7C,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,KAAK,gBAAgB,GAAG,CACtB,GAAG,EAAE,eAAe,KACjB,cAAc,CAAC,eAAe,CAAC,CAAC;AAkFrC;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,eAAgB,YAAW,UAAU,EAAE,QAAQ;;IA4C1D,YACE,EAAE,EAAE,UAAU,EACd,cAAc,EAAE,uBAAuB,EACvC,gBAAgB,EAAE,gBAAgB,EAClC,UAAU,GAAE,UAA6B,EACzC,YAAY,SAA0B,EACtC,YAAY,SAA0B,EACtC,oBAAoB,SAAyB,EAU9C;IAED,GAAG,CAAC,aAAa,EAAE,MAAM,EAAE,eAAe,EAAE,eAAe,EAAE,QAU5D;IA4bD;;;OAGG;IACH,QAAQ,CAAC,OAAO,EAAE,mBAAmB,GAAG,IAAI,CA6M3C;IAED,MAAM,IAAI,IAAI,CAab;CACF;AAED,uBAAe,mBAAoB,SAAQ,KAAK;IAC9C,YAAY,EAAE,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,EAM5D;CACF;AA6BD;;;;;GAKG;AACH,qBAAa,0BAA2B,SAAQ,mBAAmB;IACjE,QAAQ,CAAC,IAAI,gCAAgC;IAE7C,YAAY,EAAE,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,EAE5D;CACF"}