@rocicorp/zero 1.2.0 → 1.3.0-canary.1

package/out/zero-cache/src/services/mutagen/pusher.js

@@ -14,7 +14,9 @@ import { Queue } from "../../../../shared/src/queue.js";
 import { Subscription } from "../../types/subscription.js";
 import { recordMutation } from "../../server/anonymous-otel-start.js";
 import { ProtocolErrorWithLevel } from "../../types/error-with-level.js";
-import { compileUrlPattern, fetchFromAPIServer } from "../../custom/fetch.js";
+import { fetchFromAPIServer } from "../../custom/fetch.js";
+import { authEquals, isAuthErrorBody } from "../../auth/auth.js";
+import { ROOT_CONTEXT, context, propagation } from "@opentelemetry/api";
 //#region ../zero-cache/src/services/mutagen/pusher.ts
 /**
 * Receives push messages from zero-client and forwards
@@ -31,44 +33,32 @@ import { compileUrlPattern, fetchFromAPIServer } from "../../custom/fetch.js";
 */
 var PusherService = class {
 	id;
+	#contextManager;
 	#pusher;
 	#queue;
-	#pushConfig;
 	#config;
 	#lc;
-	#pushURLPatterns;
 	#stopped;
 	#refCount = 0;
 	#isStopped = false;
-	constructor(appConfig, pushConfig, lc, clientGroupID) {
+	constructor(appConfig, lc, clientGroupID, contextManager) {
+		this.#contextManager = contextManager;
 		this.#config = appConfig;
 		this.#lc = lc.withContext("component", "pusherService");
-		this.#pushURLPatterns = pushConfig.url.map(compileUrlPattern);
 		this.#queue = new Queue();
-		this.#pusher = new PushWorker(appConfig, lc, pushConfig.url, pushConfig.apiKey, pushConfig.allowedClientHeaders, this.#queue);
+		this.#pusher = new PushWorker(appConfig, lc, this.#contextManager, this.#queue);
 		this.id = clientGroupID;
-		this.#pushConfig = pushConfig;
 	}
-	get pushURL() {
-		return this.#pusher.pushURLs[0];
+	initConnection(selector) {
+		return this.#pusher.initConnection(selector);
 	}
-	initConnection(clientID, wsID, userPushURL, userPushHeaders, onAuthFailure) {
-		return this.#pusher.initConnection(clientID, wsID, userPushURL, userPushHeaders, onAuthFailure);
-	}
-	enqueuePush(clientID, push, auth, httpCookie, origin) {
-		if (!this.#pushConfig.forwardCookies) httpCookie = void 0;
-		this.#queue.enqueue({
-			push,
-			auth,
-			clientID,
-			httpCookie,
-			origin
-		});
+	enqueuePush(selector, push) {
+		this.#pusher.enqueuePush(this.#contextManager.mustGetConnectionContext(selector), push);
 		return { type: "ok" };
 	}
-	async ackMutationResponses(upToID) {
-		const url = this.#pusher.effectivePushURL;
-		if (!url) return;
+	async ackMutationResponses(requester, upToID) {
+		const ctx = this.#contextManager.getConnectionContext(requester);
+		if (!ctx?.pushContext?.url) return;
 		const cleanupBody = {
 			clientGroupID: this.id,
 			mutations: [{
@@ -89,18 +79,24 @@ var PusherService = class {
 			requestID: `cleanup-${this.id}-${upToID.clientID}-${upToID.id}`
 		};
 		try {
-			await fetchFromAPIServer(pushResponseSchema, "push", this.#lc, url, this.#pushURLPatterns, {
+			await fetchFromAPIServer(pushResponseSchema, "push", this.#lc, ctx, {
 				appID: this.#config.app.id,
 				shardNum: this.#config.shard.num
-			}, { apiKey: this.#pushConfig.apiKey }, cleanupBody);
+			}, cleanupBody);
 		} catch (e) {
 			this.#lc.warn?.("Failed to send cleanup mutation", { error: getErrorMessage(e) });
 		}
 	}
-	async deleteClientMutations(clientIDs) {
+	/**
+	* Bulk cleanup is routed through the requester's push context.
+	*
+	* This assumes the client group shares a compatible push endpoint/auth
+	* context.
+	*/
+	async deleteClientMutations(requester, clientIDs) {
 		if (clientIDs.length === 0) return;
-		const url = this.#pusher.effectivePushURL;
-		if (!url) return;
+		const ctx = this.#contextManager.getConnectionContext(requester);
+		if (!ctx?.pushContext?.url) return;
 		const cleanupBody = {
 			clientGroupID: this.id,
 			mutations: [{
@@ -120,10 +116,10 @@ var PusherService = class {
 			requestID: `cleanup-bulk-${this.id}-${Date.now()}`
 		};
 		try {
-			await fetchFromAPIServer(pushResponseSchema, "push", this.#lc, url, this.#pushURLPatterns, {
+			await fetchFromAPIServer(pushResponseSchema, "push", this.#lc, ctx, {
 				appID: this.#config.app.id,
 				shardNum: this.#config.shard.num
-			}, { apiKey: this.#pushConfig.apiKey }, cleanupBody);
+			}, cleanupBody);
 		} catch (e) {
 			this.#lc.warn?.("Failed to send bulk cleanup mutation", { error: getErrorMessage(e) });
 		}
@@ -156,65 +152,49 @@ var PusherService = class {
 * to the user's API server.
 */
 var PushWorker = class {
-	#pushURLs;
-	#pushURLPatterns;
-	#apiKey;
-	#allowedClientHeaders;
+	#contextManager;
 	#queue;
 	#lc;
 	#config;
 	#clients;
-	#userPushURL;
-	#userPushHeaders;
 	#customMutations = getOrCreateCounter("mutation", "custom", "Number of custom mutations processed");
 	#pushes = getOrCreateCounter("mutation", "pushes", "Number of pushes processed by the pusher");
-	constructor(config, lc, pushURL, apiKey, allowedClientHeaders, queue) {
-		this.#pushURLs = pushURL;
+	constructor(config, lc, contextManager, queue) {
 		this.#lc = lc.withContext("component", "pusher");
-		this.#pushURLPatterns = pushURL.map(compileUrlPattern);
-		this.#apiKey = apiKey;
-		this.#allowedClientHeaders = allowedClientHeaders;
+		this.#contextManager = contextManager;
 		this.#queue = queue;
 		this.#config = config;
 		this.#clients = /* @__PURE__ */ new Map();
 	}
-	get pushURLs() {
-		return this.#pushURLs;
-	}
-	get effectivePushURL() {
-		return this.#userPushURL ?? this.#pushURLs[0];
-	}
 	/**
 	* Returns a new downstream stream if the clientID,wsID pair has not been seen before.
 	* If a clientID already exists with a different wsID, that client's downstream is cancelled.
 	*/
-	initConnection(clientID, wsID, userPushURL, userPushHeaders, onAuthFailure) {
-		const existing = this.#clients.get(clientID);
-		if (existing && existing.wsID === wsID) throw new Error("Connection was already initialized");
+	initConnection(selector) {
+		const existing = this.#clients.get(selector.clientID);
+		if (existing && existing.wsID === selector.wsID) throw new Error("Connection was already initialized");
 		if (existing) existing.downstream.cancel();
-		if (this.#userPushURL === void 0) {
-			this.#userPushURL = userPushURL;
-			this.#userPushHeaders = userPushHeaders;
-		} else if (this.#userPushURL !== userPushURL) this.#lc.warn?.("Client provided different mutate parameters than client group", {
-			clientID,
-			clientURL: userPushURL,
-			clientGroupURL: this.#userPushURL
-		});
 		const downstream = Subscription.create({ cleanup: () => {
-			this.#clients.delete(clientID);
+			this.#clients.delete(selector.clientID);
 		} });
-		this.#clients.set(clientID, {
-			wsID,
-			downstream,
-			onAuthFailure
+		this.#clients.set(selector.clientID, {
+			wsID: selector.wsID,
+			downstream
 		});
 		return downstream;
 	}
+	enqueuePush(context, push) {
+		this.#queue.enqueue({
+			push,
+			context
+		});
+	}
 	async run() {
 		for (;;) {
 			const [pushes, terminate] = combinePushes([await this.#queue.dequeue(), ...this.#queue.drain()]);
 			for (const push of pushes) {
-				const response = await this.#processPush(push);
+				const parentContext = push.push.traceparent ? propagation.extract(ROOT_CONTEXT, { traceparent: push.push.traceparent }) : context.active();
+				const response = await context.with(parentContext, () => this.#processPush(push));
 				await this.#fanOutResponses(response);
 			}
 			if (terminate) break;
@@ -256,17 +236,8 @@ var PushWorker = class {
 						message: response.error === "zeroPusher" ? response.details : response.error === "unsupportedSchemaVersion" ? "Unsupported schema version" : "An unknown error occurred while pushing to the API server"
 					};
 					this.#failDownstream(client.downstream, pushFailedBody);
-					if (isPushAuthFailure(pushFailedBody)) {
-						this.#lc.debug?.("Auth failure detected in push response");
-						client.onAuthFailure?.();
-					}
-				} else if ("kind" in response) {
-					this.#failDownstream(client.downstream, response);
-					if (isPushAuthFailure(response)) {
-						this.#lc.debug?.("Auth failure detected in push response");
-						client.onAuthFailure?.();
-					}
-				} else unreachable(response);
+				} else if ("kind" in response) this.#failDownstream(client.downstream, response);
+				else unreachable(response);
 			}
 		} else {
 			const groupedMutations = groupBy(response.mutations, (m) => m.id.clientID);
@@ -303,7 +274,7 @@ var PushWorker = class {
 		this.#customMutations.add(entry.push.mutations.length, { clientGroupID: entry.push.clientGroupID });
 		this.#pushes.add(1, { clientGroupID: entry.push.clientGroupID });
 		recordMutation("custom", entry.push.mutations.length);
-		const url = this.#userPushURL ?? must(this.#pushURLs[0], "ZERO_MUTATE_URL is not set");
+		const url = must(entry.context.pushContext.url, "ZERO_MUTATE_URL is not set");
 		this.#lc.debug?.("pushing to", url, "with", entry.push.mutations.length, "mutations");
 		let mutationIDs = [];
 		try {
@@ -311,22 +282,37 @@ var PushWorker = class {
 				id: m.id,
 				clientID: m.clientID
 			}));
-			return await fetchFromAPIServer(pushResponseSchema, "push", this.#lc, url, this.#pushURLPatterns, {
+			const response = await fetchFromAPIServer(pushResponseSchema, "push", this.#lc, entry.context, {
 				appID: this.#config.app.id,
 				shardNum: this.#config.shard.num
-			}, {
-				apiKey: this.#apiKey,
-				customHeaders: this.#userPushHeaders,
-				allowedClientHeaders: this.#allowedClientHeaders,
-				token: entry.auth,
-				cookie: entry.httpCookie,
-				origin: entry.origin
 			}, entry.push);
+			if ("kind" in response || "error" in response) {
+				if (isAuthErrorBody(response)) {
+					this.#lc.warn?.("Push auth failed; invalidating connection", {
+						clientID: entry.context.clientID,
+						response: "kind" in response ? response.message : void 0
+					});
+					this.#contextManager.failConnection(entry.context, entry.context.revision);
+				}
+				return response;
+			}
+			this.#contextManager.validateConnection(entry.context, entry.context.revision);
+			return response;
 		} catch (e) {
-			if (isProtocolError(e) && e.errorBody.kind === "PushFailed") return {
-				...e.errorBody,
-				mutationIDs
-			};
+			if (isProtocolError(e) && e.errorBody.kind === "PushFailed") {
+				const response = {
+					...e.errorBody,
+					mutationIDs
+				};
+				if (isAuthErrorBody(response)) {
+					this.#lc.warn?.("Push auth failed; invalidating connection", {
+						clientID: entry.context.clientID,
+						response: "kind" in response ? response.message : void 0
+					});
+					this.#contextManager.failConnection(entry.context, entry.context.revision);
+				}
+				return response;
+			}
 			return {
 				kind: PushFailed,
 				origin: ZeroCache,
@@ -340,20 +326,17 @@ var PushWorker = class {
 		downstream.fail(new ProtocolErrorWithLevel(errorBody, "warn"));
 	}
 };
-function isPushAuthFailure(errorBody) {
-	return errorBody.reason === "http" && (errorBody.status === 401 || errorBody.status === 403);
-}
 /**
-* Pushes for different clientIDs could theoretically be interleaved.
+* Pushes for different clients, sockets, or auth revisions could be interleaved.
 *
-* In order to do efficient batching to the user's API server,
-* we collect all pushes for the same clientID into a single push.
+* In order to batch safely, we only combine pushes from the same
+* clientID/wsID/revision snapshot.
 */
 function combinePushes(entries) {
-	const pushesByClientID = /* @__PURE__ */ new Map();
+	const pushesByConnection = /* @__PURE__ */ new Map();
 	function collect() {
 		const ret = [];
-		for (const entries of pushesByClientID.values()) {
+		for (const entries of pushesByConnection.values()) {
 			const composite = {
 				...entries[0],
 				push: {
@@ -371,20 +354,24 @@ function combinePushes(entries) {
 	}
 	for (const entry of entries) {
 		if (entry === "stop" || entry === void 0) return [collect(), true];
-		const { clientID } = entry;
-		const existing = pushesByClientID.get(clientID);
+		const key = `${entry.context.clientID}:${entry.context.wsID}:${entry.context.revision}`;
+		const existing = pushesByConnection.get(key);
 		if (existing) existing.push(entry);
-		else pushesByClientID.set(clientID, [entry]);
+		else pushesByConnection.set(key, [entry]);
 	}
 	return [collect(), false];
 }
 function assertAreCompatiblePushes(left, right) {
-	assert(left.clientID === right.clientID, "clientID must be the same for all pushes");
-	assert(left.auth === right.auth, "auth must be the same for all pushes with the same clientID");
+	assert(left.context.clientID === right.context.clientID, "clientID must be the same for all pushes");
+	assert(left.context.wsID === right.context.wsID, "wsID must be the same for all pushes");
+	assert(left.context.revision === right.context.revision, "revision must be the same for all pushes");
+	assert(authEquals(left.context.auth, right.context.auth), "auth must be the same for all pushes with the same clientID");
 	assert(left.push.schemaVersion === right.push.schemaVersion, "schemaVersion must be the same for all pushes with the same clientID");
 	assert(left.push.pushVersion === right.push.pushVersion, "pushVersion must be the same for all pushes with the same clientID");
-	assert(left.httpCookie === right.httpCookie, "httpCookie must be the same for all pushes with the same clientID");
-	assert(left.origin === right.origin, "origin must be the same for all pushes with the same clientID");
+	assert(left.context.pushContext.headerOptions.cookie === right.context.pushContext.headerOptions.cookie, "httpCookie must be the same for all pushes with the same clientID");
+	assert(left.context.pushContext.headerOptions.origin === right.context.pushContext.headerOptions.origin, "origin must be the same for all pushes with the same clientID");
+	assert(left.context.userID === right.context.userID, "userID must be the same for all pushes with the same clientID");
+	assert(left.context.pushContext.url === right.context.pushContext.url, "userPushURL must be the same for all pushes with the same clientID");
 }
 //#endregion
 export { PusherService };

package/out/zero-cache/src/services/mutagen/pusher.js.map

@@ -1 +1 @@
-{"version":3,"file":"pusher.js","names":["#pusher","#queue","#pushConfig","#config","#lc","#pushURLPatterns","#isStopped","#refCount","#stopped","#pushURLs","#apiKey","#allowedClientHeaders","#clients","#customMutations","#pushes","#userPushURL","#userPushHeaders","#processPush","#fanOutResponses","#failDownstream"],"sources":["../../../../../../zero-cache/src/services/mutagen/pusher.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {groupBy} from '../../../../shared/src/arrays.ts';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport {getErrorMessage} from '../../../../shared/src/error.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Queue} from '../../../../shared/src/queue.ts';\nimport type {Downstream} from '../../../../zero-protocol/src/down.ts';\nimport {ErrorKind} from '../../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../../zero-protocol/src/error-origin.ts';\nimport {ErrorReason} from '../../../../zero-protocol/src/error-reason.ts';\nimport {\n  isProtocolError,\n  type PushFailedBody,\n} from '../../../../zero-protocol/src/error.ts';\nimport * as MutationType from '../../../../zero-protocol/src/mutation-type-enum.ts';\nimport {\n  CLEANUP_RESULTS_MUTATION_NAME,\n  pushResponseSchema,\n  type MutationID,\n  type PushBody,\n  type PushResponse,\n} from '../../../../zero-protocol/src/push.ts';\nimport {type ZeroConfig} from '../../config/zero-config.ts';\nimport {compileUrlPattern, fetchFromAPIServer} from '../../custom/fetch.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {recordMutation} from '../../server/anonymous-otel-start.ts';\nimport {ProtocolErrorWithLevel} from '../../types/error-with-level.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {HandlerResult, StreamResult} from '../../workers/connection.ts';\nimport type {RefCountedService, Service} from '../service.ts';\n\nexport interface Pusher extends RefCountedService {\n  readonly pushURL: string | undefined;\n\n  initConnection(\n    clientID: string,\n    wsID: string,\n    userPushURL: string | undefined,\n    userPushHeaders: Record<string, string> | undefined,\n    onAuthFailure?: () => void,\n  ): Source<Downstream>;\n  enqueuePush(\n    clientID: string,\n    push: PushBody,\n    auth: string | undefined,\n    httpCookie: string | undefined,\n    origin: string | undefined,\n  ): HandlerResult;\n  ackMutationResponses(upToID: MutationID): Promise<void>;\n  deleteClientMutations(clientIDs: string[]): Promise<void>;\n}\n\ntype Config = Pick<ZeroConfig, 'app' | 'shard'>;\n\n/**\n * Receives push messages from zero-client and forwards\n * them the the user's API server.\n *\n * If the user's API server is taking too long to process\n * the push, the PusherService will add the push to a queue\n * and send pushes in bulk the next time the user's API server\n * is available.\n *\n * - One PusherService exists per client group.\n * - Mutations for a given client are always sent in-order\n * - Mutations for different clients in the same group may be interleaved\n */\nexport class PusherService implements Service, Pusher {\n  readonly id: string;\n  readonly #pusher: PushWorker;\n  readonly #queue: Queue<PusherEntryOrStop>;\n  readonly #pushConfig: ZeroConfig['push'] & {url: string[]};\n  readonly #config: Config;\n  readonly #lc: LogContext;\n  readonly #pushURLPatterns: URLPattern[];\n  #stopped: Promise<void> | undefined;\n  #refCount = 0;\n  #isStopped = false;\n\n  constructor(\n    appConfig: Config,\n    pushConfig: ZeroConfig['push'] & {url: string[]},\n    lc: LogContext,\n    clientGroupID: string,\n  ) {\n    this.#config = appConfig;\n    this.#lc = lc.withContext('component', 'pusherService');\n    this.#pushURLPatterns = pushConfig.url.map(compileUrlPattern);\n    this.#queue = new Queue();\n    this.#pusher = new PushWorker(\n      appConfig,\n      lc,\n      pushConfig.url,\n      pushConfig.apiKey,\n      pushConfig.allowedClientHeaders,\n      this.#queue,\n    );\n    this.id = clientGroupID;\n    this.#pushConfig = pushConfig;\n  }\n\n  get pushURL(): string | undefined {\n    return this.#pusher.pushURLs[0];\n  }\n\n  initConnection(\n    clientID: string,\n    wsID: string,\n    userPushURL: string | undefined,\n    userPushHeaders: Record<string, string> | undefined,\n    onAuthFailure?: () => void,\n  ) {\n    return this.#pusher.initConnection(\n      clientID,\n      wsID,\n      userPushURL,\n      userPushHeaders,\n      onAuthFailure,\n    );\n  }\n\n  enqueuePush(\n    clientID: string,\n    push: PushBody,\n    auth: string | undefined,\n    httpCookie: string | undefined,\n    origin: string | undefined,\n  ): Exclude<HandlerResult, StreamResult> {\n    if (!this.#pushConfig.forwardCookies) {\n      httpCookie = undefined; // remove cookies if not forwarded\n    }\n    this.#queue.enqueue({push, auth, clientID, httpCookie, origin});\n\n    return {\n      type: 'ok',\n    };\n  }\n\n  async ackMutationResponses(upToID: MutationID) {\n    const url = this.#pusher.effectivePushURL;\n    if (!url) {\n      // No push URL configured, skip cleanup\n      return;\n    }\n\n    const cleanupBody: PushBody = {\n      clientGroupID: this.id,\n      mutations: [\n        {\n          type: MutationType.Custom,\n          id: 0, // Not tracked - this is fire-and-forget\n          clientID: upToID.clientID,\n          name: CLEANUP_RESULTS_MUTATION_NAME,\n          args: [\n            {\n              type: 'single',\n              clientGroupID: this.id,\n              clientID: upToID.clientID,\n              upToMutationID: upToID.id,\n            },\n          ],\n          timestamp: Date.now(),\n        },\n      ],\n      pushVersion: 1,\n      timestamp: Date.now(),\n      requestID: `cleanup-${this.id}-${upToID.clientID}-${upToID.id}`,\n    };\n\n    try {\n      await fetchFromAPIServer(\n        pushResponseSchema,\n        'push',\n        this.#lc,\n        url,\n        this.#pushURLPatterns,\n        {appID: this.#config.app.id, shardNum: this.#config.shard.num},\n        {apiKey: this.#pushConfig.apiKey},\n        cleanupBody,\n      );\n    } catch (e) {\n      this.#lc.warn?.('Failed to send cleanup mutation', {\n        error: getErrorMessage(e),\n      });\n    }\n  }\n\n  async deleteClientMutations(clientIDs: string[]) {\n    if (clientIDs.length === 0) {\n      return;\n    }\n    const url = this.#pusher.effectivePushURL;\n    if (!url) {\n      // No push URL configured, skip cleanup\n      return;\n    }\n\n    const cleanupBody: PushBody = {\n      clientGroupID: this.id,\n      mutations: [\n        {\n          type: MutationType.Custom,\n          id: 0, // Not tracked - this is fire-and-forget\n          clientID: clientIDs[0], // Use first client as sender\n          name: CLEANUP_RESULTS_MUTATION_NAME,\n          args: [\n            {\n              type: 'bulk',\n              clientGroupID: this.id,\n              clientIDs,\n            },\n          ],\n          timestamp: Date.now(),\n        },\n      ],\n      pushVersion: 1,\n      timestamp: Date.now(),\n      requestID: `cleanup-bulk-${this.id}-${Date.now()}`,\n    };\n\n    try {\n      await fetchFromAPIServer(\n        pushResponseSchema,\n        'push',\n        this.#lc,\n        url,\n        this.#pushURLPatterns,\n        {appID: this.#config.app.id, shardNum: this.#config.shard.num},\n        {apiKey: this.#pushConfig.apiKey},\n        cleanupBody,\n      );\n    } catch (e) {\n      this.#lc.warn?.('Failed to send bulk cleanup mutation', {\n        error: getErrorMessage(e),\n      });\n    }\n  }\n\n  ref() {\n    assert(!this.#isStopped, 'PusherService is already stopped');\n    ++this.#refCount;\n  }\n\n  unref() {\n    assert(!this.#isStopped, 'PusherService is already stopped');\n    --this.#refCount;\n    if (this.#refCount <= 0) {\n      void this.stop();\n    }\n  }\n\n  hasRefs(): boolean {\n    return this.#refCount > 0;\n  }\n\n  run(): Promise<void> {\n    this.#stopped = this.#pusher.run();\n    return this.#stopped;\n  }\n\n  stop(): Promise<void> {\n    if (this.#isStopped) {\n      return must(this.#stopped, 'Stop was called before `run`');\n    }\n    this.#isStopped = true;\n    this.#queue.enqueue('stop');\n    return must(this.#stopped, 'Stop was called before `run`');\n  }\n}\n\ntype PusherEntry = {\n  push: PushBody;\n  auth: string | undefined;\n  httpCookie: string | undefined;\n  origin: string | undefined;\n  clientID: string;\n};\ntype PusherEntryOrStop = PusherEntry | 'stop';\n\n/**\n * Awaits items in the queue then drains and sends them all\n * to the user's API server.\n */\nclass PushWorker {\n  readonly #pushURLs: string[];\n  readonly #pushURLPatterns: URLPattern[];\n  readonly #apiKey: string | undefined;\n  readonly #allowedClientHeaders: readonly string[] | undefined;\n  readonly #queue: Queue<PusherEntryOrStop>;\n  readonly #lc: LogContext;\n  readonly #config: Config;\n  readonly #clients: Map<\n    string,\n    {\n      wsID: string;\n      downstream: Subscription<Downstream>;\n      onAuthFailure: (() => void) | undefined;\n    }\n  >;\n  #userPushURL?: string | undefined;\n  #userPushHeaders?: Record<string, string> | undefined;\n\n  readonly #customMutations = getOrCreateCounter(\n    'mutation',\n    'custom',\n    'Number of custom mutations processed',\n  );\n  readonly #pushes = getOrCreateCounter(\n    'mutation',\n    'pushes',\n    'Number of pushes processed by the pusher',\n  );\n\n  constructor(\n    config: Config,\n    lc: LogContext,\n    pushURL: string[],\n    apiKey: string | undefined,\n    allowedClientHeaders: readonly string[] | undefined,\n    queue: Queue<PusherEntryOrStop>,\n  ) {\n    this.#pushURLs = pushURL;\n    this.#lc = lc.withContext('component', 'pusher');\n    this.#pushURLPatterns = pushURL.map(compileUrlPattern);\n    this.#apiKey = apiKey;\n    this.#allowedClientHeaders = allowedClientHeaders;\n    this.#queue = queue;\n    this.#config = config;\n    this.#clients = new Map();\n  }\n\n  get pushURLs() {\n    return this.#pushURLs;\n  }\n\n  get effectivePushURL(): string | undefined {\n    return this.#userPushURL ?? this.#pushURLs[0];\n  }\n\n  /**\n   * Returns a new downstream stream if the clientID,wsID pair has not been seen before.\n   * If a clientID already exists with a different wsID, that client's downstream is cancelled.\n   */\n  initConnection(\n    clientID: string,\n    wsID: string,\n    userPushURL: string | undefined,\n    userPushHeaders: Record<string, string> | undefined,\n    onAuthFailure?: () => void,\n  ) {\n    const existing = this.#clients.get(clientID);\n    if (existing && existing.wsID === wsID) {\n      // already initialized for this socket\n      throw new Error('Connection was already initialized');\n    }\n\n    // client is back on a new connection\n    if (existing) {\n      existing.downstream.cancel();\n    }\n\n    // Handle client group level URL parameters\n    if (this.#userPushURL === undefined) {\n      // First client in the group - store its URL and headers\n      this.#userPushURL = userPushURL;\n      this.#userPushHeaders = userPushHeaders;\n    } else {\n      // Validate that subsequent clients have compatible parameters\n      if (this.#userPushURL !== userPushURL) {\n        this.#lc.warn?.(\n          'Client provided different mutate parameters than client group',\n          {\n            clientID,\n            clientURL: userPushURL,\n            clientGroupURL: this.#userPushURL,\n          },\n        );\n      }\n    }\n\n    const downstream = Subscription.create<Downstream>({\n      cleanup: () => {\n        this.#clients.delete(clientID);\n      },\n    });\n    this.#clients.set(clientID, {wsID, downstream, onAuthFailure});\n    return downstream;\n  }\n\n  async run() {\n    for (;;) {\n      const task = await this.#queue.dequeue();\n      const rest = this.#queue.drain();\n      const [pushes, terminate] = combinePushes([task, ...rest]);\n      for (const push of pushes) {\n        const response = await this.#processPush(push);\n        await this.#fanOutResponses(response);\n      }\n\n      if (terminate) {\n        break;\n      }\n    }\n  }\n\n  /**\n   * 1. If the entire `push` fails, we send the error to relevant clients.\n   * 2. If the push succeeds, we look for any mutation failure that should cause the connection to terminate\n   *  and terminate the connection for those clients.\n   */\n  #fanOutResponses(response: PushResponse) {\n    const connectionTerminations: (() => void)[] = [];\n\n    // if the entire push failed, send that to the client.\n    if ('kind' in response || 'error' in response) {\n      this.#lc.warn?.(\n        'The server behind ZERO_MUTATE_URL returned a push error.',\n        response,\n      );\n      const groupedMutationIDs = groupBy(\n        response.mutationIDs ?? [],\n        m => m.clientID,\n      );\n      for (const [clientID, mutationIDs] of groupedMutationIDs) {\n        const client = this.#clients.get(clientID);\n        if (!client) {\n          continue;\n        }\n\n        // We do not resolve mutations on the client if the push fails\n        // as those mutations will be retried.\n        if ('error' in response) {\n          // This error code path will eventually be removed when we\n          // no longer support the legacy push error format.\n          const pushFailedBody: PushFailedBody =\n            response.error === 'http'\n              ? {\n                  kind: ErrorKind.PushFailed,\n                  origin: ErrorOrigin.ZeroCache,\n                  reason: ErrorReason.HTTP,\n                  status: response.status,\n                  bodyPreview: response.details,\n                  mutationIDs,\n                  message: `Fetch from API server returned non-OK status ${response.status}`,\n                }\n              : response.error === 'unsupportedPushVersion'\n                ? {\n                    kind: ErrorKind.PushFailed,\n                    origin: ErrorOrigin.Server,\n                    reason: ErrorReason.UnsupportedPushVersion,\n                    mutationIDs,\n                    message: `Unsupported push version`,\n                  }\n                : {\n                    kind: ErrorKind.PushFailed,\n                    origin: ErrorOrigin.Server,\n                    reason: ErrorReason.Internal,\n                    mutationIDs,\n                    message:\n                      response.error === 'zeroPusher'\n                        ? response.details\n                        : response.error === 'unsupportedSchemaVersion'\n                          ? 'Unsupported schema version'\n                          : 'An unknown error occurred while pushing to the API server',\n                  };\n\n          this.#failDownstream(client.downstream, pushFailedBody);\n          if (isPushAuthFailure(pushFailedBody)) {\n            this.#lc.debug?.('Auth failure detected in push response');\n            client.onAuthFailure?.();\n          }\n        } else if ('kind' in response) {\n          this.#failDownstream(client.downstream, response);\n          if (isPushAuthFailure(response)) {\n            this.#lc.debug?.('Auth failure detected in push response');\n            client.onAuthFailure?.();\n          }\n        } else {\n          unreachable(response);\n        }\n      }\n    } else {\n      // Look for mutations results that should cause us to terminate the connection\n      const groupedMutations = groupBy(response.mutations, m => m.id.clientID);\n      for (const [clientID, mutations] of groupedMutations) {\n        const client = this.#clients.get(clientID);\n        if (!client) {\n          continue;\n        }\n\n        let failure: PushFailedBody | undefined;\n        let i = 0;\n        for (; i < mutations.length; i++) {\n          const m = mutations[i];\n          if ('error' in m.result) {\n            this.#lc.warn?.(\n              'The server behind ZERO_MUTATE_URL returned a mutation error.',\n              m.result,\n            );\n          }\n          // This error code path will eventually be removed,\n          // keeping this for backwards compatibility, but the server\n          // should now return a PushFailedBody with the mutationIDs\n          if ('error' in m.result && m.result.error === 'oooMutation') {\n            failure = {\n              kind: ErrorKind.PushFailed,\n              origin: ErrorOrigin.Server,\n              reason: ErrorReason.OutOfOrderMutation,\n              message: 'mutation was out of order',\n              details: m.result.details,\n              mutationIDs: mutations.map(m => ({\n                clientID: m.id.clientID,\n                id: m.id.id,\n              })),\n            };\n            break;\n          }\n        }\n\n        if (failure && i < mutations.length - 1) {\n          this.#lc.warn?.(\n            'push-response contains mutations after a mutation which should fatal the connection',\n          );\n        }\n\n        if (failure) {\n          connectionTerminations.push(() =>\n            this.#failDownstream(client.downstream, failure),\n          );\n        }\n      }\n    }\n\n    connectionTerminations.forEach(cb => cb());\n  }\n\n  async #processPush(entry: PusherEntry): Promise<PushResponse> {\n    this.#customMutations.add(entry.push.mutations.length, {\n      clientGroupID: entry.push.clientGroupID,\n    });\n    this.#pushes.add(1, {\n      clientGroupID: entry.push.clientGroupID,\n    });\n\n    // Record custom mutations for telemetry\n    recordMutation('custom', entry.push.mutations.length);\n\n    const url =\n      this.#userPushURL ??\n      must(this.#pushURLs[0], 'ZERO_MUTATE_URL is not set');\n\n    this.#lc.debug?.(\n      'pushing to',\n      url,\n      'with',\n      entry.push.mutations.length,\n      'mutations',\n    );\n\n    let mutationIDs: MutationID[] = [];\n\n    try {\n      mutationIDs = entry.push.mutations.map(m => ({\n        id: m.id,\n        clientID: m.clientID,\n      }));\n\n      return await fetchFromAPIServer(\n        pushResponseSchema,\n        'push',\n        this.#lc,\n        url,\n        this.#pushURLPatterns,\n        {\n          appID: this.#config.app.id,\n          shardNum: this.#config.shard.num,\n        },\n        {\n          apiKey: this.#apiKey,\n          customHeaders: this.#userPushHeaders,\n          allowedClientHeaders: this.#allowedClientHeaders,\n          token: entry.auth,\n          cookie: entry.httpCookie,\n          origin: entry.origin,\n        },\n        entry.push,\n      );\n    } catch (e) {\n      if (isProtocolError(e) && e.errorBody.kind === ErrorKind.PushFailed) {\n        return {\n          ...e.errorBody,\n          mutationIDs,\n        } as const satisfies PushFailedBody;\n      }\n\n      return {\n        kind: ErrorKind.PushFailed,\n        origin: ErrorOrigin.ZeroCache,\n        reason: ErrorReason.Internal,\n        message: `Failed to push: ${getErrorMessage(e)}`,\n        mutationIDs,\n      } as const satisfies PushFailedBody;\n    }\n  }\n\n  #failDownstream(\n    downstream: Subscription<Downstream>,\n    errorBody: PushFailedBody,\n  ): void {\n    downstream.fail(new ProtocolErrorWithLevel(errorBody, 'warn'));\n  }\n}\n\nfunction isPushAuthFailure(errorBody: PushFailedBody): boolean {\n  return (\n    errorBody.reason === ErrorReason.HTTP &&\n    (errorBody.status === 401 || errorBody.status === 403)\n  );\n}\n\n/**\n * Pushes for different clientIDs could theoretically be interleaved.\n *\n * In order to do efficient batching to the user's API server,\n * we collect all pushes for the same clientID into a single push.\n */\nexport function combinePushes(\n  entries: readonly (PusherEntryOrStop | undefined)[],\n): [PusherEntry[], boolean] {\n  const pushesByClientID = new Map<string, PusherEntry[]>();\n\n  function collect() {\n    const ret: PusherEntry[] = [];\n    for (const entries of pushesByClientID.values()) {\n      const composite: PusherEntry = {\n        ...entries[0],\n        push: {\n          ...entries[0].push,\n          mutations: [],\n        },\n      };\n      ret.push(composite);\n      for (const entry of entries) {\n        assertAreCompatiblePushes(composite, entry);\n        composite.push.mutations.push(...entry.push.mutations);\n      }\n    }\n    return ret;\n  }\n\n  for (const entry of entries) {\n    if (entry === 'stop' || entry === undefined) {\n      return [collect(), true];\n    }\n\n    const {clientID} = entry;\n    const existing = pushesByClientID.get(clientID);\n    if (existing) {\n      existing.push(entry);\n    } else {\n      pushesByClientID.set(clientID, [entry]);\n    }\n  }\n\n  return [collect(), false] as const;\n}\n\n// These invariants should always be true for a given clientID.\n// If they are not, we have a bug in the code somewhere.\nfunction assertAreCompatiblePushes(left: PusherEntry, right: PusherEntry) {\n  assert(\n    left.clientID === right.clientID,\n    'clientID must be the same for all pushes',\n  );\n  assert(\n    left.auth === right.auth,\n    'auth must be the same for all pushes with the same clientID',\n  );\n  assert(\n    left.push.schemaVersion === right.push.schemaVersion,\n    'schemaVersion must be the same for all pushes with the same clientID',\n  );\n  assert(\n    left.push.pushVersion === right.push.pushVersion,\n    'pushVersion must be the same for all pushes with the same clientID',\n  );\n  assert(\n    left.httpCookie === right.httpCookie,\n    'httpCookie must be the same for all pushes with the same clientID',\n  );\n  assert(\n    left.origin === right.origin,\n    'origin must be the same for all pushes with the same clientID',\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoEA,IAAa,gBAAb,MAAsD;CACpD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA,YAAY;CACZ,aAAa;CAEb,YACE,WACA,YACA,IACA,eACA;AACA,QAAA,SAAe;AACf,QAAA,KAAW,GAAG,YAAY,aAAa,gBAAgB;AACvD,QAAA,kBAAwB,WAAW,IAAI,IAAI,kBAAkB;AAC7D,QAAA,QAAc,IAAI,OAAO;AACzB,QAAA,SAAe,IAAI,WACjB,WACA,IACA,WAAW,KACX,WAAW,QACX,WAAW,sBACX,MAAA,MACD;AACD,OAAK,KAAK;AACV,QAAA,aAAmB;;CAGrB,IAAI,UAA8B;AAChC,SAAO,MAAA,OAAa,SAAS;;CAG/B,eACE,UACA,MACA,aACA,iBACA,eACA;AACA,SAAO,MAAA,OAAa,eAClB,UACA,MACA,aACA,iBACA,cACD;;CAGH,YACE,UACA,MACA,MACA,YACA,QACsC;AACtC,MAAI,CAAC,MAAA,WAAiB,eACpB,cAAa,KAAA;AAEf,QAAA,MAAY,QAAQ;GAAC;GAAM;GAAM;GAAU;GAAY;GAAO,CAAC;AAE/D,SAAO,EACL,MAAM,MACP;;CAGH,MAAM,qBAAqB,QAAoB;EAC7C,MAAM,MAAM,MAAA,OAAa;AACzB,MAAI,CAAC,IAEH;EAGF,MAAM,cAAwB;GAC5B,eAAe,KAAK;GACpB,WAAW,CACT;IACE,MAAM;IACN,IAAI;IACJ,UAAU,OAAO;IACjB,MAAM;IACN,MAAM,CACJ;KACE,MAAM;KACN,eAAe,KAAK;KACpB,UAAU,OAAO;KACjB,gBAAgB,OAAO;KACxB,CACF;IACD,WAAW,KAAK,KAAK;IACtB,CACF;GACD,aAAa;GACb,WAAW,KAAK,KAAK;GACrB,WAAW,WAAW,KAAK,GAAG,GAAG,OAAO,SAAS,GAAG,OAAO;GAC5D;AAED,MAAI;AACF,SAAM,mBACJ,oBACA,QACA,MAAA,IACA,KACA,MAAA,iBACA;IAAC,OAAO,MAAA,OAAa,IAAI;IAAI,UAAU,MAAA,OAAa,MAAM;IAAI,EAC9D,EAAC,QAAQ,MAAA,WAAiB,QAAO,EACjC,YACD;WACM,GAAG;AACV,SAAA,GAAS,OAAO,mCAAmC,EACjD,OAAO,gBAAgB,EAAE,EAC1B,CAAC;;;CAIN,MAAM,sBAAsB,WAAqB;AAC/C,MAAI,UAAU,WAAW,EACvB;EAEF,MAAM,MAAM,MAAA,OAAa;AACzB,MAAI,CAAC,IAEH;EAGF,MAAM,cAAwB;GAC5B,eAAe,KAAK;GACpB,WAAW,CACT;IACE,MAAM;IACN,IAAI;IACJ,UAAU,UAAU;IACpB,MAAM;IACN,MAAM,CACJ;KACE,MAAM;KACN,eAAe,KAAK;KACpB;KACD,CACF;IACD,WAAW,KAAK,KAAK;IACtB,CACF;GACD,aAAa;GACb,WAAW,KAAK,KAAK;GACrB,WAAW,gBAAgB,KAAK,GAAG,GAAG,KAAK,KAAK;GACjD;AAED,MAAI;AACF,SAAM,mBACJ,oBACA,QACA,MAAA,IACA,KACA,MAAA,iBACA;IAAC,OAAO,MAAA,OAAa,IAAI;IAAI,UAAU,MAAA,OAAa,MAAM;IAAI,EAC9D,EAAC,QAAQ,MAAA,WAAiB,QAAO,EACjC,YACD;WACM,GAAG;AACV,SAAA,GAAS,OAAO,wCAAwC,EACtD,OAAO,gBAAgB,EAAE,EAC1B,CAAC;;;CAIN,MAAM;AACJ,SAAO,CAAC,MAAA,WAAiB,mCAAmC;AAC5D,IAAE,MAAA;;CAGJ,QAAQ;AACN,SAAO,CAAC,MAAA,WAAiB,mCAAmC;AAC5D,IAAE,MAAA;AACF,MAAI,MAAA,YAAkB,EACf,MAAK,MAAM;;CAIpB,UAAmB;AACjB,SAAO,MAAA,WAAiB;;CAG1B,MAAqB;AACnB,QAAA,UAAgB,MAAA,OAAa,KAAK;AAClC,SAAO,MAAA;;CAGT,OAAsB;AACpB,MAAI,MAAA,UACF,QAAO,KAAK,MAAA,SAAe,+BAA+B;AAE5D,QAAA,YAAkB;AAClB,QAAA,MAAY,QAAQ,OAAO;AAC3B,SAAO,KAAK,MAAA,SAAe,+BAA+B;;;;;;;AAiB9D,IAAM,aAAN,MAAiB;CACf;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAQA;CACA;CAEA,mBAA4B,mBAC1B,YACA,UACA,uCACD;CACD,UAAmB,mBACjB,YACA,UACA,2CACD;CAED,YACE,QACA,IACA,SACA,QACA,sBACA,OACA;AACA,QAAA,WAAiB;AACjB,QAAA,KAAW,GAAG,YAAY,aAAa,SAAS;AAChD,QAAA,kBAAwB,QAAQ,IAAI,kBAAkB;AACtD,QAAA,SAAe;AACf,QAAA,uBAA6B;AAC7B,QAAA,QAAc;AACd,QAAA,SAAe;AACf,QAAA,0BAAgB,IAAI,KAAK;;CAG3B,IAAI,WAAW;AACb,SAAO,MAAA;;CAGT,IAAI,mBAAuC;AACzC,SAAO,MAAA,eAAqB,MAAA,SAAe;;;;;;CAO7C,eACE,UACA,MACA,aACA,iBACA,eACA;EACA,MAAM,WAAW,MAAA,QAAc,IAAI,SAAS;AAC5C,MAAI,YAAY,SAAS,SAAS,KAEhC,OAAM,IAAI,MAAM,qCAAqC;AAIvD,MAAI,SACF,UAAS,WAAW,QAAQ;AAI9B,MAAI,MAAA,gBAAsB,KAAA,GAAW;AAEnC,SAAA,cAAoB;AACpB,SAAA,kBAAwB;aAGpB,MAAA,gBAAsB,YACxB,OAAA,GAAS,OACP,iEACA;GACE;GACA,WAAW;GACX,gBAAgB,MAAA;GACjB,CACF;EAIL,MAAM,aAAa,aAAa,OAAmB,EACjD,eAAe;AACb,SAAA,QAAc,OAAO,SAAS;KAEjC,CAAC;AACF,QAAA,QAAc,IAAI,UAAU;GAAC;GAAM;GAAY;GAAc,CAAC;AAC9D,SAAO;;CAGT,MAAM,MAAM;AACV,WAAS;GAGP,MAAM,CAAC,QAAQ,aAAa,cAAc,CAF7B,MAAM,MAAA,MAAY,SAAS,EAES,GADpC,MAAA,MAAY,OAAO,CACyB,CAAC;AAC1D,QAAK,MAAM,QAAQ,QAAQ;IACzB,MAAM,WAAW,MAAM,MAAA,YAAkB,KAAK;AAC9C,UAAM,MAAA,gBAAsB,SAAS;;AAGvC,OAAI,UACF;;;;;;;;CAUN,iBAAiB,UAAwB;EACvC,MAAM,yBAAyC,EAAE;AAGjD,MAAI,UAAU,YAAY,WAAW,UAAU;AAC7C,SAAA,GAAS,OACP,4DACA,SACD;GACD,MAAM,qBAAqB,QACzB,SAAS,eAAe,EAAE,GAC1B,MAAK,EAAE,SACR;AACD,QAAK,MAAM,CAAC,UAAU,gBAAgB,oBAAoB;IACxD,MAAM,SAAS,MAAA,QAAc,IAAI,SAAS;AAC1C,QAAI,CAAC,OACH;AAKF,QAAI,WAAW,UAAU;KAGvB,MAAM,iBACJ,SAAS,UAAU,SACf;MACE,MAAM;MACN,QAAQ;MACR,QAAQ;MACR,QAAQ,SAAS;MACjB,aAAa,SAAS;MACtB;MACA,SAAS,gDAAgD,SAAS;MACnE,GACD,SAAS,UAAU,2BACjB;MACE,MAAM;MACN,QAAQ;MACR,QAAQ;MACR;MACA,SAAS;MACV,GACD;MACE,MAAM;MACN,QAAQ;MACR,QAAQ;MACR;MACA,SACE,SAAS,UAAU,eACf,SAAS,UACT,SAAS,UAAU,6BACjB,+BACA;MACT;AAET,WAAA,eAAqB,OAAO,YAAY,eAAe;AACvD,SAAI,kBAAkB,eAAe,EAAE;AACrC,YAAA,GAAS,QAAQ,yCAAyC;AAC1D,aAAO,iBAAiB;;eAEjB,UAAU,UAAU;AAC7B,WAAA,eAAqB,OAAO,YAAY,SAAS;AACjD,SAAI,kBAAkB,SAAS,EAAE;AAC/B,YAAA,GAAS,QAAQ,yCAAyC;AAC1D,aAAO,iBAAiB;;UAG1B,aAAY,SAAS;;SAGpB;GAEL,MAAM,mBAAmB,QAAQ,SAAS,YAAW,MAAK,EAAE,GAAG,SAAS;AACxE,QAAK,MAAM,CAAC,UAAU,cAAc,kBAAkB;IACpD,MAAM,SAAS,MAAA,QAAc,IAAI,SAAS;AAC1C,QAAI,CAAC,OACH;IAGF,IAAI;IACJ,IAAI,IAAI;AACR,WAAO,IAAI,UAAU,QAAQ,KAAK;KAChC,MAAM,IAAI,UAAU;AACpB,SAAI,WAAW,EAAE,OACf,OAAA,GAAS,OACP,gEACA,EAAE,OACH;AAKH,SAAI,WAAW,EAAE,UAAU,EAAE,OAAO,UAAU,eAAe;AAC3D,gBAAU;OACR,MAAM;OACN,QAAQ;OACR,QAAQ;OACR,SAAS;OACT,SAAS,EAAE,OAAO;OAClB,aAAa,UAAU,KAAI,OAAM;QAC/B,UAAU,EAAE,GAAG;QACf,IAAI,EAAE,GAAG;QACV,EAAE;OACJ;AACD;;;AAIJ,QAAI,WAAW,IAAI,UAAU,SAAS,EACpC,OAAA,GAAS,OACP,sFACD;AAGH,QAAI,QACF,wBAAuB,WACrB,MAAA,eAAqB,OAAO,YAAY,QAAQ,CACjD;;;AAKP,yBAAuB,SAAQ,OAAM,IAAI,CAAC;;CAG5C,OAAA,YAAmB,OAA2C;AAC5D,QAAA,gBAAsB,IAAI,MAAM,KAAK,UAAU,QAAQ,EACrD,eAAe,MAAM,KAAK,eAC3B,CAAC;AACF,QAAA,OAAa,IAAI,GAAG,EAClB,eAAe,MAAM,KAAK,eAC3B,CAAC;AAGF,iBAAe,UAAU,MAAM,KAAK,UAAU,OAAO;EAErD,MAAM,MACJ,MAAA,eACA,KAAK,MAAA,SAAe,IAAI,6BAA6B;AAEvD,QAAA,GAAS,QACP,cACA,KACA,QACA,MAAM,KAAK,UAAU,QACrB,YACD;EAED,IAAI,cAA4B,EAAE;AAElC,MAAI;AACF,iBAAc,MAAM,KAAK,UAAU,KAAI,OAAM;IAC3C,IAAI,EAAE;IACN,UAAU,EAAE;IACb,EAAE;AAEH,UAAO,MAAM,mBACX,oBACA,QACA,MAAA,IACA,KACA,MAAA,iBACA;IACE,OAAO,MAAA,OAAa,IAAI;IACxB,UAAU,MAAA,OAAa,MAAM;IAC9B,EACD;IACE,QAAQ,MAAA;IACR,eAAe,MAAA;IACf,sBAAsB,MAAA;IACtB,OAAO,MAAM;IACb,QAAQ,MAAM;IACd,QAAQ,MAAM;IACf,EACD,MAAM,KACP;WACM,GAAG;AACV,OAAI,gBAAgB,EAAE,IAAI,EAAE,UAAU,SAAS,aAC7C,QAAO;IACL,GAAG,EAAE;IACL;IACD;AAGH,UAAO;IACL,MAAM;IACN,QAAQ;IACR,QAAQ;IACR,SAAS,mBAAmB,gBAAgB,EAAE;IAC9C;IACD;;;CAIL,gBACE,YACA,WACM;AACN,aAAW,KAAK,IAAI,uBAAuB,WAAW,OAAO,CAAC;;;AAIlE,SAAS,kBAAkB,WAAoC;AAC7D,QACE,UAAU,WAAW,WACpB,UAAU,WAAW,OAAO,UAAU,WAAW;;;;;;;;AAUtD,SAAgB,cACd,SAC0B;CAC1B,MAAM,mCAAmB,IAAI,KAA4B;CAEzD,SAAS,UAAU;EACjB,MAAM,MAAqB,EAAE;AAC7B,OAAK,MAAM,WAAW,iBAAiB,QAAQ,EAAE;GAC/C,MAAM,YAAyB;IAC7B,GAAG,QAAQ;IACX,MAAM;KACJ,GAAG,QAAQ,GAAG;KACd,WAAW,EAAE;KACd;IACF;AACD,OAAI,KAAK,UAAU;AACnB,QAAK,MAAM,SAAS,SAAS;AAC3B,8BAA0B,WAAW,MAAM;AAC3C,cAAU,KAAK,UAAU,KAAK,GAAG,MAAM,KAAK,UAAU;;;AAG1D,SAAO;;AAGT,MAAK,MAAM,SAAS,SAAS;AAC3B,MAAI,UAAU,UAAU,UAAU,KAAA,EAChC,QAAO,CAAC,SAAS,EAAE,KAAK;EAG1B,MAAM,EAAC,aAAY;EACnB,MAAM,WAAW,iBAAiB,IAAI,SAAS;AAC/C,MAAI,SACF,UAAS,KAAK,MAAM;MAEpB,kBAAiB,IAAI,UAAU,CAAC,MAAM,CAAC;;AAI3C,QAAO,CAAC,SAAS,EAAE,MAAM;;AAK3B,SAAS,0BAA0B,MAAmB,OAAoB;AACxE,QACE,KAAK,aAAa,MAAM,UACxB,2CACD;AACD,QACE,KAAK,SAAS,MAAM,MACpB,8DACD;AACD,QACE,KAAK,KAAK,kBAAkB,MAAM,KAAK,eACvC,uEACD;AACD,QACE,KAAK,KAAK,gBAAgB,MAAM,KAAK,aACrC,qEACD;AACD,QACE,KAAK,eAAe,MAAM,YAC1B,oEACD;AACD,QACE,KAAK,WAAW,MAAM,QACtB,gEACD"}
+{"version":3,"file":"pusher.js","names":["#contextManager","#pusher","#queue","#config","#lc","#isStopped","#refCount","#stopped","#clients","#customMutations","#pushes","#processPush","#fanOutResponses","#failDownstream"],"sources":["../../../../../../zero-cache/src/services/mutagen/pusher.ts"],"sourcesContent":["import {ROOT_CONTEXT, context, propagation} from '@opentelemetry/api';\nimport type {LogContext} from '@rocicorp/logger';\nimport {groupBy} from '../../../../shared/src/arrays.ts';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport {getErrorMessage} from '../../../../shared/src/error.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Queue} from '../../../../shared/src/queue.ts';\nimport type {Downstream} from '../../../../zero-protocol/src/down.ts';\nimport {ErrorKind} from '../../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../../zero-protocol/src/error-origin.ts';\nimport {ErrorReason} from '../../../../zero-protocol/src/error-reason.ts';\nimport {\n  isProtocolError,\n  type PushFailedBody,\n} from '../../../../zero-protocol/src/error.ts';\nimport * as MutationType from '../../../../zero-protocol/src/mutation-type-enum.ts';\nimport {\n  CLEANUP_RESULTS_MUTATION_NAME,\n  pushResponseSchema,\n  type MutationID,\n  type PushBody,\n  type PushResponse,\n} from '../../../../zero-protocol/src/push.ts';\nimport {authEquals, isAuthErrorBody} from '../../auth/auth.ts';\nimport {type ZeroConfig} from '../../config/zero-config.ts';\nimport {fetchFromAPIServer} from '../../custom/fetch.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {recordMutation} from '../../server/anonymous-otel-start.ts';\nimport {ProtocolErrorWithLevel} from '../../types/error-with-level.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {HandlerResult, StreamResult} from '../../workers/connection.ts';\nimport type {RefCountedService, Service} from '../service.ts';\nimport type {\n  ConnectionContext,\n  ConnectionContextManager,\n  ConnectionSelector,\n} from '../view-syncer/connection-context-manager.ts';\n\nexport interface Pusher extends RefCountedService {\n  initConnection(selector: ConnectionSelector): Source<Downstream>;\n  enqueuePush(selector: ConnectionSelector, push: PushBody): HandlerResult;\n  ackMutationResponses(\n    requester: ConnectionSelector,\n    upToID: MutationID,\n  ): Promise<void>;\n  deleteClientMutations(\n    requester: ConnectionSelector,\n    clientIDs: string[],\n  ): Promise<void>;\n}\n\ntype Config = Pick<ZeroConfig, 'app' | 'shard'>;\n\n/**\n * Receives push messages from zero-client and forwards\n * them the the user's API server.\n *\n * If the user's API server is taking too long to process\n * the push, the PusherService will add the push to a queue\n * and send pushes in bulk the next time the user's API server\n * is available.\n *\n * - One PusherService exists per client group.\n * - Mutations for a given client are always sent in-order\n * - Mutations for different clients in the same group may be interleaved\n */\nexport class PusherService implements Service, Pusher {\n  readonly id: string;\n  readonly #contextManager: ConnectionContextManager;\n  readonly #pusher: PushWorker;\n  readonly #queue: Queue<PusherEntryOrStop>;\n  readonly #config: Config;\n  readonly #lc: LogContext;\n  #stopped: Promise<void> | undefined;\n  #refCount = 0;\n  #isStopped = false;\n\n  constructor(\n    appConfig: Config,\n    lc: LogContext,\n    clientGroupID: string,\n    contextManager: ConnectionContextManager,\n  ) {\n    this.#contextManager = contextManager;\n    this.#config = appConfig;\n    this.#lc = lc.withContext('component', 'pusherService');\n    this.#queue = new Queue();\n    this.#pusher = new PushWorker(\n      appConfig,\n      lc,\n      this.#contextManager,\n      this.#queue,\n    );\n    this.id = clientGroupID;\n  }\n\n  initConnection(selector: ConnectionSelector) {\n    return this.#pusher.initConnection(selector);\n  }\n\n  enqueuePush(\n    selector: ConnectionSelector,\n    push: PushBody,\n  ): Exclude<HandlerResult, StreamResult> {\n    this.#pusher.enqueuePush(\n      this.#contextManager.mustGetConnectionContext(selector),\n      push,\n    );\n\n    return {\n      type: 'ok',\n    };\n  }\n\n  async ackMutationResponses(\n    requester: ConnectionSelector,\n    upToID: MutationID,\n  ): Promise<void> {\n    const ctx = this.#contextManager.getConnectionContext(requester);\n    if (!ctx?.pushContext?.url) {\n      // No push URL configured, skip cleanup\n      return;\n    }\n\n    const cleanupBody: PushBody = {\n      clientGroupID: this.id,\n      mutations: [\n        {\n          type: MutationType.Custom,\n          id: 0, // Not tracked - this is fire-and-forget\n          clientID: upToID.clientID,\n          name: CLEANUP_RESULTS_MUTATION_NAME,\n          args: [\n            {\n              type: 'single',\n              clientGroupID: this.id,\n              clientID: upToID.clientID,\n              upToMutationID: upToID.id,\n            },\n          ],\n          timestamp: Date.now(),\n        },\n      ],\n      pushVersion: 1,\n      timestamp: Date.now(),\n      requestID: `cleanup-${this.id}-${upToID.clientID}-${upToID.id}`,\n    };\n\n    try {\n      await fetchFromAPIServer(\n        pushResponseSchema,\n        'push',\n        this.#lc,\n        ctx,\n        {appID: this.#config.app.id, shardNum: this.#config.shard.num},\n        cleanupBody,\n      );\n    } catch (e) {\n      this.#lc.warn?.('Failed to send cleanup mutation', {\n        error: getErrorMessage(e),\n      });\n    }\n  }\n\n  /**\n   * Bulk cleanup is routed through the requester's push context.\n   *\n   * This assumes the client group shares a compatible push endpoint/auth\n   * context.\n   */\n  async deleteClientMutations(\n    requester: ConnectionSelector,\n    clientIDs: string[],\n  ): Promise<void> {\n    if (clientIDs.length === 0) {\n      return;\n    }\n\n    const ctx = this.#contextManager.getConnectionContext(requester);\n    if (!ctx?.pushContext?.url) {\n      // No push URL configured, skip cleanup\n      return;\n    }\n\n    const cleanupBody: PushBody = {\n      clientGroupID: this.id,\n      mutations: [\n        {\n          type: MutationType.Custom,\n          id: 0, // Not tracked - this is fire-and-forget\n          clientID: clientIDs[0], // Use first client as sender\n          name: CLEANUP_RESULTS_MUTATION_NAME,\n          args: [\n            {\n              type: 'bulk',\n              clientGroupID: this.id,\n              clientIDs,\n            },\n          ],\n          timestamp: Date.now(),\n        },\n      ],\n      pushVersion: 1,\n      timestamp: Date.now(),\n      requestID: `cleanup-bulk-${this.id}-${Date.now()}`,\n    };\n\n    try {\n      await fetchFromAPIServer(\n        pushResponseSchema,\n        'push',\n        this.#lc,\n        ctx,\n        {appID: this.#config.app.id, shardNum: this.#config.shard.num},\n        cleanupBody,\n      );\n    } catch (e) {\n      this.#lc.warn?.('Failed to send bulk cleanup mutation', {\n        error: getErrorMessage(e),\n      });\n    }\n  }\n\n  ref() {\n    assert(!this.#isStopped, 'PusherService is already stopped');\n    ++this.#refCount;\n  }\n\n  unref() {\n    assert(!this.#isStopped, 'PusherService is already stopped');\n    --this.#refCount;\n    if (this.#refCount <= 0) {\n      void this.stop();\n    }\n  }\n\n  hasRefs(): boolean {\n    return this.#refCount > 0;\n  }\n\n  run(): Promise<void> {\n    this.#stopped = this.#pusher.run();\n    return this.#stopped;\n  }\n\n  stop(): Promise<void> {\n    if (this.#isStopped) {\n      return must(this.#stopped, 'Stop was called before `run`');\n    }\n    this.#isStopped = true;\n    this.#queue.enqueue('stop');\n    return must(this.#stopped, 'Stop was called before `run`');\n  }\n}\n\ntype PusherEntry = {\n  push: PushBody;\n  context: ConnectionContext;\n};\ntype PusherEntryOrStop = PusherEntry | 'stop';\n\n/**\n * Awaits items in the queue then drains and sends them all\n * to the user's API server.\n */\nclass PushWorker {\n  readonly #contextManager: ConnectionContextManager;\n  readonly #queue: Queue<PusherEntryOrStop>;\n  readonly #lc: LogContext;\n  readonly #config: Config;\n  readonly #clients: Map<\n    string,\n    {wsID: string; downstream: Subscription<Downstream>}\n  >;\n\n  readonly #customMutations = getOrCreateCounter(\n    'mutation',\n    'custom',\n    'Number of custom mutations processed',\n  );\n  readonly #pushes = getOrCreateCounter(\n    'mutation',\n    'pushes',\n    'Number of pushes processed by the pusher',\n  );\n\n  constructor(\n    config: Config,\n    lc: LogContext,\n    contextManager: ConnectionContextManager,\n    queue: Queue<PusherEntryOrStop>,\n  ) {\n    this.#lc = lc.withContext('component', 'pusher');\n    this.#contextManager = contextManager;\n    this.#queue = queue;\n    this.#config = config;\n    this.#clients = new Map();\n  }\n\n  /**\n   * Returns a new downstream stream if the clientID,wsID pair has not been seen before.\n   * If a clientID already exists with a different wsID, that client's downstream is cancelled.\n   */\n  initConnection(selector: ConnectionSelector) {\n    const existing = this.#clients.get(selector.clientID);\n    if (existing && existing.wsID === selector.wsID) {\n      // already initialized for this socket\n      throw new Error('Connection was already initialized');\n    }\n\n    // client is back on a new connection\n    if (existing) {\n      existing.downstream.cancel();\n    }\n\n    const downstream = Subscription.create<Downstream>({\n      cleanup: () => {\n        this.#clients.delete(selector.clientID);\n      },\n    });\n    this.#clients.set(selector.clientID, {\n      wsID: selector.wsID,\n      downstream,\n    });\n    return downstream;\n  }\n\n  enqueuePush(context: ConnectionContext, push: PushBody) {\n    this.#queue.enqueue({\n      push,\n      context,\n    });\n  }\n\n  async run() {\n    for (;;) {\n      const task = await this.#queue.dequeue();\n      const rest = this.#queue.drain();\n      const [pushes, terminate] = combinePushes([task, ...rest]);\n      for (const push of pushes) {\n        const parentContext = push.push.traceparent\n          ? propagation.extract(ROOT_CONTEXT, {\n              traceparent: push.push.traceparent,\n            })\n          : context.active();\n        const response = await context.with(parentContext, () =>\n          this.#processPush(push),\n        );\n        await this.#fanOutResponses(response);\n      }\n\n      if (terminate) {\n        break;\n      }\n    }\n  }\n\n  /**\n   * 1. If the entire `push` fails, we send the error to relevant clients.\n   * 2. If the push succeeds, we look for any mutation failure that should cause the connection to terminate\n   *  and terminate the connection for those clients.\n   */\n  #fanOutResponses(response: PushResponse) {\n    const connectionTerminations: (() => void)[] = [];\n\n    // if the entire push failed, send that to the client.\n    if ('kind' in response || 'error' in response) {\n      this.#lc.warn?.(\n        'The server behind ZERO_MUTATE_URL returned a push error.',\n        response,\n      );\n      // TODO(0xcadams): Fanout is keyed only by clientID here. If a response arrives\n      // after reconnect or re-auth, `#clients.get(clientID)` may point at a\n      // newer wsID/revision and fail the replacement downstream instead.\n      const groupedMutationIDs = groupBy(\n        response.mutationIDs ?? [],\n        m => m.clientID,\n      );\n      for (const [clientID, mutationIDs] of groupedMutationIDs) {\n        const client = this.#clients.get(clientID);\n        if (!client) {\n          continue;\n        }\n\n        // We do not resolve mutations on the client if the push fails\n        // as those mutations will be retried.\n        if ('error' in response) {\n          // This error code path will eventually be removed when we\n          // no longer support the legacy push error format.\n          const pushFailedBody: PushFailedBody =\n            response.error === 'http'\n              ? {\n                  kind: ErrorKind.PushFailed,\n                  origin: ErrorOrigin.ZeroCache,\n                  reason: ErrorReason.HTTP,\n                  status: response.status,\n                  bodyPreview: response.details,\n                  mutationIDs,\n                  message: `Fetch from API server returned non-OK status ${response.status}`,\n                }\n              : response.error === 'unsupportedPushVersion'\n                ? {\n                    kind: ErrorKind.PushFailed,\n                    origin: ErrorOrigin.Server,\n                    reason: ErrorReason.UnsupportedPushVersion,\n                    mutationIDs,\n                    message: `Unsupported push version`,\n                  }\n                : {\n                    kind: ErrorKind.PushFailed,\n                    origin: ErrorOrigin.Server,\n                    reason: ErrorReason.Internal,\n                    mutationIDs,\n                    message:\n                      response.error === 'zeroPusher'\n                        ? response.details\n                        : response.error === 'unsupportedSchemaVersion'\n                          ? 'Unsupported schema version'\n                          : 'An unknown error occurred while pushing to the API server',\n                  };\n\n          this.#failDownstream(client.downstream, pushFailedBody);\n        } else if ('kind' in response) {\n          this.#failDownstream(client.downstream, response);\n        } else {\n          unreachable(response);\n        }\n      }\n    } else {\n      // Look for mutations results that should cause us to terminate the connection\n      // TODO(0xcadams): Same stale-routing issue as above: fatal mutation results are\n      // still mapped to the current downstream by clientID only.\n      const groupedMutations = groupBy(response.mutations, m => m.id.clientID);\n      for (const [clientID, mutations] of groupedMutations) {\n        const client = this.#clients.get(clientID);\n        if (!client) {\n          continue;\n        }\n\n        let failure: PushFailedBody | undefined;\n        let i = 0;\n        for (; i < mutations.length; i++) {\n          const m = mutations[i];\n          if ('error' in m.result) {\n            this.#lc.warn?.(\n              'The server behind ZERO_MUTATE_URL returned a mutation error.',\n              m.result,\n            );\n          }\n          // This error code path will eventually be removed,\n          // keeping this for backwards compatibility, but the server\n          // should now return a PushFailedBody with the mutationIDs\n          if ('error' in m.result && m.result.error === 'oooMutation') {\n            failure = {\n              kind: ErrorKind.PushFailed,\n              origin: ErrorOrigin.Server,\n              reason: ErrorReason.OutOfOrderMutation,\n              message: 'mutation was out of order',\n              details: m.result.details,\n              mutationIDs: mutations.map(m => ({\n                clientID: m.id.clientID,\n                id: m.id.id,\n              })),\n            };\n            break;\n          }\n        }\n\n        if (failure && i < mutations.length - 1) {\n          this.#lc.warn?.(\n            'push-response contains mutations after a mutation which should fatal the connection',\n          );\n        }\n\n        if (failure) {\n          connectionTerminations.push(() =>\n            this.#failDownstream(client.downstream, failure),\n          );\n        }\n      }\n    }\n\n    connectionTerminations.forEach(cb => cb());\n  }\n\n  async #processPush(entry: PusherEntry): Promise<PushResponse> {\n    this.#customMutations.add(entry.push.mutations.length, {\n      clientGroupID: entry.push.clientGroupID,\n    });\n    this.#pushes.add(1, {\n      clientGroupID: entry.push.clientGroupID,\n    });\n\n    // Record custom mutations for telemetry\n    recordMutation('custom', entry.push.mutations.length);\n\n    const url = must(\n      entry.context.pushContext.url,\n      'ZERO_MUTATE_URL is not set',\n    );\n\n    this.#lc.debug?.(\n      'pushing to',\n      url,\n      'with',\n      entry.push.mutations.length,\n      'mutations',\n    );\n\n    let mutationIDs: MutationID[] = [];\n\n    try {\n      mutationIDs = entry.push.mutations.map(m => ({\n        id: m.id,\n        clientID: m.clientID,\n      }));\n\n      const response = await fetchFromAPIServer(\n        pushResponseSchema,\n        'push',\n        this.#lc,\n        entry.context,\n        {\n          appID: this.#config.app.id,\n          shardNum: this.#config.shard.num,\n        },\n        entry.push,\n      );\n      if ('kind' in response || 'error' in response) {\n        if (isAuthErrorBody(response)) {\n          this.#lc.warn?.('Push auth failed; invalidating connection', {\n            clientID: entry.context.clientID,\n            response: 'kind' in response ? response.message : undefined,\n          });\n          this.#contextManager.failConnection(\n            entry.context,\n            entry.context.revision,\n          );\n        }\n        return response;\n      }\n      // A successful push also validates this connection's current auth snapshot.\n      // That lets later shared work reuse it without trusting stale credentials.\n      this.#contextManager.validateConnection(\n        entry.context,\n        entry.context.revision,\n      );\n      return response;\n    } catch (e) {\n      if (isProtocolError(e) && e.errorBody.kind === ErrorKind.PushFailed) {\n        const response = {\n          ...e.errorBody,\n          mutationIDs,\n        } as const satisfies PushFailedBody;\n        if (isAuthErrorBody(response)) {\n          this.#lc.warn?.('Push auth failed; invalidating connection', {\n            clientID: entry.context.clientID,\n            response: 'kind' in response ? response.message : undefined,\n          });\n          this.#contextManager.failConnection(\n            entry.context,\n            entry.context.revision,\n          );\n        }\n        return response;\n      }\n\n      return {\n        kind: ErrorKind.PushFailed,\n        origin: ErrorOrigin.ZeroCache,\n        reason: ErrorReason.Internal,\n        message: `Failed to push: ${getErrorMessage(e)}`,\n        mutationIDs,\n      } as const satisfies PushFailedBody;\n    }\n  }\n\n  #failDownstream(\n    downstream: Subscription<Downstream>,\n    errorBody: PushFailedBody,\n  ): void {\n    downstream.fail(new ProtocolErrorWithLevel(errorBody, 'warn'));\n  }\n}\n\n/**\n * Pushes for different clients, sockets, or auth revisions could be interleaved.\n *\n * In order to batch safely, we only combine pushes from the same\n * clientID/wsID/revision snapshot.\n */\nexport function combinePushes(\n  entries: readonly (PusherEntryOrStop | undefined)[],\n): [PusherEntry[], boolean] {\n  const pushesByConnection = new Map<string, PusherEntry[]>();\n\n  function collect() {\n    const ret: PusherEntry[] = [];\n    for (const entries of pushesByConnection.values()) {\n      const composite: PusherEntry = {\n        ...entries[0],\n        push: {\n          ...entries[0].push,\n          mutations: [],\n        },\n      };\n      ret.push(composite);\n      for (const entry of entries) {\n        assertAreCompatiblePushes(composite, entry);\n        composite.push.mutations.push(...entry.push.mutations);\n      }\n    }\n    return ret;\n  }\n\n  for (const entry of entries) {\n    if (entry === 'stop' || entry === undefined) {\n      return [collect(), true];\n    }\n\n    const key = `${entry.context.clientID}:${entry.context.wsID}:${entry.context.revision}`;\n    const existing = pushesByConnection.get(key);\n    if (existing) {\n      existing.push(entry);\n    } else {\n      pushesByConnection.set(key, [entry]);\n    }\n  }\n\n  return [collect(), false] as const;\n}\n\n// These invariants should always be true for a given clientID.\n// If they are not, we have a bug in the code somewhere.\nfunction assertAreCompatiblePushes(left: PusherEntry, right: PusherEntry) {\n  assert(\n    left.context.clientID === right.context.clientID,\n    'clientID must be the same for all pushes',\n  );\n  assert(\n    left.context.wsID === right.context.wsID,\n    'wsID must be the same for all pushes',\n  );\n  assert(\n    left.context.revision === right.context.revision,\n    'revision must be the same for all pushes',\n  );\n  assert(\n    authEquals(left.context.auth, right.context.auth),\n    'auth must be the same for all pushes with the same clientID',\n  );\n  assert(\n    left.push.schemaVersion === right.push.schemaVersion,\n    'schemaVersion must be the same for all pushes with the same clientID',\n  );\n  assert(\n    left.push.pushVersion === right.push.pushVersion,\n    'pushVersion must be the same for all pushes with the same clientID',\n  );\n  assert(\n    left.context.pushContext.headerOptions.cookie ===\n      right.context.pushContext.headerOptions.cookie,\n    'httpCookie must be the same for all pushes with the same clientID',\n  );\n  assert(\n    left.context.pushContext.headerOptions.origin ===\n      right.context.pushContext.headerOptions.origin,\n    'origin must be the same for all pushes with the same clientID',\n  );\n  assert(\n    left.context.userID === right.context.userID,\n    'userID must be the same for all pushes with the same clientID',\n  );\n  assert(\n    left.context.pushContext.url === right.context.pushContext.url,\n    'userPushURL must be the same for all pushes with the same clientID',\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAmEA,IAAa,gBAAb,MAAsD;CACpD;CACA;CACA;CACA;CACA;CACA;CACA;CACA,YAAY;CACZ,aAAa;CAEb,YACE,WACA,IACA,eACA,gBACA;AACA,QAAA,iBAAuB;AACvB,QAAA,SAAe;AACf,QAAA,KAAW,GAAG,YAAY,aAAa,gBAAgB;AACvD,QAAA,QAAc,IAAI,OAAO;AACzB,QAAA,SAAe,IAAI,WACjB,WACA,IACA,MAAA,gBACA,MAAA,MACD;AACD,OAAK,KAAK;;CAGZ,eAAe,UAA8B;AAC3C,SAAO,MAAA,OAAa,eAAe,SAAS;;CAG9C,YACE,UACA,MACsC;AACtC,QAAA,OAAa,YACX,MAAA,eAAqB,yBAAyB,SAAS,EACvD,KACD;AAED,SAAO,EACL,MAAM,MACP;;CAGH,MAAM,qBACJ,WACA,QACe;EACf,MAAM,MAAM,MAAA,eAAqB,qBAAqB,UAAU;AAChE,MAAI,CAAC,KAAK,aAAa,IAErB;EAGF,MAAM,cAAwB;GAC5B,eAAe,KAAK;GACpB,WAAW,CACT;IACE,MAAM;IACN,IAAI;IACJ,UAAU,OAAO;IACjB,MAAM;IACN,MAAM,CACJ;KACE,MAAM;KACN,eAAe,KAAK;KACpB,UAAU,OAAO;KACjB,gBAAgB,OAAO;KACxB,CACF;IACD,WAAW,KAAK,KAAK;IACtB,CACF;GACD,aAAa;GACb,WAAW,KAAK,KAAK;GACrB,WAAW,WAAW,KAAK,GAAG,GAAG,OAAO,SAAS,GAAG,OAAO;GAC5D;AAED,MAAI;AACF,SAAM,mBACJ,oBACA,QACA,MAAA,IACA,KACA;IAAC,OAAO,MAAA,OAAa,IAAI;IAAI,UAAU,MAAA,OAAa,MAAM;IAAI,EAC9D,YACD;WACM,GAAG;AACV,SAAA,GAAS,OAAO,mCAAmC,EACjD,OAAO,gBAAgB,EAAE,EAC1B,CAAC;;;;;;;;;CAUN,MAAM,sBACJ,WACA,WACe;AACf,MAAI,UAAU,WAAW,EACvB;EAGF,MAAM,MAAM,MAAA,eAAqB,qBAAqB,UAAU;AAChE,MAAI,CAAC,KAAK,aAAa,IAErB;EAGF,MAAM,cAAwB;GAC5B,eAAe,KAAK;GACpB,WAAW,CACT;IACE,MAAM;IACN,IAAI;IACJ,UAAU,UAAU;IACpB,MAAM;IACN,MAAM,CACJ;KACE,MAAM;KACN,eAAe,KAAK;KACpB;KACD,CACF;IACD,WAAW,KAAK,KAAK;IACtB,CACF;GACD,aAAa;GACb,WAAW,KAAK,KAAK;GACrB,WAAW,gBAAgB,KAAK,GAAG,GAAG,KAAK,KAAK;GACjD;AAED,MAAI;AACF,SAAM,mBACJ,oBACA,QACA,MAAA,IACA,KACA;IAAC,OAAO,MAAA,OAAa,IAAI;IAAI,UAAU,MAAA,OAAa,MAAM;IAAI,EAC9D,YACD;WACM,GAAG;AACV,SAAA,GAAS,OAAO,wCAAwC,EACtD,OAAO,gBAAgB,EAAE,EAC1B,CAAC;;;CAIN,MAAM;AACJ,SAAO,CAAC,MAAA,WAAiB,mCAAmC;AAC5D,IAAE,MAAA;;CAGJ,QAAQ;AACN,SAAO,CAAC,MAAA,WAAiB,mCAAmC;AAC5D,IAAE,MAAA;AACF,MAAI,MAAA,YAAkB,EACf,MAAK,MAAM;;CAIpB,UAAmB;AACjB,SAAO,MAAA,WAAiB;;CAG1B,MAAqB;AACnB,QAAA,UAAgB,MAAA,OAAa,KAAK;AAClC,SAAO,MAAA;;CAGT,OAAsB;AACpB,MAAI,MAAA,UACF,QAAO,KAAK,MAAA,SAAe,+BAA+B;AAE5D,QAAA,YAAkB;AAClB,QAAA,MAAY,QAAQ,OAAO;AAC3B,SAAO,KAAK,MAAA,SAAe,+BAA+B;;;;;;;AAc9D,IAAM,aAAN,MAAiB;CACf;CACA;CACA;CACA;CACA;CAKA,mBAA4B,mBAC1B,YACA,UACA,uCACD;CACD,UAAmB,mBACjB,YACA,UACA,2CACD;CAED,YACE,QACA,IACA,gBACA,OACA;AACA,QAAA,KAAW,GAAG,YAAY,aAAa,SAAS;AAChD,QAAA,iBAAuB;AACvB,QAAA,QAAc;AACd,QAAA,SAAe;AACf,QAAA,0BAAgB,IAAI,KAAK;;;;;;CAO3B,eAAe,UAA8B;EAC3C,MAAM,WAAW,MAAA,QAAc,IAAI,SAAS,SAAS;AACrD,MAAI,YAAY,SAAS,SAAS,SAAS,KAEzC,OAAM,IAAI,MAAM,qCAAqC;AAIvD,MAAI,SACF,UAAS,WAAW,QAAQ;EAG9B,MAAM,aAAa,aAAa,OAAmB,EACjD,eAAe;AACb,SAAA,QAAc,OAAO,SAAS,SAAS;KAE1C,CAAC;AACF,QAAA,QAAc,IAAI,SAAS,UAAU;GACnC,MAAM,SAAS;GACf;GACD,CAAC;AACF,SAAO;;CAGT,YAAY,SAA4B,MAAgB;AACtD,QAAA,MAAY,QAAQ;GAClB;GACA;GACD,CAAC;;CAGJ,MAAM,MAAM;AACV,WAAS;GAGP,MAAM,CAAC,QAAQ,aAAa,cAAc,CAF7B,MAAM,MAAA,MAAY,SAAS,EAES,GADpC,MAAA,MAAY,OAAO,CACyB,CAAC;AAC1D,QAAK,MAAM,QAAQ,QAAQ;IACzB,MAAM,gBAAgB,KAAK,KAAK,cAC5B,YAAY,QAAQ,cAAc,EAChC,aAAa,KAAK,KAAK,aACxB,CAAC,GACF,QAAQ,QAAQ;IACpB,MAAM,WAAW,MAAM,QAAQ,KAAK,qBAClC,MAAA,YAAkB,KAAK,CACxB;AACD,UAAM,MAAA,gBAAsB,SAAS;;AAGvC,OAAI,UACF;;;;;;;;CAUN,iBAAiB,UAAwB;EACvC,MAAM,yBAAyC,EAAE;AAGjD,MAAI,UAAU,YAAY,WAAW,UAAU;AAC7C,SAAA,GAAS,OACP,4DACA,SACD;GAID,MAAM,qBAAqB,QACzB,SAAS,eAAe,EAAE,GAC1B,MAAK,EAAE,SACR;AACD,QAAK,MAAM,CAAC,UAAU,gBAAgB,oBAAoB;IACxD,MAAM,SAAS,MAAA,QAAc,IAAI,SAAS;AAC1C,QAAI,CAAC,OACH;AAKF,QAAI,WAAW,UAAU;KAGvB,MAAM,iBACJ,SAAS,UAAU,SACf;MACE,MAAM;MACN,QAAQ;MACR,QAAQ;MACR,QAAQ,SAAS;MACjB,aAAa,SAAS;MACtB;MACA,SAAS,gDAAgD,SAAS;MACnE,GACD,SAAS,UAAU,2BACjB;MACE,MAAM;MACN,QAAQ;MACR,QAAQ;MACR;MACA,SAAS;MACV,GACD;MACE,MAAM;MACN,QAAQ;MACR,QAAQ;MACR;MACA,SACE,SAAS,UAAU,eACf,SAAS,UACT,SAAS,UAAU,6BACjB,+BACA;MACT;AAET,WAAA,eAAqB,OAAO,YAAY,eAAe;eAC9C,UAAU,SACnB,OAAA,eAAqB,OAAO,YAAY,SAAS;QAEjD,aAAY,SAAS;;SAGpB;GAIL,MAAM,mBAAmB,QAAQ,SAAS,YAAW,MAAK,EAAE,GAAG,SAAS;AACxE,QAAK,MAAM,CAAC,UAAU,cAAc,kBAAkB;IACpD,MAAM,SAAS,MAAA,QAAc,IAAI,SAAS;AAC1C,QAAI,CAAC,OACH;IAGF,IAAI;IACJ,IAAI,IAAI;AACR,WAAO,IAAI,UAAU,QAAQ,KAAK;KAChC,MAAM,IAAI,UAAU;AACpB,SAAI,WAAW,EAAE,OACf,OAAA,GAAS,OACP,gEACA,EAAE,OACH;AAKH,SAAI,WAAW,EAAE,UAAU,EAAE,OAAO,UAAU,eAAe;AAC3D,gBAAU;OACR,MAAM;OACN,QAAQ;OACR,QAAQ;OACR,SAAS;OACT,SAAS,EAAE,OAAO;OAClB,aAAa,UAAU,KAAI,OAAM;QAC/B,UAAU,EAAE,GAAG;QACf,IAAI,EAAE,GAAG;QACV,EAAE;OACJ;AACD;;;AAIJ,QAAI,WAAW,IAAI,UAAU,SAAS,EACpC,OAAA,GAAS,OACP,sFACD;AAGH,QAAI,QACF,wBAAuB,WACrB,MAAA,eAAqB,OAAO,YAAY,QAAQ,CACjD;;;AAKP,yBAAuB,SAAQ,OAAM,IAAI,CAAC;;CAG5C,OAAA,YAAmB,OAA2C;AAC5D,QAAA,gBAAsB,IAAI,MAAM,KAAK,UAAU,QAAQ,EACrD,eAAe,MAAM,KAAK,eAC3B,CAAC;AACF,QAAA,OAAa,IAAI,GAAG,EAClB,eAAe,MAAM,KAAK,eAC3B,CAAC;AAGF,iBAAe,UAAU,MAAM,KAAK,UAAU,OAAO;EAErD,MAAM,MAAM,KACV,MAAM,QAAQ,YAAY,KAC1B,6BACD;AAED,QAAA,GAAS,QACP,cACA,KACA,QACA,MAAM,KAAK,UAAU,QACrB,YACD;EAED,IAAI,cAA4B,EAAE;AAElC,MAAI;AACF,iBAAc,MAAM,KAAK,UAAU,KAAI,OAAM;IAC3C,IAAI,EAAE;IACN,UAAU,EAAE;IACb,EAAE;GAEH,MAAM,WAAW,MAAM,mBACrB,oBACA,QACA,MAAA,IACA,MAAM,SACN;IACE,OAAO,MAAA,OAAa,IAAI;IACxB,UAAU,MAAA,OAAa,MAAM;IAC9B,EACD,MAAM,KACP;AACD,OAAI,UAAU,YAAY,WAAW,UAAU;AAC7C,QAAI,gBAAgB,SAAS,EAAE;AAC7B,WAAA,GAAS,OAAO,6CAA6C;MAC3D,UAAU,MAAM,QAAQ;MACxB,UAAU,UAAU,WAAW,SAAS,UAAU,KAAA;MACnD,CAAC;AACF,WAAA,eAAqB,eACnB,MAAM,SACN,MAAM,QAAQ,SACf;;AAEH,WAAO;;AAIT,SAAA,eAAqB,mBACnB,MAAM,SACN,MAAM,QAAQ,SACf;AACD,UAAO;WACA,GAAG;AACV,OAAI,gBAAgB,EAAE,IAAI,EAAE,UAAU,SAAS,cAAsB;IACnE,MAAM,WAAW;KACf,GAAG,EAAE;KACL;KACD;AACD,QAAI,gBAAgB,SAAS,EAAE;AAC7B,WAAA,GAAS,OAAO,6CAA6C;MAC3D,UAAU,MAAM,QAAQ;MACxB,UAAU,UAAU,WAAW,SAAS,UAAU,KAAA;MACnD,CAAC;AACF,WAAA,eAAqB,eACnB,MAAM,SACN,MAAM,QAAQ,SACf;;AAEH,WAAO;;AAGT,UAAO;IACL,MAAM;IACN,QAAQ;IACR,QAAQ;IACR,SAAS,mBAAmB,gBAAgB,EAAE;IAC9C;IACD;;;CAIL,gBACE,YACA,WACM;AACN,aAAW,KAAK,IAAI,uBAAuB,WAAW,OAAO,CAAC;;;;;;;;;AAUlE,SAAgB,cACd,SAC0B;CAC1B,MAAM,qCAAqB,IAAI,KAA4B;CAE3D,SAAS,UAAU;EACjB,MAAM,MAAqB,EAAE;AAC7B,OAAK,MAAM,WAAW,mBAAmB,QAAQ,EAAE;GACjD,MAAM,YAAyB;IAC7B,GAAG,QAAQ;IACX,MAAM;KACJ,GAAG,QAAQ,GAAG;KACd,WAAW,EAAE;KACd;IACF;AACD,OAAI,KAAK,UAAU;AACnB,QAAK,MAAM,SAAS,SAAS;AAC3B,8BAA0B,WAAW,MAAM;AAC3C,cAAU,KAAK,UAAU,KAAK,GAAG,MAAM,KAAK,UAAU;;;AAG1D,SAAO;;AAGT,MAAK,MAAM,SAAS,SAAS;AAC3B,MAAI,UAAU,UAAU,UAAU,KAAA,EAChC,QAAO,CAAC,SAAS,EAAE,KAAK;EAG1B,MAAM,MAAM,GAAG,MAAM,QAAQ,SAAS,GAAG,MAAM,QAAQ,KAAK,GAAG,MAAM,QAAQ;EAC7E,MAAM,WAAW,mBAAmB,IAAI,IAAI;AAC5C,MAAI,SACF,UAAS,KAAK,MAAM;MAEpB,oBAAmB,IAAI,KAAK,CAAC,MAAM,CAAC;;AAIxC,QAAO,CAAC,SAAS,EAAE,MAAM;;AAK3B,SAAS,0BAA0B,MAAmB,OAAoB;AACxE,QACE,KAAK,QAAQ,aAAa,MAAM,QAAQ,UACxC,2CACD;AACD,QACE,KAAK,QAAQ,SAAS,MAAM,QAAQ,MACpC,uCACD;AACD,QACE,KAAK,QAAQ,aAAa,MAAM,QAAQ,UACxC,2CACD;AACD,QACE,WAAW,KAAK,QAAQ,MAAM,MAAM,QAAQ,KAAK,EACjD,8DACD;AACD,QACE,KAAK,KAAK,kBAAkB,MAAM,KAAK,eACvC,uEACD;AACD,QACE,KAAK,KAAK,gBAAgB,MAAM,KAAK,aACrC,qEACD;AACD,QACE,KAAK,QAAQ,YAAY,cAAc,WACrC,MAAM,QAAQ,YAAY,cAAc,QAC1C,oEACD;AACD,QACE,KAAK,QAAQ,YAAY,cAAc,WACrC,MAAM,QAAQ,YAAY,cAAc,QAC1C,gEACD;AACD,QACE,KAAK,QAAQ,WAAW,MAAM,QAAQ,QACtC,gEACD;AACD,QACE,KAAK,QAAQ,YAAY,QAAQ,MAAM,QAAQ,YAAY,KAC3D,qEACD"}

package/out/zero-cache/src/services/replicator/change-processor.js

@@ -424,7 +424,7 @@ var TransactionProcessor = class {
 		const rowKeyCols = relation.rowKey.columns;
 		const cols = [...rowKeyCols, ...columns];
 		const insertColsStr = [...cols, ZERO_VERSION_COLUMN_NAME].map(id).join(",");
-		const qMarks = Array.from({ length: cols.length + 1 }, () => "?").join(",");
+		const qMarks = Array.from({ length: cols.length + 1 }).fill("?").join(",");
 		const rowKeyColsStr = rowKeyCols.map(id).join(",");
 		let backfilled = 0;
 		let skipped = 0;