@rocicorp/zero 0.26.1 → 0.26.2-canary.1

package/out/zero-cache/src/services/mutagen/pusher.js

@@ -1,504 +1,389 @@
-import { groupBy } from "../../../../shared/src/arrays.js";
 import { assert, unreachable } from "../../../../shared/src/asserts.js";
 import { getErrorMessage } from "../../../../shared/src/error.js";
+import { groupBy } from "../../../../shared/src/arrays.js";
 import { must } from "../../../../shared/src/must.js";
-import { Queue } from "../../../../shared/src/queue.js";
 import { PushFailed } from "../../../../zero-protocol/src/error-kind-enum.js";
-import { ZeroCache, Server } from "../../../../zero-protocol/src/error-origin-enum.js";
-import { HTTP, UnsupportedPushVersion, Internal, OutOfOrderMutation } from "../../../../zero-protocol/src/error-reason-enum.js";
+import { Server, ZeroCache } from "../../../../zero-protocol/src/error-origin-enum.js";
+import { HTTP, Internal, OutOfOrderMutation, UnsupportedPushVersion } from "../../../../zero-protocol/src/error-reason-enum.js";
 import { isProtocolError } from "../../../../zero-protocol/src/error.js";
 import { Custom } from "../../../../zero-protocol/src/mutation-type-enum.js";
 import { CLEANUP_RESULTS_MUTATION_NAME, pushResponseSchema } from "../../../../zero-protocol/src/push.js";
 import "../../config/zero-config.js";
-import { compileUrlPattern, fetchFromAPIServer } from "../../custom/fetch.js";
 import { getOrCreateCounter } from "../../observability/metrics.js";
+import { Queue } from "../../../../shared/src/queue.js";
+import { Subscription } from "../../types/subscription.js";
 import { recordMutation } from "../../server/anonymous-otel-start.js";
 import { ProtocolErrorWithLevel } from "../../types/error-with-level.js";
-import { Subscription } from "../../types/subscription.js";
-class PusherService {
-  id;
-  #pusher;
-  #queue;
-  #pushConfig;
-  #config;
-  #lc;
-  #pushURLPatterns;
-  #stopped;
-  #refCount = 0;
-  #isStopped = false;
-  constructor(appConfig, pushConfig, lc, clientGroupID) {
-    this.#config = appConfig;
-    this.#lc = lc.withContext("component", "pusherService");
-    this.#pushURLPatterns = pushConfig.url.map(compileUrlPattern);
-    this.#queue = new Queue();
-    this.#pusher = new PushWorker(
-      appConfig,
-      lc,
-      pushConfig.url,
-      pushConfig.apiKey,
-      pushConfig.allowedClientHeaders,
-      this.#queue
-    );
-    this.id = clientGroupID;
-    this.#pushConfig = pushConfig;
-  }
-  get pushURL() {
-    return this.#pusher.pushURL[0];
-  }
-  initConnection(clientID, wsID, userPushURL, userPushHeaders, onAuthFailure) {
-    return this.#pusher.initConnection(
-      clientID,
-      wsID,
-      userPushURL,
-      userPushHeaders,
-      onAuthFailure
-    );
-  }
-  enqueuePush(clientID, push, auth, httpCookie, origin) {
-    if (!this.#pushConfig.forwardCookies) {
-      httpCookie = void 0;
-    }
-    this.#queue.enqueue({ push, auth, clientID, httpCookie, origin });
-    return {
-      type: "ok"
-    };
-  }
-  async ackMutationResponses(upToID) {
-    const url = this.#pushConfig.url[0];
-    if (!url) {
-      return;
-    }
-    const cleanupBody = {
-      clientGroupID: this.id,
-      mutations: [
-        {
-          type: Custom,
-          id: 0,
-          // Not tracked - this is fire-and-forget
-          clientID: upToID.clientID,
-          name: CLEANUP_RESULTS_MUTATION_NAME,
-          args: [
-            {
-              type: "single",
-              clientGroupID: this.id,
-              clientID: upToID.clientID,
-              upToMutationID: upToID.id
-            }
-          ],
-          timestamp: Date.now()
-        }
-      ],
-      pushVersion: 1,
-      timestamp: Date.now(),
-      requestID: `cleanup-${this.id}-${upToID.clientID}-${upToID.id}`
-    };
-    try {
-      await fetchFromAPIServer(
-        pushResponseSchema,
-        "push",
-        this.#lc,
-        url,
-        this.#pushURLPatterns,
-        { appID: this.#config.app.id, shardNum: this.#config.shard.num },
-        { apiKey: this.#pushConfig.apiKey },
-        cleanupBody
-      );
-    } catch (e) {
-      this.#lc.warn?.("Failed to send cleanup mutation", {
-        error: getErrorMessage(e)
-      });
-    }
-  }
-  async deleteClientMutations(clientIDs) {
-    if (clientIDs.length === 0) {
-      return;
-    }
-    const url = this.#pushConfig.url[0];
-    if (!url) {
-      return;
-    }
-    const cleanupBody = {
-      clientGroupID: this.id,
-      mutations: [
-        {
-          type: Custom,
-          id: 0,
-          // Not tracked - this is fire-and-forget
-          clientID: clientIDs[0],
-          // Use first client as sender
-          name: CLEANUP_RESULTS_MUTATION_NAME,
-          args: [
-            {
-              type: "bulk",
-              clientGroupID: this.id,
-              clientIDs
-            }
-          ],
-          timestamp: Date.now()
-        }
-      ],
-      pushVersion: 1,
-      timestamp: Date.now(),
-      requestID: `cleanup-bulk-${this.id}-${Date.now()}`
-    };
-    try {
-      await fetchFromAPIServer(
-        pushResponseSchema,
-        "push",
-        this.#lc,
-        url,
-        this.#pushURLPatterns,
-        { appID: this.#config.app.id, shardNum: this.#config.shard.num },
-        { apiKey: this.#pushConfig.apiKey },
-        cleanupBody
-      );
-    } catch (e) {
-      this.#lc.warn?.("Failed to send bulk cleanup mutation", {
-        error: getErrorMessage(e)
-      });
-    }
-  }
-  ref() {
-    assert(!this.#isStopped, "PusherService is already stopped");
-    ++this.#refCount;
-  }
-  unref() {
-    assert(!this.#isStopped, "PusherService is already stopped");
-    --this.#refCount;
-    if (this.#refCount <= 0) {
-      void this.stop();
-    }
-  }
-  hasRefs() {
-    return this.#refCount > 0;
-  }
-  run() {
-    this.#stopped = this.#pusher.run();
-    return this.#stopped;
-  }
-  stop() {
-    if (this.#isStopped) {
-      return must(this.#stopped, "Stop was called before `run`");
-    }
-    this.#isStopped = true;
-    this.#queue.enqueue("stop");
-    return must(this.#stopped, "Stop was called before `run`");
-  }
-}
-class PushWorker {
-  #pushURLs;
-  #pushURLPatterns;
-  #apiKey;
-  #allowedClientHeaders;
-  #queue;
-  #lc;
-  #config;
-  #clients;
-  #userPushURL;
-  #userPushHeaders;
-  #customMutations = getOrCreateCounter(
-    "mutation",
-    "custom",
-    "Number of custom mutations processed"
-  );
-  #pushes = getOrCreateCounter(
-    "mutation",
-    "pushes",
-    "Number of pushes processed by the pusher"
-  );
-  constructor(config, lc, pushURL, apiKey, allowedClientHeaders, queue) {
-    this.#pushURLs = pushURL;
-    this.#lc = lc.withContext("component", "pusher");
-    this.#pushURLPatterns = pushURL.map(compileUrlPattern);
-    this.#apiKey = apiKey;
-    this.#allowedClientHeaders = allowedClientHeaders;
-    this.#queue = queue;
-    this.#config = config;
-    this.#clients = /* @__PURE__ */ new Map();
-  }
-  get pushURL() {
-    return this.#pushURLs;
-  }
-  /**
-   * Returns a new downstream stream if the clientID,wsID pair has not been seen before.
-   * If a clientID already exists with a different wsID, that client's downstream is cancelled.
-   */
-  initConnection(clientID, wsID, userPushURL, userPushHeaders, onAuthFailure) {
-    const existing = this.#clients.get(clientID);
-    if (existing && existing.wsID === wsID) {
-      throw new Error("Connection was already initialized");
-    }
-    if (existing) {
-      existing.downstream.cancel();
-    }
-    if (this.#userPushURL === void 0) {
-      this.#userPushURL = userPushURL;
-      this.#userPushHeaders = userPushHeaders;
-    } else {
-      if (this.#userPushURL !== userPushURL) {
-        this.#lc.warn?.(
-          "Client provided different mutate parameters than client group",
-          {
-            clientID,
-            clientURL: userPushURL,
-            clientGroupURL: this.#userPushURL
-          }
-        );
-      }
-    }
-    const downstream = Subscription.create({
-      cleanup: () => {
-        this.#clients.delete(clientID);
-      }
-    });
-    this.#clients.set(clientID, { wsID, downstream, onAuthFailure });
-    return downstream;
-  }
-  async run() {
-    for (; ; ) {
-      const task = await this.#queue.dequeue();
-      const rest = this.#queue.drain();
-      const [pushes, terminate] = combinePushes([task, ...rest]);
-      for (const push of pushes) {
-        const response = await this.#processPush(push);
-        await this.#fanOutResponses(response);
-      }
-      if (terminate) {
-        break;
-      }
-    }
-  }
-  /**
-   * 1. If the entire `push` fails, we send the error to relevant clients.
-   * 2. If the push succeeds, we look for any mutation failure that should cause the connection to terminate
-   *  and terminate the connection for those clients.
-   */
-  #fanOutResponses(response) {
-    const connectionTerminations = [];
-    if ("kind" in response || "error" in response) {
-      this.#lc.warn?.(
-        "The server behind ZERO_MUTATE_URL returned a push error.",
-        response
-      );
-      const groupedMutationIDs = groupBy(
-        response.mutationIDs ?? [],
-        (m) => m.clientID
-      );
-      for (const [clientID, mutationIDs] of groupedMutationIDs) {
-        const client = this.#clients.get(clientID);
-        if (!client) {
-          continue;
-        }
-        if ("error" in response) {
-          const pushFailedBody = response.error === "http" ? {
-            kind: PushFailed,
-            origin: ZeroCache,
-            reason: HTTP,
-            status: response.status,
-            bodyPreview: response.details,
-            mutationIDs,
-            message: `Fetch from API server returned non-OK status ${response.status}`
-          } : response.error === "unsupportedPushVersion" ? {
-            kind: PushFailed,
-            origin: Server,
-            reason: UnsupportedPushVersion,
-            mutationIDs,
-            message: `Unsupported push version`
-          } : {
-            kind: PushFailed,
-            origin: Server,
-            reason: Internal,
-            mutationIDs,
-            message: response.error === "zeroPusher" ? response.details : response.error === "unsupportedSchemaVersion" ? "Unsupported schema version" : "An unknown error occurred while pushing to the API server"
-          };
-          this.#failDownstream(client.downstream, pushFailedBody);
-          if (isPushAuthFailure(pushFailedBody)) {
-            this.#lc.debug?.("Auth failure detected in push response");
-            client.onAuthFailure?.();
-          }
-        } else if ("kind" in response) {
-          this.#failDownstream(client.downstream, response);
-          if (isPushAuthFailure(response)) {
-            this.#lc.debug?.("Auth failure detected in push response");
-            client.onAuthFailure?.();
-          }
-        } else {
-          unreachable();
-        }
-      }
-    } else {
-      const groupedMutations = groupBy(response.mutations, (m) => m.id.clientID);
-      for (const [clientID, mutations] of groupedMutations) {
-        const client = this.#clients.get(clientID);
-        if (!client) {
-          continue;
-        }
-        let failure;
-        let i = 0;
-        for (; i < mutations.length; i++) {
-          const m = mutations[i];
-          if ("error" in m.result) {
-            this.#lc.warn?.(
-              "The server behind ZERO_MUTATE_URL returned a mutation error.",
-              m.result
-            );
-          }
-          if ("error" in m.result && m.result.error === "oooMutation") {
-            failure = {
-              kind: PushFailed,
-              origin: Server,
-              reason: OutOfOrderMutation,
-              message: "mutation was out of order",
-              details: m.result.details,
-              mutationIDs: mutations.map((m2) => ({
-                clientID: m2.id.clientID,
-                id: m2.id.id
-              }))
-            };
-            break;
-          }
-        }
-        if (failure && i < mutations.length - 1) {
-          this.#lc.warn?.(
-            "push-response contains mutations after a mutation which should fatal the connection"
-          );
-        }
-        if (failure) {
-          connectionTerminations.push(
-            () => this.#failDownstream(client.downstream, failure)
-          );
-        }
-      }
-    }
-    connectionTerminations.forEach((cb) => cb());
-  }
-  async #processPush(entry) {
-    this.#customMutations.add(entry.push.mutations.length, {
-      clientGroupID: entry.push.clientGroupID
-    });
-    this.#pushes.add(1, {
-      clientGroupID: entry.push.clientGroupID
-    });
-    recordMutation("custom", entry.push.mutations.length);
-    const url = this.#userPushURL ?? must(this.#pushURLs[0], "ZERO_MUTATE_URL is not set");
-    this.#lc.debug?.(
-      "pushing to",
-      url,
-      "with",
-      entry.push.mutations.length,
-      "mutations"
-    );
-    let mutationIDs = [];
-    try {
-      mutationIDs = entry.push.mutations.map((m) => ({
-        id: m.id,
-        clientID: m.clientID
-      }));
-      return await fetchFromAPIServer(
-        pushResponseSchema,
-        "push",
-        this.#lc,
-        url,
-        this.#pushURLPatterns,
-        {
-          appID: this.#config.app.id,
-          shardNum: this.#config.shard.num
-        },
-        {
-          apiKey: this.#apiKey,
-          customHeaders: this.#userPushHeaders,
-          allowedClientHeaders: this.#allowedClientHeaders,
-          token: entry.auth,
-          cookie: entry.httpCookie,
-          origin: entry.origin
-        },
-        entry.push
-      );
-    } catch (e) {
-      if (isProtocolError(e) && e.errorBody.kind === PushFailed) {
-        return {
-          ...e.errorBody,
-          mutationIDs
-        };
-      }
-      return {
-        kind: PushFailed,
-        origin: ZeroCache,
-        reason: Internal,
-        message: `Failed to push: ${getErrorMessage(e)}`,
-        mutationIDs
-      };
-    }
-  }
-  #failDownstream(downstream, errorBody) {
-    downstream.fail(new ProtocolErrorWithLevel(errorBody, "warn"));
-  }
-}
+import { compileUrlPattern, fetchFromAPIServer } from "../../custom/fetch.js";
+//#region ../zero-cache/src/services/mutagen/pusher.ts
+/**
+* Receives push messages from zero-client and forwards
+* them the the user's API server.
+*
+* If the user's API server is taking too long to process
+* the push, the PusherService will add the push to a queue
+* and send pushes in bulk the next time the user's API server
+* is available.
+*
+* - One PusherService exists per client group.
+* - Mutations for a given client are always sent in-order
+* - Mutations for different clients in the same group may be interleaved
+*/
+var PusherService = class {
+	id;
+	#pusher;
+	#queue;
+	#pushConfig;
+	#config;
+	#lc;
+	#pushURLPatterns;
+	#stopped;
+	#refCount = 0;
+	#isStopped = false;
+	constructor(appConfig, pushConfig, lc, clientGroupID) {
+		this.#config = appConfig;
+		this.#lc = lc.withContext("component", "pusherService");
+		this.#pushURLPatterns = pushConfig.url.map(compileUrlPattern);
+		this.#queue = new Queue();
+		this.#pusher = new PushWorker(appConfig, lc, pushConfig.url, pushConfig.apiKey, pushConfig.allowedClientHeaders, this.#queue);
+		this.id = clientGroupID;
+		this.#pushConfig = pushConfig;
+	}
+	get pushURL() {
+		return this.#pusher.pushURL[0];
+	}
+	initConnection(clientID, wsID, userPushURL, userPushHeaders, onAuthFailure) {
+		return this.#pusher.initConnection(clientID, wsID, userPushURL, userPushHeaders, onAuthFailure);
+	}
+	enqueuePush(clientID, push, auth, httpCookie, origin) {
+		if (!this.#pushConfig.forwardCookies) httpCookie = void 0;
+		this.#queue.enqueue({
+			push,
+			auth,
+			clientID,
+			httpCookie,
+			origin
+		});
+		return { type: "ok" };
+	}
+	async ackMutationResponses(upToID) {
+		const url = this.#pushConfig.url[0];
+		if (!url) return;
+		const cleanupBody = {
+			clientGroupID: this.id,
+			mutations: [{
+				type: Custom,
+				id: 0,
+				clientID: upToID.clientID,
+				name: CLEANUP_RESULTS_MUTATION_NAME,
+				args: [{
+					type: "single",
+					clientGroupID: this.id,
+					clientID: upToID.clientID,
+					upToMutationID: upToID.id
+				}],
+				timestamp: Date.now()
+			}],
+			pushVersion: 1,
+			timestamp: Date.now(),
+			requestID: `cleanup-${this.id}-${upToID.clientID}-${upToID.id}`
+		};
+		try {
+			await fetchFromAPIServer(pushResponseSchema, "push", this.#lc, url, this.#pushURLPatterns, {
+				appID: this.#config.app.id,
+				shardNum: this.#config.shard.num
+			}, { apiKey: this.#pushConfig.apiKey }, cleanupBody);
+		} catch (e) {
+			this.#lc.warn?.("Failed to send cleanup mutation", { error: getErrorMessage(e) });
+		}
+	}
+	async deleteClientMutations(clientIDs) {
+		if (clientIDs.length === 0) return;
+		const url = this.#pushConfig.url[0];
+		if (!url) return;
+		const cleanupBody = {
+			clientGroupID: this.id,
+			mutations: [{
+				type: Custom,
+				id: 0,
+				clientID: clientIDs[0],
+				name: CLEANUP_RESULTS_MUTATION_NAME,
+				args: [{
+					type: "bulk",
+					clientGroupID: this.id,
+					clientIDs
+				}],
+				timestamp: Date.now()
+			}],
+			pushVersion: 1,
+			timestamp: Date.now(),
+			requestID: `cleanup-bulk-${this.id}-${Date.now()}`
+		};
+		try {
+			await fetchFromAPIServer(pushResponseSchema, "push", this.#lc, url, this.#pushURLPatterns, {
+				appID: this.#config.app.id,
+				shardNum: this.#config.shard.num
+			}, { apiKey: this.#pushConfig.apiKey }, cleanupBody);
+		} catch (e) {
+			this.#lc.warn?.("Failed to send bulk cleanup mutation", { error: getErrorMessage(e) });
+		}
+	}
+	ref() {
+		assert(!this.#isStopped, "PusherService is already stopped");
+		++this.#refCount;
+	}
+	unref() {
+		assert(!this.#isStopped, "PusherService is already stopped");
+		--this.#refCount;
+		if (this.#refCount <= 0) this.stop();
+	}
+	hasRefs() {
+		return this.#refCount > 0;
+	}
+	run() {
+		this.#stopped = this.#pusher.run();
+		return this.#stopped;
+	}
+	stop() {
+		if (this.#isStopped) return must(this.#stopped, "Stop was called before `run`");
+		this.#isStopped = true;
+		this.#queue.enqueue("stop");
+		return must(this.#stopped, "Stop was called before `run`");
+	}
+};
+/**
+* Awaits items in the queue then drains and sends them all
+* to the user's API server.
+*/
+var PushWorker = class {
+	#pushURLs;
+	#pushURLPatterns;
+	#apiKey;
+	#allowedClientHeaders;
+	#queue;
+	#lc;
+	#config;
+	#clients;
+	#userPushURL;
+	#userPushHeaders;
+	#customMutations = getOrCreateCounter("mutation", "custom", "Number of custom mutations processed");
+	#pushes = getOrCreateCounter("mutation", "pushes", "Number of pushes processed by the pusher");
+	constructor(config, lc, pushURL, apiKey, allowedClientHeaders, queue) {
+		this.#pushURLs = pushURL;
+		this.#lc = lc.withContext("component", "pusher");
+		this.#pushURLPatterns = pushURL.map(compileUrlPattern);
+		this.#apiKey = apiKey;
+		this.#allowedClientHeaders = allowedClientHeaders;
+		this.#queue = queue;
+		this.#config = config;
+		this.#clients = /* @__PURE__ */ new Map();
+	}
+	get pushURL() {
+		return this.#pushURLs;
+	}
+	/**
+	* Returns a new downstream stream if the clientID,wsID pair has not been seen before.
+	* If a clientID already exists with a different wsID, that client's downstream is cancelled.
+	*/
+	initConnection(clientID, wsID, userPushURL, userPushHeaders, onAuthFailure) {
+		const existing = this.#clients.get(clientID);
+		if (existing && existing.wsID === wsID) throw new Error("Connection was already initialized");
+		if (existing) existing.downstream.cancel();
+		if (this.#userPushURL === void 0) {
+			this.#userPushURL = userPushURL;
+			this.#userPushHeaders = userPushHeaders;
+		} else if (this.#userPushURL !== userPushURL) this.#lc.warn?.("Client provided different mutate parameters than client group", {
+			clientID,
+			clientURL: userPushURL,
+			clientGroupURL: this.#userPushURL
+		});
+		const downstream = Subscription.create({ cleanup: () => {
+			this.#clients.delete(clientID);
+		} });
+		this.#clients.set(clientID, {
+			wsID,
+			downstream,
+			onAuthFailure
+		});
+		return downstream;
+	}
+	async run() {
+		for (;;) {
+			const [pushes, terminate] = combinePushes([await this.#queue.dequeue(), ...this.#queue.drain()]);
+			for (const push of pushes) {
+				const response = await this.#processPush(push);
+				await this.#fanOutResponses(response);
+			}
+			if (terminate) break;
+		}
+	}
+	/**
+	* 1. If the entire `push` fails, we send the error to relevant clients.
+	* 2. If the push succeeds, we look for any mutation failure that should cause the connection to terminate
+	*  and terminate the connection for those clients.
+	*/
+	#fanOutResponses(response) {
+		const connectionTerminations = [];
+		if ("kind" in response || "error" in response) {
+			this.#lc.warn?.("The server behind ZERO_MUTATE_URL returned a push error.", response);
+			const groupedMutationIDs = groupBy(response.mutationIDs ?? [], (m) => m.clientID);
+			for (const [clientID, mutationIDs] of groupedMutationIDs) {
+				const client = this.#clients.get(clientID);
+				if (!client) continue;
+				if ("error" in response) {
+					const pushFailedBody = response.error === "http" ? {
+						kind: PushFailed,
+						origin: ZeroCache,
+						reason: HTTP,
+						status: response.status,
+						bodyPreview: response.details,
+						mutationIDs,
+						message: `Fetch from API server returned non-OK status ${response.status}`
+					} : response.error === "unsupportedPushVersion" ? {
+						kind: PushFailed,
+						origin: Server,
+						reason: UnsupportedPushVersion,
+						mutationIDs,
+						message: `Unsupported push version`
+					} : {
+						kind: PushFailed,
+						origin: Server,
+						reason: Internal,
+						mutationIDs,
+						message: response.error === "zeroPusher" ? response.details : response.error === "unsupportedSchemaVersion" ? "Unsupported schema version" : "An unknown error occurred while pushing to the API server"
+					};
+					this.#failDownstream(client.downstream, pushFailedBody);
+					if (isPushAuthFailure(pushFailedBody)) {
+						this.#lc.debug?.("Auth failure detected in push response");
+						client.onAuthFailure?.();
+					}
+				} else if ("kind" in response) {
+					this.#failDownstream(client.downstream, response);
+					if (isPushAuthFailure(response)) {
+						this.#lc.debug?.("Auth failure detected in push response");
+						client.onAuthFailure?.();
+					}
+				} else unreachable(response);
+			}
+		} else {
+			const groupedMutations = groupBy(response.mutations, (m) => m.id.clientID);
+			for (const [clientID, mutations] of groupedMutations) {
+				const client = this.#clients.get(clientID);
+				if (!client) continue;
+				let failure;
+				let i = 0;
+				for (; i < mutations.length; i++) {
+					const m = mutations[i];
+					if ("error" in m.result) this.#lc.warn?.("The server behind ZERO_MUTATE_URL returned a mutation error.", m.result);
+					if ("error" in m.result && m.result.error === "oooMutation") {
+						failure = {
+							kind: PushFailed,
+							origin: Server,
+							reason: OutOfOrderMutation,
+							message: "mutation was out of order",
+							details: m.result.details,
+							mutationIDs: mutations.map((m) => ({
+								clientID: m.id.clientID,
+								id: m.id.id
+							}))
+						};
+						break;
+					}
+				}
+				if (failure && i < mutations.length - 1) this.#lc.warn?.("push-response contains mutations after a mutation which should fatal the connection");
+				if (failure) connectionTerminations.push(() => this.#failDownstream(client.downstream, failure));
+			}
+		}
+		connectionTerminations.forEach((cb) => cb());
+	}
+	async #processPush(entry) {
+		this.#customMutations.add(entry.push.mutations.length, { clientGroupID: entry.push.clientGroupID });
+		this.#pushes.add(1, { clientGroupID: entry.push.clientGroupID });
+		recordMutation("custom", entry.push.mutations.length);
+		const url = this.#userPushURL ?? must(this.#pushURLs[0], "ZERO_MUTATE_URL is not set");
+		this.#lc.debug?.("pushing to", url, "with", entry.push.mutations.length, "mutations");
+		let mutationIDs = [];
+		try {
+			mutationIDs = entry.push.mutations.map((m) => ({
+				id: m.id,
+				clientID: m.clientID
+			}));
+			return await fetchFromAPIServer(pushResponseSchema, "push", this.#lc, url, this.#pushURLPatterns, {
+				appID: this.#config.app.id,
+				shardNum: this.#config.shard.num
+			}, {
+				apiKey: this.#apiKey,
+				customHeaders: this.#userPushHeaders,
+				allowedClientHeaders: this.#allowedClientHeaders,
+				token: entry.auth,
+				cookie: entry.httpCookie,
+				origin: entry.origin
+			}, entry.push);
+		} catch (e) {
+			if (isProtocolError(e) && e.errorBody.kind === "PushFailed") return {
+				...e.errorBody,
+				mutationIDs
+			};
+			return {
+				kind: PushFailed,
+				origin: ZeroCache,
+				reason: Internal,
+				message: `Failed to push: ${getErrorMessage(e)}`,
+				mutationIDs
+			};
+		}
+	}
+	#failDownstream(downstream, errorBody) {
+		downstream.fail(new ProtocolErrorWithLevel(errorBody, "warn"));
+	}
+};
 function isPushAuthFailure(errorBody) {
-  return errorBody.reason === HTTP && (errorBody.status === 401 || errorBody.status === 403);
+	return errorBody.reason === "http" && (errorBody.status === 401 || errorBody.status === 403);
 }
+/**
+* Pushes for different clientIDs could theoretically be interleaved.
+*
+* In order to do efficient batching to the user's API server,
+* we collect all pushes for the same clientID into a single push.
+*/
 function combinePushes(entries) {
-  const pushesByClientID = /* @__PURE__ */ new Map();
-  function collect() {
-    const ret = [];
-    for (const entries2 of pushesByClientID.values()) {
-      const composite = {
-        ...entries2[0],
-        push: {
-          ...entries2[0].push,
-          mutations: []
-        }
-      };
-      ret.push(composite);
-      for (const entry of entries2) {
-        assertAreCompatiblePushes(composite, entry);
-        composite.push.mutations.push(...entry.push.mutations);
-      }
-    }
-    return ret;
-  }
-  for (const entry of entries) {
-    if (entry === "stop" || entry === void 0) {
-      return [collect(), true];
-    }
-    const { clientID } = entry;
-    const existing = pushesByClientID.get(clientID);
-    if (existing) {
-      existing.push(entry);
-    } else {
-      pushesByClientID.set(clientID, [entry]);
-    }
-  }
-  return [collect(), false];
+	const pushesByClientID = /* @__PURE__ */ new Map();
+	function collect() {
+		const ret = [];
+		for (const entries of pushesByClientID.values()) {
+			const composite = {
+				...entries[0],
+				push: {
+					...entries[0].push,
+					mutations: []
+				}
+			};
+			ret.push(composite);
+			for (const entry of entries) {
+				assertAreCompatiblePushes(composite, entry);
+				composite.push.mutations.push(...entry.push.mutations);
+			}
+		}
+		return ret;
+	}
+	for (const entry of entries) {
+		if (entry === "stop" || entry === void 0) return [collect(), true];
+		const { clientID } = entry;
+		const existing = pushesByClientID.get(clientID);
+		if (existing) existing.push(entry);
+		else pushesByClientID.set(clientID, [entry]);
+	}
+	return [collect(), false];
 }
 function assertAreCompatiblePushes(left, right) {
-  assert(
-    left.clientID === right.clientID,
-    "clientID must be the same for all pushes"
-  );
-  assert(
-    left.auth === right.auth,
-    "auth must be the same for all pushes with the same clientID"
-  );
-  assert(
-    left.push.schemaVersion === right.push.schemaVersion,
-    "schemaVersion must be the same for all pushes with the same clientID"
-  );
-  assert(
-    left.push.pushVersion === right.push.pushVersion,
-    "pushVersion must be the same for all pushes with the same clientID"
-  );
-  assert(
-    left.httpCookie === right.httpCookie,
-    "httpCookie must be the same for all pushes with the same clientID"
-  );
-  assert(
-    left.origin === right.origin,
-    "origin must be the same for all pushes with the same clientID"
-  );
+	assert(left.clientID === right.clientID, "clientID must be the same for all pushes");
+	assert(left.auth === right.auth, "auth must be the same for all pushes with the same clientID");
+	assert(left.push.schemaVersion === right.push.schemaVersion, "schemaVersion must be the same for all pushes with the same clientID");
+	assert(left.push.pushVersion === right.push.pushVersion, "pushVersion must be the same for all pushes with the same clientID");
+	assert(left.httpCookie === right.httpCookie, "httpCookie must be the same for all pushes with the same clientID");
+	assert(left.origin === right.origin, "origin must be the same for all pushes with the same clientID");
 }
-export {
-  PusherService,
-  combinePushes
-};
-//# sourceMappingURL=pusher.js.map
+//#endregion
+export { PusherService };
+
+//# sourceMappingURL=pusher.js.map