@rocicorp/zero 1.12.0-canary.7 → 1.12.0-canary.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/zero/package.js +1 -1
- package/out/zero/package.js.map +1 -1
- package/out/zero-cache/src/services/mutagen/pusher.js +57 -14
- package/out/zero-cache/src/services/mutagen/pusher.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/drain-coordinator.js +1 -1
- package/out/zero-cache/src/services/view-syncer/drain-coordinator.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.js +3 -0
- package/out/zero-cache/src/services/view-syncer/view-syncer.js.map +1 -1
- package/out/zero-cache/src/workers/syncer.js +9 -1
- package/out/zero-cache/src/workers/syncer.js.map +1 -1
- package/out/zero-client/src/client/version.js +1 -1
- package/package.json +1 -1
package/out/zero/package.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
//#region package.json
|
|
2
2
|
var name = "@rocicorp/zero";
|
|
3
|
-
var version = "1.12.0-canary.
|
|
3
|
+
var version = "1.12.0-canary.8";
|
|
4
4
|
var description = "Zero is a web framework for serverless web development.";
|
|
5
5
|
var homepage = "https://zero.rocicorp.dev";
|
|
6
6
|
var bugs = { "url": "https://bugs.rocicorp.dev" };
|
package/out/zero/package.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"package.js","names":[],"sources":["../../package.json"],"sourcesContent":["{\n \"name\": \"@rocicorp/zero\",\n \"version\": \"1.12.0-canary.
|
|
1
|
+
{"version":3,"file":"package.js","names":[],"sources":["../../package.json"],"sourcesContent":["{\n \"name\": \"@rocicorp/zero\",\n \"version\": \"1.12.0-canary.8\",\n \"description\": \"Zero is a web framework for serverless web development.\",\n \"homepage\": \"https://zero.rocicorp.dev\",\n \"bugs\": {\n \"url\": \"https://bugs.rocicorp.dev\"\n },\n \"license\": \"Apache-2.0\",\n \"author\": \"Rocicorp, Inc.\",\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/rocicorp/mono.git\",\n \"directory\": \"packages/zero\"\n },\n \"bin\": {\n \"ast-to-zql\": \"./out/zero/src/ast-to-zql.js\",\n \"transform-query\": \"./out/zero/src/transform-query.js\",\n \"zero-build-schema\": \"./out/zero/src/build-schema.js\",\n \"zero-cache\": \"./out/zero/src/cli.js\",\n \"zero-cache-dev\": \"./out/zero/src/zero-cache-dev.js\",\n \"zero-deploy-permissions\": \"./out/zero/src/deploy-permissions.js\",\n \"zero-out\": \"./out/zero/src/zero-out.js\"\n },\n \"files\": [\n \"out\",\n \"!*.tsbuildinfo\"\n ],\n \"type\": \"module\",\n \"main\": \"out/zero/src/zero.js\",\n \"module\": \"out/zero/src/zero.js\",\n \"types\": \"out/zero/src/zero.d.ts\",\n \"exports\": {\n \".\": {\n \"types\": \"./out/zero/src/zero.d.ts\",\n \"default\": \"./out/zero/src/zero.js\"\n },\n \"./analyze\": {\n \"types\": \"./out/zero/src/analyze.d.ts\",\n \"default\": \"./out/zero/src/analyze.js\"\n },\n \"./bindings\": {\n \"types\": \"./out/zero/src/bindings.d.ts\",\n \"default\": \"./out/zero/src/bindings.js\"\n },\n \"./change-protocol/v0\": {\n \"types\": \"./out/zero/src/change-protocol/v0.d.ts\",\n \"default\": \"./out/zero/src/change-protocol/v0.js\"\n },\n \"./expo-sqlite\": {\n \"types\": \"./out/zero/src/expo-sqlite.d.ts\",\n \"default\": \"./out/zero/src/expo-sqlite.js\"\n },\n \"./op-sqlite\": {\n \"types\": \"./out/zero/src/op-sqlite.d.ts\",\n \"default\": \"./out/zero/src/op-sqlite.js\"\n },\n \"./pg\": {\n \"types\": \"./out/zero/src/pg.d.ts\",\n \"default\": \"./out/zero/src/pg.js\"\n },\n \"./react\": {\n \"types\": \"./out/zero/src/react.d.ts\",\n \"default\": \"./out/zero/src/react.js\"\n },\n \"./react-native\": {\n \"types\": \"./out/zero/src/react-native.d.ts\",\n \"default\": \"./out/zero/src/react-native.js\"\n },\n \"./server\": {\n \"types\": \"./out/zero/src/server.d.ts\",\n \"default\": \"./out/zero/src/server.js\"\n },\n \"./server/adapters/drizzle\": {\n \"types\": \"./out/zero/src/adapters/drizzle.d.ts\",\n \"default\": \"./out/zero/src/adapters/drizzle.js\"\n },\n \"./server/adapters/kysely\": {\n \"types\": \"./out/zero/src/adapters/kysely.d.ts\",\n \"default\": \"./out/zero/src/adapters/kysely.js\"\n },\n \"./server/adapters/prisma\": {\n \"types\": \"./out/zero/src/adapters/prisma.d.ts\",\n \"default\": \"./out/zero/src/adapters/prisma.js\"\n },\n \"./server/adapters/pg\": {\n \"types\": \"./out/zero/src/adapters/pg.d.ts\",\n \"default\": \"./out/zero/src/adapters/pg.js\"\n },\n \"./server/adapters/postgresjs\": {\n \"types\": \"./out/zero/src/adapters/postgresjs.d.ts\",\n \"default\": \"./out/zero/src/adapters/postgresjs.js\"\n },\n \"./solid\": {\n \"types\": \"./out/zero/src/solid.d.ts\",\n \"default\": \"./out/zero/src/solid.js\"\n },\n \"./sqlite\": {\n \"types\": \"./out/zero/src/sqlite.d.ts\",\n \"default\": \"./out/zero/src/sqlite.js\"\n },\n \"./zqlite\": {\n \"types\": \"./out/zero/src/zqlite.d.ts\",\n \"default\": \"./out/zero/src/zqlite.js\"\n }\n },\n \"scripts\": {\n \"build\": \"node --experimental-strip-types --no-warnings tool/build.ts\",\n \"build:watch\": \"node --experimental-strip-types --no-warnings tool/build.ts --watch\",\n \"check-types\": \"tsc -p tsconfig.client.json && tsc -p tsconfig.server.json\",\n \"check-types:client:watch\": \"tsc -p tsconfig.client.json --watch\",\n \"check-types:server:watch\": \"tsc -p tsconfig.server.json --watch\",\n \"format\": \"oxfmt .\",\n \"check-format\": \"oxfmt --check .\",\n \"lint\": \"oxlint --quiet --config ../../oxlint.config.ts src/\",\n \"docs\": \"node --experimental-strip-types --no-warnings tool/generate-docs.ts\",\n \"docs:server\": \"node --watch --experimental-strip-types --no-warnings tool/generate-docs.ts --server\",\n \"fmt\": \"oxfmt .\",\n \"check-fmt\": \"oxfmt --check .\"\n },\n \"dependencies\": {\n \"@badrap/valita\": \"0.5.5\",\n \"@databases/escape-identifier\": \"^1.0.3\",\n \"@databases/sql\": \"^3.3.0\",\n \"@dotenvx/dotenvx\": \"^1.66.0\",\n \"@drdgvhbh/postgres-error-codes\": \"^0.0.6\",\n \"@fastify/cors\": \"^10.1.0\",\n \"@fastify/websocket\": \"^11.2.0\",\n \"@google-cloud/precise-date\": \"^4.0.0\",\n \"@opentelemetry/api\": \"1.9.1\",\n \"@opentelemetry/api-logs\": \"0.223.0\",\n \"@opentelemetry/auto-instrumentations-node\": \"0.81.2\",\n \"@opentelemetry/exporter-metrics-otlp-http\": \"0.223.0\",\n \"@opentelemetry/resources\": \"2.12.0\",\n \"@opentelemetry/sdk-metrics\": \"2.12.0\",\n \"@opentelemetry/sdk-node\": \"0.223.0\",\n \"@opentelemetry/sdk-trace-node\": \"2.12.0\",\n \"@opentelemetry/semantic-conventions\": \"1.43.0\",\n \"@postgresql-typed/oids\": \"^0.2.0\",\n \"@rocicorp/lock\": \"^1.0.4\",\n \"@rocicorp/logger\": \"^6.1.0\",\n \"@rocicorp/resolver\": \"^1.0.2\",\n \"@rocicorp/zero-sqlite3\": \"^1.1.7\",\n \"@standard-schema/spec\": \"^1.1.0\",\n \"@types/basic-auth\": \"^1.1.8\",\n \"@types/ws\": \"^8.18.1\",\n \"basic-auth\": \"^2.0.1\",\n \"chalk-template\": \"^1.1.2\",\n \"chokidar\": \"^4.0.3\",\n \"cloudevents\": \"^10.0.0\",\n \"command-line-args\": \"^6.0.2\",\n \"command-line-usage\": \"^7.0.4\",\n \"compare-utf8\": \"^0.2.3\",\n \"defu\": \"^6.1.7\",\n \"eventemitter3\": \"^5.0.4\",\n \"fastify\": \"^5.12.7\",\n \"is-in-subnet\": \"^4.0.1\",\n \"jose\": \"^5.10.0\",\n \"json-custom-numbers\": \"^3.1.1\",\n \"kasi\": \"^1.1.2\",\n \"nanoid\": \"^5.1.16\",\n \"oxc-parser\": \"^0.129.0\",\n \"oxfmt\": \"^0.74.0\",\n \"parse-prometheus-text-format\": \"^1.1.1\",\n \"pg-format\": \"npm:pg-format-fix@^1.0.5\",\n \"postgres\": \"3.4.7\",\n \"semver\": \"^7.8.0\",\n \"url-pattern\": \"^1.0.3\",\n \"urlpattern-polyfill\": \"^10.1.0\",\n \"ws\": \"^8.22.0\"\n },\n \"devDependencies\": {\n \"@op-engineering/op-sqlite\": \"^18.2.5\",\n \"analyze-query\": \"workspace:*\",\n \"ast-to-zql\": \"workspace:*\",\n \"expo\": \"^58.0.14\",\n \"expo-sqlite\": \"^58.0.13\",\n \"react-native\": \"^0.88.0\",\n \"replicache\": \"workspace:*\",\n \"shared\": \"workspace:*\",\n \"syncpack\": \"^14.3.1\",\n \"typedoc\": \"^0.28.19\",\n \"typescript\": \"~7.0.2\",\n \"vite\": \"^8.3.4\",\n \"vitest\": \"^5.0.5\",\n \"zero-cache\": \"workspace:*\",\n \"zero-client\": \"workspace:*\",\n \"zero-pg\": \"workspace:*\",\n \"zero-react\": \"workspace:*\",\n \"zero-server\": \"workspace:*\",\n \"zero-solid\": \"workspace:*\",\n \"zqlite\": \"workspace:*\"\n },\n \"engines\": {\n \"node\": \">=22\"\n },\n \"packageManager\": \"pnpm@11.11.0\"\n}\n"],"mappings":""}
|
|
@@ -167,6 +167,18 @@ var PushWorker = class {
|
|
|
167
167
|
#lc;
|
|
168
168
|
#config;
|
|
169
169
|
#clients;
|
|
170
|
+
/**
|
|
171
|
+
* For each client, the wsID of its socket whose push most recently failed.
|
|
172
|
+
* Pushes queued behind the failed one on that socket are not sent: they
|
|
173
|
+
* start past the failed mutation, so the API server could only reject them
|
|
174
|
+
* as out of order. The socket is failed (or was already replaced), and the
|
|
175
|
+
* client re-sends every unacknowledged mutation on its next socket.
|
|
176
|
+
*
|
|
177
|
+
* An entry is kept until the socket can no longer queue pushes (it closed or
|
|
178
|
+
* the client reconnected) and the queue has been drained since, so it is not
|
|
179
|
+
* removed while pushes from the failed socket may still be queued.
|
|
180
|
+
*/
|
|
181
|
+
#failedSockets = /* @__PURE__ */ new Map();
|
|
170
182
|
#customMutations = getOrCreateCounter("mutation", "custom", "Number of custom mutations processed");
|
|
171
183
|
#pushes = getOrCreateCounter("mutation", "pushes", "Number of pushes processed by the pusher");
|
|
172
184
|
constructor(config, lc, connContextManager, queue) {
|
|
@@ -201,28 +213,47 @@ var PushWorker = class {
|
|
|
201
213
|
}
|
|
202
214
|
async run() {
|
|
203
215
|
for (;;) {
|
|
216
|
+
const closedSockets = [...this.#failedSockets].filter(([clientID, wsID]) => !this.#connContextManager.getConnectionContext({
|
|
217
|
+
clientID,
|
|
218
|
+
wsID
|
|
219
|
+
}));
|
|
204
220
|
const [pushes, terminate] = combinePushes([await this.#queue.dequeue(), ...this.#queue.drain()]);
|
|
205
221
|
for (const push of pushes) {
|
|
222
|
+
const { clientID, wsID } = push.connCtx;
|
|
223
|
+
if (this.#failedSockets.get(clientID) === wsID) {
|
|
224
|
+
this.#lc.debug?.("Not sending push queued behind a failed push on the same socket; the client re-sends it on its next socket", {
|
|
225
|
+
clientID,
|
|
226
|
+
wsID,
|
|
227
|
+
mutations: push.push.mutations.length
|
|
228
|
+
});
|
|
229
|
+
continue;
|
|
230
|
+
}
|
|
206
231
|
const parentContext = push.push.traceparent ? propagation.extract(ROOT_CONTEXT, { traceparent: push.push.traceparent }) : context.active();
|
|
207
232
|
const response = await context.with(parentContext, () => this.#processPush(push));
|
|
208
|
-
await this.#fanOutResponses(response);
|
|
233
|
+
await this.#fanOutResponses(push.connCtx, response);
|
|
209
234
|
}
|
|
235
|
+
for (const [clientID, wsID] of closedSockets) if (this.#failedSockets.get(clientID) === wsID) this.#failedSockets.delete(clientID);
|
|
210
236
|
if (terminate) break;
|
|
211
237
|
}
|
|
212
238
|
}
|
|
213
239
|
/**
|
|
214
|
-
* 1. If the entire `push` fails, we send the error to
|
|
240
|
+
* 1. If the entire `push` fails, we send the error to `origin`, the
|
|
241
|
+
* connection the push was sent on.
|
|
215
242
|
* 2. If the push succeeds, we look for any mutation failure that should cause the connection to terminate
|
|
216
|
-
* and terminate
|
|
243
|
+
* and terminate `origin`.
|
|
244
|
+
*
|
|
245
|
+
* Only `origin` is failed, even when the push carried other clients'
|
|
246
|
+
* mutations (the client group's commit chain includes other tabs' pending
|
|
247
|
+
* mutations). A client tracks what it has sent per socket, so the gap a
|
|
248
|
+
* failed push leaves is on `origin` alone. Other clients send their own
|
|
249
|
+
* mutations on their own sockets, and see any lasting error there.
|
|
217
250
|
*/
|
|
218
|
-
#fanOutResponses(response) {
|
|
251
|
+
#fanOutResponses(origin, response) {
|
|
219
252
|
const connectionTerminations = [];
|
|
220
253
|
if ("kind" in response && response.kind === "PushFailed" || "error" in response) {
|
|
221
254
|
this.#lc.warn?.("The server behind ZERO_MUTATE_URL returned a push error.", response);
|
|
222
|
-
const
|
|
223
|
-
|
|
224
|
-
const client = this.#clients.get(clientID);
|
|
225
|
-
if (!client) continue;
|
|
255
|
+
const mutationIDs = response.mutationIDs ?? [];
|
|
256
|
+
if (mutationIDs.length > 0) {
|
|
226
257
|
if ("error" in response) {
|
|
227
258
|
const pushFailedBody = response.error === "http" ? {
|
|
228
259
|
kind: PushFailed,
|
|
@@ -245,14 +276,12 @@ var PushWorker = class {
|
|
|
245
276
|
mutationIDs,
|
|
246
277
|
message: response.error === "zeroPusher" ? response.details : response.error === "unsupportedSchemaVersion" ? "Unsupported schema version" : "An unknown error occurred while pushing to the API server"
|
|
247
278
|
};
|
|
248
|
-
this.#
|
|
249
|
-
} else this.#
|
|
279
|
+
this.#failOrigin(origin, pushFailedBody);
|
|
280
|
+
} else this.#failOrigin(origin, response);
|
|
250
281
|
}
|
|
251
282
|
} else {
|
|
252
283
|
const groupedMutations = groupBy(response.mutations, (m) => m.id.clientID);
|
|
253
|
-
for (const
|
|
254
|
-
const client = this.#clients.get(clientID);
|
|
255
|
-
if (!client) continue;
|
|
284
|
+
for (const mutations of groupedMutations.values()) {
|
|
256
285
|
let failure;
|
|
257
286
|
let i = 0;
|
|
258
287
|
for (; i < mutations.length; i++) {
|
|
@@ -274,11 +303,25 @@ var PushWorker = class {
|
|
|
274
303
|
}
|
|
275
304
|
}
|
|
276
305
|
if (failure && i < mutations.length - 1) this.#lc.warn?.("push-response contains mutations after a mutation which should fatal the connection");
|
|
277
|
-
if (failure) connectionTerminations.push(() => this.#
|
|
306
|
+
if (failure) connectionTerminations.push(() => this.#failOrigin(origin, failure));
|
|
278
307
|
}
|
|
279
308
|
}
|
|
280
309
|
connectionTerminations.forEach((cb) => cb());
|
|
281
310
|
}
|
|
311
|
+
/**
|
|
312
|
+
* Fails `origin` after a push sent on it failed, and records it in
|
|
313
|
+
* `#failedSockets` so the pushes it queued behind that one are not sent.
|
|
314
|
+
*
|
|
315
|
+
* If the client has reconnected since sending the push, the old socket is
|
|
316
|
+
* already gone and the client re-sends its mutations on the new one. Failing
|
|
317
|
+
* the new socket would put the client in its error state, rejecting its
|
|
318
|
+
* outstanding mutations, for a failure it has already recovered from.
|
|
319
|
+
*/
|
|
320
|
+
#failOrigin(origin, errorBody) {
|
|
321
|
+
this.#failedSockets.set(origin.clientID, origin.wsID);
|
|
322
|
+
const client = this.#clients.get(origin.clientID);
|
|
323
|
+
if (client?.wsID === origin.wsID) this.#failDownstream(client.downstream, errorBody);
|
|
324
|
+
}
|
|
282
325
|
async #processPush(entry) {
|
|
283
326
|
this.#customMutations.add(entry.push.mutations.length, { clientGroupID: entry.push.clientGroupID });
|
|
284
327
|
this.#pushes.add(1, { clientGroupID: entry.push.clientGroupID });
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pusher.js","names":["MutationType.Custom","ErrorKind.PushFailed","ErrorOrigin.ZeroCache","ErrorReason.HTTP","ErrorOrigin.Server","ErrorReason.UnsupportedPushVersion","ErrorReason.Internal","ErrorReason.OutOfOrderMutation"],"sources":["../../../../../../zero-cache/src/services/mutagen/pusher.ts"],"sourcesContent":["import {context, propagation, ROOT_CONTEXT} from '@opentelemetry/api';\nimport type {LogContext} from '@rocicorp/logger';\nimport {groupBy, newArray} from '../../../../shared/src/arrays.ts';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {getErrorMessage} from '../../../../shared/src/error.ts';\nimport {getOrInsertComputed} from '../../../../shared/src/map.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Queue} from '../../../../shared/src/queue.ts';\nimport type {Downstream} from '../../../../zero-protocol/src/down.ts';\nimport {ErrorKind} from '../../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../../zero-protocol/src/error-origin.ts';\nimport {ErrorReason} from '../../../../zero-protocol/src/error-reason.ts';\nimport {\n isProtocolError,\n type PushFailedBody,\n} from '../../../../zero-protocol/src/error.ts';\nimport {\n mutateResponseSchema,\n type MutateResponse,\n} from '../../../../zero-protocol/src/mutate-server.ts';\nimport type {MutationID} from '../../../../zero-protocol/src/mutation-id.ts';\nimport * as MutationType from '../../../../zero-protocol/src/mutation-type-enum.ts';\nimport {CLEANUP_RESULTS_MUTATION_NAME} from '../../../../zero-protocol/src/mutation.ts';\nimport {type PushBody} from '../../../../zero-protocol/src/push.ts';\nimport {authEquals, isAuthErrorBody} from '../../auth/auth.ts';\nimport {type ZeroConfig} from '../../config/zero-config.ts';\nimport {fetchFromAPIServer} from '../../custom/fetch.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {recordMutation} from '../../server/anonymous-otel-start.ts';\nimport {ProtocolErrorWithLevel} from '../../types/error-with-level.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {HandlerResult, StreamResult} from '../../workers/connection.ts';\nimport type {RefCountedService, Service} from '../service.ts';\nimport type {\n ConnectionContext,\n ConnectionContextManager,\n ConnectionSelector,\n} from '../view-syncer/connection-context-manager.ts';\n\nexport interface Pusher extends RefCountedService {\n initConnection(selector: ConnectionSelector): Source<Downstream>;\n enqueuePush(selector: ConnectionSelector, push: PushBody): HandlerResult;\n ackMutationResponses(\n requester: ConnectionSelector,\n upToID: MutationID,\n ): Promise<void>;\n deleteClientMutations(\n requester: ConnectionSelector,\n clientIDs: string[],\n ): Promise<void>;\n}\n\ntype Config = Pick<ZeroConfig, 'app' | 'shard'>;\n\n/**\n * Receives push messages from zero-client and forwards\n * them the the user's API server.\n *\n * If the user's API server is taking too long to process\n * the push, the PusherService will add the push to a queue\n * and send pushes in bulk the next time the user's API server\n * is available.\n *\n * - One PusherService exists per client group.\n * - Mutations for a given client are always sent in-order\n * - Mutations for different clients in the same group may be interleaved\n */\nexport class PusherService implements Service, Pusher {\n readonly id: string;\n readonly #connContextManager: ConnectionContextManager;\n readonly #pusher: PushWorker;\n readonly #queue: Queue<PusherEntryOrStop>;\n readonly #config: Config;\n readonly #lc: LogContext;\n #stopped: Promise<void> | undefined;\n #refCount = 0;\n #isStopped = false;\n\n constructor(\n appConfig: Config,\n lc: LogContext,\n clientGroupID: string,\n connContextManager: ConnectionContextManager,\n ) {\n this.#connContextManager = connContextManager;\n this.#config = appConfig;\n this.#lc = lc.withContext('component', 'pusherService');\n this.#queue = new Queue();\n this.#pusher = new PushWorker(\n appConfig,\n lc,\n this.#connContextManager,\n this.#queue,\n );\n this.id = clientGroupID;\n }\n\n initConnection(selector: ConnectionSelector): Source<Downstream> {\n return this.#pusher.initConnection(selector);\n }\n\n enqueuePush(\n selector: ConnectionSelector,\n push: PushBody,\n ): Exclude<HandlerResult, StreamResult> {\n const connCtx = this.#connContextManager.getConnectionContext(selector);\n if (connCtx) {\n this.#pusher.enqueuePush(connCtx, push);\n }\n\n return {\n type: 'ok',\n };\n }\n\n async ackMutationResponses(\n requester: ConnectionSelector,\n upToID: MutationID,\n ): Promise<void> {\n const connCtx = this.#connContextManager.getConnectionContext(requester);\n if (!connCtx?.mutateContext?.url) {\n // No push URL configured, skip cleanup\n return;\n }\n\n const cleanupBody: PushBody = {\n clientGroupID: this.id,\n mutations: [\n {\n type: MutationType.Custom,\n id: 0, // Not tracked - this is fire-and-forget\n clientID: upToID.clientID,\n name: CLEANUP_RESULTS_MUTATION_NAME,\n args: [\n {\n type: 'single',\n clientGroupID: this.id,\n clientID: upToID.clientID,\n upToMutationID: upToID.id,\n },\n ],\n timestamp: Date.now(),\n },\n ],\n pushVersion: 1,\n timestamp: Date.now(),\n requestID: `cleanup-${this.id}-${upToID.clientID}-${upToID.id}`,\n };\n\n try {\n await fetchFromAPIServer(\n mutateResponseSchema,\n 'push',\n this.#lc,\n connCtx,\n {appID: this.#config.app.id, shardNum: this.#config.shard.num},\n cleanupBody,\n {operation: 'cleanup', cleanupType: 'single'},\n );\n } catch (e) {\n this.#lc.warn?.('Failed to send cleanup mutation', {\n error: getErrorMessage(e),\n });\n }\n }\n\n /**\n * Bulk cleanup is routed through the requester's push context.\n *\n * This assumes the client group shares a compatible push endpoint/auth\n * context.\n */\n async deleteClientMutations(\n requester: ConnectionSelector,\n clientIDs: string[],\n ): Promise<void> {\n if (clientIDs.length === 0) {\n return;\n }\n\n const connCtx = this.#connContextManager.getConnectionContext(requester);\n if (!connCtx?.mutateContext?.url) {\n // No push URL configured, skip cleanup\n return;\n }\n\n const cleanupBody: PushBody = {\n clientGroupID: this.id,\n mutations: [\n {\n type: MutationType.Custom,\n id: 0, // Not tracked - this is fire-and-forget\n clientID: clientIDs[0], // Use first client as sender\n name: CLEANUP_RESULTS_MUTATION_NAME,\n args: [\n {\n type: 'bulk',\n clientGroupID: this.id,\n clientIDs,\n },\n ],\n timestamp: Date.now(),\n },\n ],\n pushVersion: 1,\n timestamp: Date.now(),\n requestID: `cleanup-bulk-${this.id}-${Date.now()}`,\n };\n\n try {\n await fetchFromAPIServer(\n mutateResponseSchema,\n 'push',\n this.#lc,\n connCtx,\n {appID: this.#config.app.id, shardNum: this.#config.shard.num},\n cleanupBody,\n {operation: 'cleanup', cleanupType: 'bulk'},\n );\n } catch (e) {\n this.#lc.warn?.('Failed to send bulk cleanup mutation', {\n error: getErrorMessage(e),\n });\n }\n }\n\n ref() {\n assert(!this.#isStopped, 'PusherService is already stopped');\n ++this.#refCount;\n }\n\n unref() {\n assert(!this.#isStopped, 'PusherService is already stopped');\n --this.#refCount;\n if (this.#refCount <= 0) {\n void this.stop();\n }\n }\n\n hasRefs(): boolean {\n return this.#refCount > 0;\n }\n\n run(): Promise<void> {\n this.#stopped = this.#pusher.run();\n return this.#stopped;\n }\n\n stop(): Promise<void> {\n if (this.#isStopped) {\n return must(this.#stopped, 'Stop was called before `run`');\n }\n this.#isStopped = true;\n this.#queue.enqueue('stop');\n return must(this.#stopped, 'Stop was called before `run`');\n }\n}\n\ntype PusherEntry = {\n push: PushBody;\n connCtx: ConnectionContext;\n};\ntype PusherEntryOrStop = PusherEntry | 'stop';\n\n/**\n * Awaits items in the queue then drains and sends them all\n * to the user's API server.\n */\nclass PushWorker {\n readonly #connContextManager: ConnectionContextManager;\n readonly #queue: Queue<PusherEntryOrStop>;\n readonly #lc: LogContext;\n readonly #config: Config;\n readonly #clients: Map<\n string,\n {wsID: string; downstream: Subscription<Downstream>}\n >;\n\n readonly #customMutations = getOrCreateCounter(\n 'mutation',\n 'custom',\n 'Number of custom mutations processed',\n );\n readonly #pushes = getOrCreateCounter(\n 'mutation',\n 'pushes',\n 'Number of pushes processed by the pusher',\n );\n\n constructor(\n config: Config,\n lc: LogContext,\n connContextManager: ConnectionContextManager,\n queue: Queue<PusherEntryOrStop>,\n ) {\n this.#lc = lc.withContext('component', 'pusher');\n this.#connContextManager = connContextManager;\n this.#queue = queue;\n this.#config = config;\n this.#clients = new Map();\n }\n\n /**\n * Returns a new downstream stream if the clientID,wsID pair has not been seen before.\n * If a clientID already exists with a different wsID, that client's downstream is cancelled.\n */\n initConnection(selector: ConnectionSelector): Subscription<Downstream> {\n const existing = this.#clients.get(selector.clientID);\n if (existing && existing.wsID === selector.wsID) {\n // already initialized for this socket\n throw new Error('Connection was already initialized');\n }\n\n // client is back on a new connection\n if (existing) {\n existing.downstream.cancel();\n }\n\n const downstream = Subscription.create<Downstream>({\n cleanup: () => {\n this.#clients.delete(selector.clientID);\n },\n });\n this.#clients.set(selector.clientID, {\n wsID: selector.wsID,\n downstream,\n });\n return downstream;\n }\n\n enqueuePush(connCtx: ConnectionContext, push: PushBody) {\n this.#queue.enqueue({\n push,\n connCtx,\n });\n }\n\n async run() {\n for (;;) {\n const task = await this.#queue.dequeue();\n const rest = this.#queue.drain();\n const [pushes, terminate] = combinePushes([task, ...rest]);\n for (const push of pushes) {\n const parentContext = push.push.traceparent\n ? propagation.extract(ROOT_CONTEXT, {\n traceparent: push.push.traceparent,\n })\n : context.active();\n const response = await context.with(parentContext, () =>\n this.#processPush(push),\n );\n await this.#fanOutResponses(response);\n }\n\n if (terminate) {\n break;\n }\n }\n }\n\n /**\n * 1. If the entire `push` fails, we send the error to relevant clients.\n * 2. If the push succeeds, we look for any mutation failure that should cause the connection to terminate\n * and terminate the connection for those clients.\n */\n #fanOutResponses(response: MutateResponse) {\n const connectionTerminations: (() => void)[] = [];\n\n // if the entire push failed, send that to the client.\n if (\n ('kind' in response && response.kind === ErrorKind.PushFailed) ||\n 'error' in response\n ) {\n this.#lc.warn?.(\n 'The server behind ZERO_MUTATE_URL returned a push error.',\n response,\n );\n // TODO(0xcadams): Fanout is keyed only by clientID here. If a response arrives\n // after reconnect or re-auth, `#clients.get(clientID)` may point at a\n // newer wsID/revision and fail the replacement downstream instead.\n const groupedMutationIDs = groupBy(\n response.mutationIDs ?? [],\n m => m.clientID,\n );\n for (const [clientID, mutationIDs] of groupedMutationIDs) {\n const client = this.#clients.get(clientID);\n if (!client) {\n continue;\n }\n\n // We do not resolve mutations on the client if the push fails\n // as those mutations will be retried.\n if ('error' in response) {\n // This error code path will eventually be removed when we\n // no longer support the legacy push error format.\n const pushFailedBody: PushFailedBody =\n response.error === 'http'\n ? {\n kind: ErrorKind.PushFailed,\n origin: ErrorOrigin.ZeroCache,\n reason: ErrorReason.HTTP,\n status: response.status,\n bodyPreview: response.details,\n mutationIDs,\n message: `Fetch from API server returned non-OK status ${response.status}`,\n }\n : response.error === 'unsupportedPushVersion'\n ? {\n kind: ErrorKind.PushFailed,\n origin: ErrorOrigin.Server,\n reason: ErrorReason.UnsupportedPushVersion,\n mutationIDs,\n message: `Unsupported push version`,\n }\n : {\n kind: ErrorKind.PushFailed,\n origin: ErrorOrigin.Server,\n reason: ErrorReason.Internal,\n mutationIDs,\n message:\n response.error === 'zeroPusher'\n ? response.details\n : response.error === 'unsupportedSchemaVersion'\n ? 'Unsupported schema version'\n : 'An unknown error occurred while pushing to the API server',\n };\n\n this.#failDownstream(client.downstream, pushFailedBody);\n } else {\n this.#failDownstream(client.downstream, response);\n }\n }\n } else {\n // Look for mutations results that should cause us to terminate the connection\n // TODO(0xcadams): Same stale-routing issue as above: fatal mutation results are\n // still mapped to the current downstream by clientID only.\n const groupedMutations = groupBy(response.mutations, m => m.id.clientID);\n for (const [clientID, mutations] of groupedMutations) {\n const client = this.#clients.get(clientID);\n if (!client) {\n continue;\n }\n\n let failure: PushFailedBody | undefined;\n let i = 0;\n for (; i < mutations.length; i++) {\n const m = mutations[i];\n if ('error' in m.result) {\n this.#lc.warn?.(\n 'The server behind ZERO_MUTATE_URL returned a mutation error.',\n m.result,\n );\n }\n // This error code path will eventually be removed,\n // keeping this for backwards compatibility, but the server\n // should now return a PushFailedBody with the mutationIDs\n if ('error' in m.result && m.result.error === 'oooMutation') {\n failure = {\n kind: ErrorKind.PushFailed,\n origin: ErrorOrigin.Server,\n reason: ErrorReason.OutOfOrderMutation,\n message: 'mutation was out of order',\n details: m.result.details,\n mutationIDs: mutations.map(m => ({\n clientID: m.id.clientID,\n id: m.id.id,\n })),\n };\n break;\n }\n }\n\n if (failure && i < mutations.length - 1) {\n this.#lc.warn?.(\n 'push-response contains mutations after a mutation which should fatal the connection',\n );\n }\n\n if (failure) {\n connectionTerminations.push(() =>\n this.#failDownstream(client.downstream, failure),\n );\n }\n }\n }\n\n connectionTerminations.forEach(cb => cb());\n }\n\n async #processPush(entry: PusherEntry): Promise<MutateResponse> {\n this.#customMutations.add(entry.push.mutations.length, {\n clientGroupID: entry.push.clientGroupID,\n });\n this.#pushes.add(1, {\n clientGroupID: entry.push.clientGroupID,\n });\n\n // Record custom mutations for telemetry\n recordMutation('custom', entry.push.mutations.length);\n\n const url = must(\n entry.connCtx.mutateContext.url,\n 'ZERO_MUTATE_URL is not set',\n );\n\n this.#lc.debug?.(\n 'pushing to',\n url,\n 'with',\n entry.push.mutations.length,\n 'mutations',\n );\n\n let mutationIDs: MutationID[] = [];\n\n try {\n mutationIDs = entry.push.mutations.map(m => ({\n id: m.id,\n clientID: m.clientID,\n }));\n\n const response = await fetchFromAPIServer(\n mutateResponseSchema,\n 'push',\n this.#lc,\n entry.connCtx,\n {\n appID: this.#config.app.id,\n shardNum: this.#config.shard.num,\n },\n entry.push,\n {operation: 'mutate'},\n );\n if (\n ('kind' in response && response.kind === ErrorKind.PushFailed) ||\n 'error' in response\n ) {\n if (isAuthErrorBody(response)) {\n this.#lc.warn?.('Push auth failed; invalidating connection', {\n clientID: entry.connCtx.clientID,\n response: 'kind' in response ? response.message : undefined,\n });\n this.#connContextManager.failConnection(\n entry.connCtx,\n entry.connCtx.revision,\n );\n }\n return response;\n }\n // A successful push also validates this connection's current auth snapshot.\n // That lets later shared work reuse it without trusting stale credentials.\n this.#connContextManager.validateConnection(\n entry.connCtx,\n entry.connCtx.revision,\n 'kind' in response &&\n response.kind === 'MutateResponse' &&\n response?.userID !== undefined\n ? {kind: 'server-validated', validatedUserID: response.userID}\n : {kind: 'client-fallback'},\n );\n return response;\n } catch (e) {\n if (isProtocolError(e) && e.errorBody.kind === ErrorKind.PushFailed) {\n const response = {\n ...e.errorBody,\n mutationIDs,\n } as const satisfies PushFailedBody;\n if (isAuthErrorBody(response)) {\n this.#lc.warn?.('Push auth failed; invalidating connection', {\n clientID: entry.connCtx.clientID,\n response: 'kind' in response ? response.message : undefined,\n });\n this.#connContextManager.failConnection(\n entry.connCtx,\n entry.connCtx.revision,\n );\n }\n return response;\n }\n\n if (isProtocolError(e) && isAuthErrorBody(e.errorBody)) {\n // The push completed far enough for local validation to reject the\n // connection, so invalidate it and surface the result as PushFailed.\n this.#lc.warn?.('Push validation failed; invalidating connection', {\n clientID: entry.connCtx.clientID,\n response: e.message,\n });\n this.#connContextManager.failConnection(\n entry.connCtx,\n entry.connCtx.revision,\n );\n return {\n kind: ErrorKind.PushFailed,\n origin: ErrorOrigin.ZeroCache,\n reason: ErrorReason.HTTP,\n message: e.message,\n status: 401,\n mutationIDs,\n } as const satisfies PushFailedBody;\n }\n\n return {\n kind: ErrorKind.PushFailed,\n origin: ErrorOrigin.ZeroCache,\n reason: ErrorReason.Internal,\n message: `Failed to push: ${getErrorMessage(e)}`,\n mutationIDs,\n } as const satisfies PushFailedBody;\n }\n }\n\n #failDownstream(\n downstream: Subscription<Downstream>,\n errorBody: PushFailedBody,\n ): void {\n downstream.fail(new ProtocolErrorWithLevel(errorBody, 'warn'));\n }\n}\n\n/**\n * Pushes for different clients, sockets, or auth revisions could be interleaved.\n *\n * In order to batch safely, we only combine pushes from the same\n * clientID/wsID/revision snapshot.\n */\nexport function combinePushes(\n entries: readonly (PusherEntryOrStop | undefined)[],\n): [PusherEntry[], boolean] {\n const pushesByConnection = new Map<string, PusherEntry[]>();\n\n function collect() {\n const ret: PusherEntry[] = [];\n for (const entries of pushesByConnection.values()) {\n const composite: PusherEntry = {\n ...entries[0],\n push: {\n ...entries[0].push,\n mutations: [],\n },\n };\n ret.push(composite);\n for (const entry of entries) {\n assertAreCompatiblePushes(composite, entry);\n composite.push.mutations.push(...entry.push.mutations);\n }\n }\n return ret;\n }\n\n for (const entry of entries) {\n if (entry === 'stop' || entry === undefined) {\n return [collect(), true];\n }\n\n const key = `${entry.connCtx.clientID}:${entry.connCtx.wsID}:${entry.connCtx.revision}`;\n getOrInsertComputed(pushesByConnection, key, newArray).push(entry);\n }\n\n return [collect(), false] as const;\n}\n\n// These invariants should always be true for a given clientID.\n// If they are not, we have a bug in the code somewhere.\nfunction assertAreCompatiblePushes(left: PusherEntry, right: PusherEntry) {\n assert(\n left.connCtx.clientID === right.connCtx.clientID,\n 'clientID must be the same for all pushes',\n );\n assert(\n left.connCtx.wsID === right.connCtx.wsID,\n 'wsID must be the same for all pushes',\n );\n assert(\n left.connCtx.revision === right.connCtx.revision,\n 'revision must be the same for all pushes',\n );\n assert(\n authEquals(left.connCtx.auth, right.connCtx.auth),\n 'auth must be the same for all pushes with the same clientID',\n );\n assert(\n left.push.schemaVersion === right.push.schemaVersion,\n 'schemaVersion must be the same for all pushes with the same clientID',\n );\n assert(\n left.push.pushVersion === right.push.pushVersion,\n 'pushVersion must be the same for all pushes with the same clientID',\n );\n assert(\n left.connCtx.mutateContext.headerOptions.cookie ===\n right.connCtx.mutateContext.headerOptions.cookie,\n 'httpCookie must be the same for all pushes with the same clientID',\n );\n assert(\n left.connCtx.mutateContext.headerOptions.origin ===\n right.connCtx.mutateContext.headerOptions.origin,\n 'origin must be the same for all pushes with the same clientID',\n );\n assert(\n left.connCtx.user.id === right.connCtx.user.id,\n 'userID must be the same for all pushes with the same clientID',\n );\n assert(\n left.connCtx.mutateContext.url === right.connCtx.mutateContext.url,\n 'userPushURL must be the same for all pushes with the same clientID',\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoEA,IAAa,gBAAb,MAAsD;CACpD;CACA;CACA;CACA;CACA;CACA;CACA;CACA,YAAY;CACZ,aAAa;CAEb,YACE,WACA,IACA,eACA,oBACA;EACA,KAAK,sBAAsB;EAC3B,KAAK,UAAU;EACf,KAAK,MAAM,GAAG,YAAY,aAAa,eAAe;EACtD,KAAK,SAAS,IAAI,MAAM;EACxB,KAAK,UAAU,IAAI,WACjB,WACA,IACA,KAAK,qBACL,KAAK,MACP;EACA,KAAK,KAAK;CACZ;CAEA,eAAe,UAAkD;EAC/D,OAAO,KAAK,QAAQ,eAAe,QAAQ;CAC7C;CAEA,YACE,UACA,MACsC;EACtC,MAAM,UAAU,KAAK,oBAAoB,qBAAqB,QAAQ;EACtE,IAAI,SACF,KAAK,QAAQ,YAAY,SAAS,IAAI;EAGxC,OAAO,EACL,MAAM,KACR;CACF;CAEA,MAAM,qBACJ,WACA,QACe;EACf,MAAM,UAAU,KAAK,oBAAoB,qBAAqB,SAAS;EACvE,IAAI,CAAC,SAAS,eAAe,KAE3B;EAGF,MAAM,cAAwB;GAC5B,eAAe,KAAK;GACpB,WAAW,CACT;IACE,MAAMA;IACN,IAAI;IACJ,UAAU,OAAO;IACjB,MAAM;IACN,MAAM,CACJ;KACE,MAAM;KACN,eAAe,KAAK;KACpB,UAAU,OAAO;KACjB,gBAAgB,OAAO;IACzB,CACF;IACA,WAAW,KAAK,IAAI;GACtB,CACF;GACA,aAAa;GACb,WAAW,KAAK,IAAI;GACpB,WAAW,WAAW,KAAK,GAAG,GAAG,OAAO,SAAS,GAAG,OAAO;EAC7D;EAEA,IAAI;GACF,MAAM,mBACJ,sBACA,QACA,KAAK,KACL,SACA;IAAC,OAAO,KAAK,QAAQ,IAAI;IAAI,UAAU,KAAK,QAAQ,MAAM;GAAG,GAC7D,aACA;IAAC,WAAW;IAAW,aAAa;GAAQ,CAC9C;EACF,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,mCAAmC,EACjD,OAAO,gBAAgB,CAAC,EAC1B,CAAC;EACH;CACF;;;;;;;CAQA,MAAM,sBACJ,WACA,WACe;EACf,IAAI,UAAU,WAAW,GACvB;EAGF,MAAM,UAAU,KAAK,oBAAoB,qBAAqB,SAAS;EACvE,IAAI,CAAC,SAAS,eAAe,KAE3B;EAGF,MAAM,cAAwB;GAC5B,eAAe,KAAK;GACpB,WAAW,CACT;IACE,MAAMA;IACN,IAAI;IACJ,UAAU,UAAU;IACpB,MAAM;IACN,MAAM,CACJ;KACE,MAAM;KACN,eAAe,KAAK;KACpB;IACF,CACF;IACA,WAAW,KAAK,IAAI;GACtB,CACF;GACA,aAAa;GACb,WAAW,KAAK,IAAI;GACpB,WAAW,gBAAgB,KAAK,GAAG,GAAG,KAAK,IAAI;EACjD;EAEA,IAAI;GACF,MAAM,mBACJ,sBACA,QACA,KAAK,KACL,SACA;IAAC,OAAO,KAAK,QAAQ,IAAI;IAAI,UAAU,KAAK,QAAQ,MAAM;GAAG,GAC7D,aACA;IAAC,WAAW;IAAW,aAAa;GAAM,CAC5C;EACF,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,wCAAwC,EACtD,OAAO,gBAAgB,CAAC,EAC1B,CAAC;EACH;CACF;CAEA,MAAM;EACJ,OAAO,CAAC,KAAK,YAAY,kCAAkC;EAC3D,EAAE,KAAK;CACT;CAEA,QAAQ;EACN,OAAO,CAAC,KAAK,YAAY,kCAAkC;EAC3D,EAAE,KAAK;EACP,IAAI,KAAK,aAAa,GACpB,KAAU,KAAK;CAEnB;CAEA,UAAmB;EACjB,OAAO,KAAK,YAAY;CAC1B;CAEA,MAAqB;EACnB,KAAK,WAAW,KAAK,QAAQ,IAAI;EACjC,OAAO,KAAK;CACd;CAEA,OAAsB;EACpB,IAAI,KAAK,YACP,OAAO,KAAK,KAAK,UAAU,8BAA8B;EAE3D,KAAK,aAAa;EAClB,KAAK,OAAO,QAAQ,MAAM;EAC1B,OAAO,KAAK,KAAK,UAAU,8BAA8B;CAC3D;AACF;;;;;AAYA,IAAM,aAAN,MAAiB;CACf;CACA;CACA;CACA;CACA;CAKA,mBAA4B,mBAC1B,YACA,UACA,sCACF;CACA,UAAmB,mBACjB,YACA,UACA,0CACF;CAEA,YACE,QACA,IACA,oBACA,OACA;EACA,KAAK,MAAM,GAAG,YAAY,aAAa,QAAQ;EAC/C,KAAK,sBAAsB;EAC3B,KAAK,SAAS;EACd,KAAK,UAAU;EACf,KAAK,2BAAW,IAAI,IAAI;CAC1B;;;;;CAMA,eAAe,UAAwD;EACrE,MAAM,WAAW,KAAK,SAAS,IAAI,SAAS,QAAQ;EACpD,IAAI,YAAY,SAAS,SAAS,SAAS,MAEzC,MAAM,IAAI,MAAM,oCAAoC;EAItD,IAAI,UACF,SAAS,WAAW,OAAO;EAG7B,MAAM,aAAa,aAAa,OAAmB,EACjD,eAAe;GACb,KAAK,SAAS,OAAO,SAAS,QAAQ;EACxC,EACF,CAAC;EACD,KAAK,SAAS,IAAI,SAAS,UAAU;GACnC,MAAM,SAAS;GACf;EACF,CAAC;EACD,OAAO;CACT;CAEA,YAAY,SAA4B,MAAgB;EACtD,KAAK,OAAO,QAAQ;GAClB;GACA;EACF,CAAC;CACH;CAEA,MAAM,MAAM;EACV,SAAS;GAGP,MAAM,CAAC,QAAQ,aAAa,cAAc,CAAC,MAFxB,KAAK,OAAO,QAAQ,GAEU,GADpC,KAAK,OAAO,MAC2B,CAAI,CAAC;GACzD,KAAK,MAAM,QAAQ,QAAQ;IACzB,MAAM,gBAAgB,KAAK,KAAK,cAC5B,YAAY,QAAQ,cAAc,EAChC,aAAa,KAAK,KAAK,YACzB,CAAC,IACD,QAAQ,OAAO;IACnB,MAAM,WAAW,MAAM,QAAQ,KAAK,qBAClC,KAAK,aAAa,IAAI,CACxB;IACA,MAAM,KAAK,iBAAiB,QAAQ;GACtC;GAEA,IAAI,WACF;EAEJ;CACF;;;;;;CAOA,iBAAiB,UAA0B;EACzC,MAAM,yBAAyC,CAAC;EAGhD,IACG,UAAU,YAAY,SAAS,SAAS,gBACzC,WAAW,UACX;GACA,KAAK,IAAI,OACP,4DACA,QACF;GAIA,MAAM,qBAAqB,QACzB,SAAS,eAAe,CAAC,IACzB,MAAK,EAAE,QACT;GACA,KAAK,MAAM,CAAC,UAAU,gBAAgB,oBAAoB;IACxD,MAAM,SAAS,KAAK,SAAS,IAAI,QAAQ;IACzC,IAAI,CAAC,QACH;IAKF,IAAI,WAAW,UAAU;KAGvB,MAAM,iBACJ,SAAS,UAAU,SACf;MACE,MAAMC;MACN,QAAQC;MACR,QAAQC;MACR,QAAQ,SAAS;MACjB,aAAa,SAAS;MACtB;MACA,SAAS,gDAAgD,SAAS;KACpE,IACA,SAAS,UAAU,2BACjB;MACE,MAAMF;MACN,QAAQG;MACR,QAAQC;MACR;MACA,SAAS;KACX,IACA;MACE,MAAMJ;MACN,QAAQG;MACR,QAAQE;MACR;MACA,SACE,SAAS,UAAU,eACf,SAAS,UACT,SAAS,UAAU,6BACjB,+BACA;KACV;KAER,KAAK,gBAAgB,OAAO,YAAY,cAAc;IACxD,OACE,KAAK,gBAAgB,OAAO,YAAY,QAAQ;GAEpD;EACF,OAAO;GAIL,MAAM,mBAAmB,QAAQ,SAAS,YAAW,MAAK,EAAE,GAAG,QAAQ;GACvE,KAAK,MAAM,CAAC,UAAU,cAAc,kBAAkB;IACpD,MAAM,SAAS,KAAK,SAAS,IAAI,QAAQ;IACzC,IAAI,CAAC,QACH;IAGF,IAAI;IACJ,IAAI,IAAI;IACR,OAAO,IAAI,UAAU,QAAQ,KAAK;KAChC,MAAM,IAAI,UAAU;KACpB,IAAI,WAAW,EAAE,QACf,KAAK,IAAI,OACP,gEACA,EAAE,MACJ;KAKF,IAAI,WAAW,EAAE,UAAU,EAAE,OAAO,UAAU,eAAe;MAC3D,UAAU;OACR,MAAML;OACN,QAAQG;OACR,QAAQG;OACR,SAAS;OACT,SAAS,EAAE,OAAO;OAClB,aAAa,UAAU,KAAI,OAAM;QAC/B,UAAU,EAAE,GAAG;QACf,IAAI,EAAE,GAAG;OACX,EAAE;MACJ;MACA;KACF;IACF;IAEA,IAAI,WAAW,IAAI,UAAU,SAAS,GACpC,KAAK,IAAI,OACP,qFACF;IAGF,IAAI,SACF,uBAAuB,WACrB,KAAK,gBAAgB,OAAO,YAAY,OAAO,CACjD;GAEJ;EACF;EAEA,uBAAuB,SAAQ,OAAM,GAAG,CAAC;CAC3C;CAEA,MAAM,aAAa,OAA6C;EAC9D,KAAK,iBAAiB,IAAI,MAAM,KAAK,UAAU,QAAQ,EACrD,eAAe,MAAM,KAAK,cAC5B,CAAC;EACD,KAAK,QAAQ,IAAI,GAAG,EAClB,eAAe,MAAM,KAAK,cAC5B,CAAC;EAGD,eAAe,UAAU,MAAM,KAAK,UAAU,MAAM;EAEpD,MAAM,MAAM,KACV,MAAM,QAAQ,cAAc,KAC5B,4BACF;EAEA,KAAK,IAAI,QACP,cACA,KACA,QACA,MAAM,KAAK,UAAU,QACrB,WACF;EAEA,IAAI,cAA4B,CAAC;EAEjC,IAAI;GACF,cAAc,MAAM,KAAK,UAAU,KAAI,OAAM;IAC3C,IAAI,EAAE;IACN,UAAU,EAAE;GACd,EAAE;GAEF,MAAM,WAAW,MAAM,mBACrB,sBACA,QACA,KAAK,KACL,MAAM,SACN;IACE,OAAO,KAAK,QAAQ,IAAI;IACxB,UAAU,KAAK,QAAQ,MAAM;GAC/B,GACA,MAAM,MACN,EAAC,WAAW,SAAQ,CACtB;GACA,IACG,UAAU,YAAY,SAAS,SAAS,gBACzC,WAAW,UACX;IACA,IAAI,gBAAgB,QAAQ,GAAG;KAC7B,KAAK,IAAI,OAAO,6CAA6C;MAC3D,UAAU,MAAM,QAAQ;MACxB,UAAU,UAAU,WAAW,SAAS,UAAU,KAAA;KACpD,CAAC;KACD,KAAK,oBAAoB,eACvB,MAAM,SACN,MAAM,QAAQ,QAChB;IACF;IACA,OAAO;GACT;GAGA,KAAK,oBAAoB,mBACvB,MAAM,SACN,MAAM,QAAQ,UACd,UAAU,YACR,SAAS,SAAS,oBAClB,UAAU,WAAW,KAAA,IACnB;IAAC,MAAM;IAAoB,iBAAiB,SAAS;GAAM,IAC3D,EAAC,MAAM,kBAAiB,CAC9B;GACA,OAAO;EACT,SAAS,GAAG;GACV,IAAI,gBAAgB,CAAC,KAAK,EAAE,UAAU,SAAS,cAAsB;IACnE,MAAM,WAAW;KACf,GAAG,EAAE;KACL;IACF;IACA,IAAI,gBAAgB,QAAQ,GAAG;KAC7B,KAAK,IAAI,OAAO,6CAA6C;MAC3D,UAAU,MAAM,QAAQ;MACxB,UAAU,UAAU,WAAW,SAAS,UAAU,KAAA;KACpD,CAAC;KACD,KAAK,oBAAoB,eACvB,MAAM,SACN,MAAM,QAAQ,QAChB;IACF;IACA,OAAO;GACT;GAEA,IAAI,gBAAgB,CAAC,KAAK,gBAAgB,EAAE,SAAS,GAAG;IAGtD,KAAK,IAAI,OAAO,mDAAmD;KACjE,UAAU,MAAM,QAAQ;KACxB,UAAU,EAAE;IACd,CAAC;IACD,KAAK,oBAAoB,eACvB,MAAM,SACN,MAAM,QAAQ,QAChB;IACA,OAAO;KACL,MAAMN;KACN,QAAQC;KACR,QAAQC;KACR,SAAS,EAAE;KACX,QAAQ;KACR;IACF;GACF;GAEA,OAAO;IACL,MAAMF;IACN,QAAQC;IACR,QAAQI;IACR,SAAS,mBAAmB,gBAAgB,CAAC;IAC7C;GACF;EACF;CACF;CAEA,gBACE,YACA,WACM;EACN,WAAW,KAAK,IAAI,uBAAuB,WAAW,MAAM,CAAC;CAC/D;AACF;;;;;;;AAQA,SAAgB,cACd,SAC0B;CAC1B,MAAM,qCAAqB,IAAI,IAA2B;CAE1D,SAAS,UAAU;EACjB,MAAM,MAAqB,CAAC;EAC5B,KAAK,MAAM,WAAW,mBAAmB,OAAO,GAAG;GACjD,MAAM,YAAyB;IAC7B,GAAG,QAAQ;IACX,MAAM;KACJ,GAAG,QAAQ,EAAE,CAAC;KACd,WAAW,CAAC;IACd;GACF;GACA,IAAI,KAAK,SAAS;GAClB,KAAK,MAAM,SAAS,SAAS;IAC3B,0BAA0B,WAAW,KAAK;IAC1C,UAAU,KAAK,UAAU,KAAK,GAAG,MAAM,KAAK,SAAS;GACvD;EACF;EACA,OAAO;CACT;CAEA,KAAK,MAAM,SAAS,SAAS;EAC3B,IAAI,UAAU,UAAU,UAAU,KAAA,GAChC,OAAO,CAAC,QAAQ,GAAG,IAAI;EAGzB,MAAM,MAAM,GAAG,MAAM,QAAQ,SAAS,GAAG,MAAM,QAAQ,KAAK,GAAG,MAAM,QAAQ;EAC7E,oBAAoB,oBAAoB,KAAK,QAAQ,CAAC,CAAC,KAAK,KAAK;CACnE;CAEA,OAAO,CAAC,QAAQ,GAAG,KAAK;AAC1B;AAIA,SAAS,0BAA0B,MAAmB,OAAoB;CACxE,OACE,KAAK,QAAQ,aAAa,MAAM,QAAQ,UACxC,0CACF;CACA,OACE,KAAK,QAAQ,SAAS,MAAM,QAAQ,MACpC,sCACF;CACA,OACE,KAAK,QAAQ,aAAa,MAAM,QAAQ,UACxC,0CACF;CACA,OACE,WAAW,KAAK,QAAQ,MAAM,MAAM,QAAQ,IAAI,GAChD,6DACF;CACA,OACE,KAAK,KAAK,kBAAkB,MAAM,KAAK,eACvC,sEACF;CACA,OACE,KAAK,KAAK,gBAAgB,MAAM,KAAK,aACrC,oEACF;CACA,OACE,KAAK,QAAQ,cAAc,cAAc,WACvC,MAAM,QAAQ,cAAc,cAAc,QAC5C,mEACF;CACA,OACE,KAAK,QAAQ,cAAc,cAAc,WACvC,MAAM,QAAQ,cAAc,cAAc,QAC5C,+DACF;CACA,OACE,KAAK,QAAQ,KAAK,OAAO,MAAM,QAAQ,KAAK,IAC5C,+DACF;CACA,OACE,KAAK,QAAQ,cAAc,QAAQ,MAAM,QAAQ,cAAc,KAC/D,oEACF;AACF"}
|
|
1
|
+
{"version":3,"file":"pusher.js","names":["MutationType.Custom","ErrorKind.PushFailed","ErrorOrigin.ZeroCache","ErrorReason.HTTP","ErrorOrigin.Server","ErrorReason.UnsupportedPushVersion","ErrorReason.Internal","ErrorReason.OutOfOrderMutation"],"sources":["../../../../../../zero-cache/src/services/mutagen/pusher.ts"],"sourcesContent":["import {context, propagation, ROOT_CONTEXT} from '@opentelemetry/api';\nimport type {LogContext} from '@rocicorp/logger';\nimport {groupBy, newArray} from '../../../../shared/src/arrays.ts';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {getErrorMessage} from '../../../../shared/src/error.ts';\nimport {getOrInsertComputed} from '../../../../shared/src/map.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Queue} from '../../../../shared/src/queue.ts';\nimport type {Downstream} from '../../../../zero-protocol/src/down.ts';\nimport {ErrorKind} from '../../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../../zero-protocol/src/error-origin.ts';\nimport {ErrorReason} from '../../../../zero-protocol/src/error-reason.ts';\nimport {\n isProtocolError,\n type PushFailedBody,\n} from '../../../../zero-protocol/src/error.ts';\nimport {\n mutateResponseSchema,\n type MutateResponse,\n} from '../../../../zero-protocol/src/mutate-server.ts';\nimport type {MutationID} from '../../../../zero-protocol/src/mutation-id.ts';\nimport * as MutationType from '../../../../zero-protocol/src/mutation-type-enum.ts';\nimport {CLEANUP_RESULTS_MUTATION_NAME} from '../../../../zero-protocol/src/mutation.ts';\nimport {type PushBody} from '../../../../zero-protocol/src/push.ts';\nimport {authEquals, isAuthErrorBody} from '../../auth/auth.ts';\nimport {type ZeroConfig} from '../../config/zero-config.ts';\nimport {fetchFromAPIServer} from '../../custom/fetch.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {recordMutation} from '../../server/anonymous-otel-start.ts';\nimport {ProtocolErrorWithLevel} from '../../types/error-with-level.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {HandlerResult, StreamResult} from '../../workers/connection.ts';\nimport type {RefCountedService, Service} from '../service.ts';\nimport type {\n ConnectionContext,\n ConnectionContextManager,\n ConnectionSelector,\n} from '../view-syncer/connection-context-manager.ts';\n\nexport interface Pusher extends RefCountedService {\n initConnection(selector: ConnectionSelector): Source<Downstream>;\n enqueuePush(selector: ConnectionSelector, push: PushBody): HandlerResult;\n ackMutationResponses(\n requester: ConnectionSelector,\n upToID: MutationID,\n ): Promise<void>;\n deleteClientMutations(\n requester: ConnectionSelector,\n clientIDs: string[],\n ): Promise<void>;\n}\n\ntype Config = Pick<ZeroConfig, 'app' | 'shard'>;\n\n/**\n * Receives push messages from zero-client and forwards\n * them the the user's API server.\n *\n * If the user's API server is taking too long to process\n * the push, the PusherService will add the push to a queue\n * and send pushes in bulk the next time the user's API server\n * is available.\n *\n * - One PusherService exists per client group.\n * - Mutations for a given client are always sent in-order\n * - Mutations for different clients in the same group may be interleaved\n */\nexport class PusherService implements Service, Pusher {\n readonly id: string;\n readonly #connContextManager: ConnectionContextManager;\n readonly #pusher: PushWorker;\n readonly #queue: Queue<PusherEntryOrStop>;\n readonly #config: Config;\n readonly #lc: LogContext;\n #stopped: Promise<void> | undefined;\n #refCount = 0;\n #isStopped = false;\n\n constructor(\n appConfig: Config,\n lc: LogContext,\n clientGroupID: string,\n connContextManager: ConnectionContextManager,\n ) {\n this.#connContextManager = connContextManager;\n this.#config = appConfig;\n this.#lc = lc.withContext('component', 'pusherService');\n this.#queue = new Queue();\n this.#pusher = new PushWorker(\n appConfig,\n lc,\n this.#connContextManager,\n this.#queue,\n );\n this.id = clientGroupID;\n }\n\n initConnection(selector: ConnectionSelector): Source<Downstream> {\n return this.#pusher.initConnection(selector);\n }\n\n enqueuePush(\n selector: ConnectionSelector,\n push: PushBody,\n ): Exclude<HandlerResult, StreamResult> {\n const connCtx = this.#connContextManager.getConnectionContext(selector);\n if (connCtx) {\n this.#pusher.enqueuePush(connCtx, push);\n }\n\n return {\n type: 'ok',\n };\n }\n\n async ackMutationResponses(\n requester: ConnectionSelector,\n upToID: MutationID,\n ): Promise<void> {\n const connCtx = this.#connContextManager.getConnectionContext(requester);\n if (!connCtx?.mutateContext?.url) {\n // No push URL configured, skip cleanup\n return;\n }\n\n const cleanupBody: PushBody = {\n clientGroupID: this.id,\n mutations: [\n {\n type: MutationType.Custom,\n id: 0, // Not tracked - this is fire-and-forget\n clientID: upToID.clientID,\n name: CLEANUP_RESULTS_MUTATION_NAME,\n args: [\n {\n type: 'single',\n clientGroupID: this.id,\n clientID: upToID.clientID,\n upToMutationID: upToID.id,\n },\n ],\n timestamp: Date.now(),\n },\n ],\n pushVersion: 1,\n timestamp: Date.now(),\n requestID: `cleanup-${this.id}-${upToID.clientID}-${upToID.id}`,\n };\n\n try {\n await fetchFromAPIServer(\n mutateResponseSchema,\n 'push',\n this.#lc,\n connCtx,\n {appID: this.#config.app.id, shardNum: this.#config.shard.num},\n cleanupBody,\n {operation: 'cleanup', cleanupType: 'single'},\n );\n } catch (e) {\n this.#lc.warn?.('Failed to send cleanup mutation', {\n error: getErrorMessage(e),\n });\n }\n }\n\n /**\n * Bulk cleanup is routed through the requester's push context.\n *\n * This assumes the client group shares a compatible push endpoint/auth\n * context.\n */\n async deleteClientMutations(\n requester: ConnectionSelector,\n clientIDs: string[],\n ): Promise<void> {\n if (clientIDs.length === 0) {\n return;\n }\n\n const connCtx = this.#connContextManager.getConnectionContext(requester);\n if (!connCtx?.mutateContext?.url) {\n // No push URL configured, skip cleanup\n return;\n }\n\n const cleanupBody: PushBody = {\n clientGroupID: this.id,\n mutations: [\n {\n type: MutationType.Custom,\n id: 0, // Not tracked - this is fire-and-forget\n clientID: clientIDs[0], // Use first client as sender\n name: CLEANUP_RESULTS_MUTATION_NAME,\n args: [\n {\n type: 'bulk',\n clientGroupID: this.id,\n clientIDs,\n },\n ],\n timestamp: Date.now(),\n },\n ],\n pushVersion: 1,\n timestamp: Date.now(),\n requestID: `cleanup-bulk-${this.id}-${Date.now()}`,\n };\n\n try {\n await fetchFromAPIServer(\n mutateResponseSchema,\n 'push',\n this.#lc,\n connCtx,\n {appID: this.#config.app.id, shardNum: this.#config.shard.num},\n cleanupBody,\n {operation: 'cleanup', cleanupType: 'bulk'},\n );\n } catch (e) {\n this.#lc.warn?.('Failed to send bulk cleanup mutation', {\n error: getErrorMessage(e),\n });\n }\n }\n\n ref() {\n assert(!this.#isStopped, 'PusherService is already stopped');\n ++this.#refCount;\n }\n\n unref() {\n assert(!this.#isStopped, 'PusherService is already stopped');\n --this.#refCount;\n if (this.#refCount <= 0) {\n void this.stop();\n }\n }\n\n hasRefs(): boolean {\n return this.#refCount > 0;\n }\n\n run(): Promise<void> {\n this.#stopped = this.#pusher.run();\n return this.#stopped;\n }\n\n stop(): Promise<void> {\n if (this.#isStopped) {\n return must(this.#stopped, 'Stop was called before `run`');\n }\n this.#isStopped = true;\n this.#queue.enqueue('stop');\n return must(this.#stopped, 'Stop was called before `run`');\n }\n}\n\ntype PusherEntry = {\n push: PushBody;\n connCtx: ConnectionContext;\n};\ntype PusherEntryOrStop = PusherEntry | 'stop';\n\n/**\n * Awaits items in the queue then drains and sends them all\n * to the user's API server.\n */\nclass PushWorker {\n readonly #connContextManager: ConnectionContextManager;\n readonly #queue: Queue<PusherEntryOrStop>;\n readonly #lc: LogContext;\n readonly #config: Config;\n readonly #clients: Map<\n string,\n {wsID: string; downstream: Subscription<Downstream>}\n >;\n /**\n * For each client, the wsID of its socket whose push most recently failed.\n * Pushes queued behind the failed one on that socket are not sent: they\n * start past the failed mutation, so the API server could only reject them\n * as out of order. The socket is failed (or was already replaced), and the\n * client re-sends every unacknowledged mutation on its next socket.\n *\n * An entry is kept until the socket can no longer queue pushes (it closed or\n * the client reconnected) and the queue has been drained since, so it is not\n * removed while pushes from the failed socket may still be queued.\n */\n readonly #failedSockets = new Map<string, string>();\n\n readonly #customMutations = getOrCreateCounter(\n 'mutation',\n 'custom',\n 'Number of custom mutations processed',\n );\n readonly #pushes = getOrCreateCounter(\n 'mutation',\n 'pushes',\n 'Number of pushes processed by the pusher',\n );\n\n constructor(\n config: Config,\n lc: LogContext,\n connContextManager: ConnectionContextManager,\n queue: Queue<PusherEntryOrStop>,\n ) {\n this.#lc = lc.withContext('component', 'pusher');\n this.#connContextManager = connContextManager;\n this.#queue = queue;\n this.#config = config;\n this.#clients = new Map();\n }\n\n /**\n * Returns a new downstream stream if the clientID,wsID pair has not been seen before.\n * If a clientID already exists with a different wsID, that client's downstream is cancelled.\n */\n initConnection(selector: ConnectionSelector): Subscription<Downstream> {\n const existing = this.#clients.get(selector.clientID);\n if (existing && existing.wsID === selector.wsID) {\n // already initialized for this socket\n throw new Error('Connection was already initialized');\n }\n\n // client is back on a new connection\n if (existing) {\n existing.downstream.cancel();\n }\n\n const downstream = Subscription.create<Downstream>({\n cleanup: () => {\n this.#clients.delete(selector.clientID);\n },\n });\n this.#clients.set(selector.clientID, {\n wsID: selector.wsID,\n downstream,\n });\n return downstream;\n }\n\n enqueuePush(connCtx: ConnectionContext, push: PushBody) {\n this.#queue.enqueue({\n push,\n connCtx,\n });\n }\n\n async run() {\n for (;;) {\n // Failed sockets that can no longer queue pushes. Everything they queued\n // is in the next drain, so they are forgotten once it has been handled.\n const closedSockets = [...this.#failedSockets].filter(\n ([clientID, wsID]) =>\n !this.#connContextManager.getConnectionContext({clientID, wsID}),\n );\n const task = await this.#queue.dequeue();\n const rest = this.#queue.drain();\n const [pushes, terminate] = combinePushes([task, ...rest]);\n for (const push of pushes) {\n const {clientID, wsID} = push.connCtx;\n if (this.#failedSockets.get(clientID) === wsID) {\n this.#lc.debug?.(\n 'Not sending push queued behind a failed push on the same socket; the client re-sends it on its next socket',\n {clientID, wsID, mutations: push.push.mutations.length},\n );\n continue;\n }\n const parentContext = push.push.traceparent\n ? propagation.extract(ROOT_CONTEXT, {\n traceparent: push.push.traceparent,\n })\n : context.active();\n const response = await context.with(parentContext, () =>\n this.#processPush(push),\n );\n await this.#fanOutResponses(push.connCtx, response);\n }\n\n for (const [clientID, wsID] of closedSockets) {\n if (this.#failedSockets.get(clientID) === wsID) {\n this.#failedSockets.delete(clientID);\n }\n }\n\n if (terminate) {\n break;\n }\n }\n }\n\n /**\n * 1. If the entire `push` fails, we send the error to `origin`, the\n * connection the push was sent on.\n * 2. If the push succeeds, we look for any mutation failure that should cause the connection to terminate\n * and terminate `origin`.\n *\n * Only `origin` is failed, even when the push carried other clients'\n * mutations (the client group's commit chain includes other tabs' pending\n * mutations). A client tracks what it has sent per socket, so the gap a\n * failed push leaves is on `origin` alone. Other clients send their own\n * mutations on their own sockets, and see any lasting error there.\n */\n #fanOutResponses(origin: ConnectionContext, response: MutateResponse) {\n const connectionTerminations: (() => void)[] = [];\n\n // if the entire push failed, send that to the client.\n if (\n ('kind' in response && response.kind === ErrorKind.PushFailed) ||\n 'error' in response\n ) {\n this.#lc.warn?.(\n 'The server behind ZERO_MUTATE_URL returned a push error.',\n response,\n );\n const mutationIDs = response.mutationIDs ?? [];\n if (mutationIDs.length > 0) {\n // We do not resolve mutations on the client if the push fails\n // as those mutations will be retried.\n if ('error' in response) {\n // This error code path will eventually be removed when we\n // no longer support the legacy push error format.\n const pushFailedBody: PushFailedBody =\n response.error === 'http'\n ? {\n kind: ErrorKind.PushFailed,\n origin: ErrorOrigin.ZeroCache,\n reason: ErrorReason.HTTP,\n status: response.status,\n bodyPreview: response.details,\n mutationIDs,\n message: `Fetch from API server returned non-OK status ${response.status}`,\n }\n : response.error === 'unsupportedPushVersion'\n ? {\n kind: ErrorKind.PushFailed,\n origin: ErrorOrigin.Server,\n reason: ErrorReason.UnsupportedPushVersion,\n mutationIDs,\n message: `Unsupported push version`,\n }\n : {\n kind: ErrorKind.PushFailed,\n origin: ErrorOrigin.Server,\n reason: ErrorReason.Internal,\n mutationIDs,\n message:\n response.error === 'zeroPusher'\n ? response.details\n : response.error === 'unsupportedSchemaVersion'\n ? 'Unsupported schema version'\n : 'An unknown error occurred while pushing to the API server',\n };\n\n this.#failOrigin(origin, pushFailedBody);\n } else {\n this.#failOrigin(origin, response);\n }\n }\n } else {\n // Look for mutations results that should cause us to terminate the connection\n const groupedMutations = groupBy(response.mutations, m => m.id.clientID);\n for (const mutations of groupedMutations.values()) {\n let failure: PushFailedBody | undefined;\n let i = 0;\n for (; i < mutations.length; i++) {\n const m = mutations[i];\n if ('error' in m.result) {\n this.#lc.warn?.(\n 'The server behind ZERO_MUTATE_URL returned a mutation error.',\n m.result,\n );\n }\n // This error code path will eventually be removed,\n // keeping this for backwards compatibility, but the server\n // should now return a PushFailedBody with the mutationIDs\n if ('error' in m.result && m.result.error === 'oooMutation') {\n failure = {\n kind: ErrorKind.PushFailed,\n origin: ErrorOrigin.Server,\n reason: ErrorReason.OutOfOrderMutation,\n message: 'mutation was out of order',\n details: m.result.details,\n mutationIDs: mutations.map(m => ({\n clientID: m.id.clientID,\n id: m.id.id,\n })),\n };\n break;\n }\n }\n\n if (failure && i < mutations.length - 1) {\n this.#lc.warn?.(\n 'push-response contains mutations after a mutation which should fatal the connection',\n );\n }\n\n if (failure) {\n connectionTerminations.push(() => this.#failOrigin(origin, failure));\n }\n }\n }\n\n connectionTerminations.forEach(cb => cb());\n }\n\n /**\n * Fails `origin` after a push sent on it failed, and records it in\n * `#failedSockets` so the pushes it queued behind that one are not sent.\n *\n * If the client has reconnected since sending the push, the old socket is\n * already gone and the client re-sends its mutations on the new one. Failing\n * the new socket would put the client in its error state, rejecting its\n * outstanding mutations, for a failure it has already recovered from.\n */\n #failOrigin(origin: ConnectionContext, errorBody: PushFailedBody): void {\n this.#failedSockets.set(origin.clientID, origin.wsID);\n const client = this.#clients.get(origin.clientID);\n if (client?.wsID === origin.wsID) {\n this.#failDownstream(client.downstream, errorBody);\n }\n }\n\n async #processPush(entry: PusherEntry): Promise<MutateResponse> {\n this.#customMutations.add(entry.push.mutations.length, {\n clientGroupID: entry.push.clientGroupID,\n });\n this.#pushes.add(1, {\n clientGroupID: entry.push.clientGroupID,\n });\n\n // Record custom mutations for telemetry\n recordMutation('custom', entry.push.mutations.length);\n\n const url = must(\n entry.connCtx.mutateContext.url,\n 'ZERO_MUTATE_URL is not set',\n );\n\n this.#lc.debug?.(\n 'pushing to',\n url,\n 'with',\n entry.push.mutations.length,\n 'mutations',\n );\n\n let mutationIDs: MutationID[] = [];\n\n try {\n mutationIDs = entry.push.mutations.map(m => ({\n id: m.id,\n clientID: m.clientID,\n }));\n\n const response = await fetchFromAPIServer(\n mutateResponseSchema,\n 'push',\n this.#lc,\n entry.connCtx,\n {\n appID: this.#config.app.id,\n shardNum: this.#config.shard.num,\n },\n entry.push,\n {operation: 'mutate'},\n );\n if (\n ('kind' in response && response.kind === ErrorKind.PushFailed) ||\n 'error' in response\n ) {\n if (isAuthErrorBody(response)) {\n this.#lc.warn?.('Push auth failed; invalidating connection', {\n clientID: entry.connCtx.clientID,\n response: 'kind' in response ? response.message : undefined,\n });\n this.#connContextManager.failConnection(\n entry.connCtx,\n entry.connCtx.revision,\n );\n }\n return response;\n }\n // A successful push also validates this connection's current auth snapshot.\n // That lets later shared work reuse it without trusting stale credentials.\n this.#connContextManager.validateConnection(\n entry.connCtx,\n entry.connCtx.revision,\n 'kind' in response &&\n response.kind === 'MutateResponse' &&\n response?.userID !== undefined\n ? {kind: 'server-validated', validatedUserID: response.userID}\n : {kind: 'client-fallback'},\n );\n return response;\n } catch (e) {\n if (isProtocolError(e) && e.errorBody.kind === ErrorKind.PushFailed) {\n const response = {\n ...e.errorBody,\n mutationIDs,\n } as const satisfies PushFailedBody;\n if (isAuthErrorBody(response)) {\n this.#lc.warn?.('Push auth failed; invalidating connection', {\n clientID: entry.connCtx.clientID,\n response: 'kind' in response ? response.message : undefined,\n });\n this.#connContextManager.failConnection(\n entry.connCtx,\n entry.connCtx.revision,\n );\n }\n return response;\n }\n\n if (isProtocolError(e) && isAuthErrorBody(e.errorBody)) {\n // The push completed far enough for local validation to reject the\n // connection, so invalidate it and surface the result as PushFailed.\n this.#lc.warn?.('Push validation failed; invalidating connection', {\n clientID: entry.connCtx.clientID,\n response: e.message,\n });\n this.#connContextManager.failConnection(\n entry.connCtx,\n entry.connCtx.revision,\n );\n return {\n kind: ErrorKind.PushFailed,\n origin: ErrorOrigin.ZeroCache,\n reason: ErrorReason.HTTP,\n message: e.message,\n status: 401,\n mutationIDs,\n } as const satisfies PushFailedBody;\n }\n\n return {\n kind: ErrorKind.PushFailed,\n origin: ErrorOrigin.ZeroCache,\n reason: ErrorReason.Internal,\n message: `Failed to push: ${getErrorMessage(e)}`,\n mutationIDs,\n } as const satisfies PushFailedBody;\n }\n }\n\n #failDownstream(\n downstream: Subscription<Downstream>,\n errorBody: PushFailedBody,\n ): void {\n downstream.fail(new ProtocolErrorWithLevel(errorBody, 'warn'));\n }\n}\n\n/**\n * Pushes for different clients, sockets, or auth revisions could be interleaved.\n *\n * In order to batch safely, we only combine pushes from the same\n * clientID/wsID/revision snapshot.\n */\nexport function combinePushes(\n entries: readonly (PusherEntryOrStop | undefined)[],\n): [PusherEntry[], boolean] {\n const pushesByConnection = new Map<string, PusherEntry[]>();\n\n function collect() {\n const ret: PusherEntry[] = [];\n for (const entries of pushesByConnection.values()) {\n const composite: PusherEntry = {\n ...entries[0],\n push: {\n ...entries[0].push,\n mutations: [],\n },\n };\n ret.push(composite);\n for (const entry of entries) {\n assertAreCompatiblePushes(composite, entry);\n composite.push.mutations.push(...entry.push.mutations);\n }\n }\n return ret;\n }\n\n for (const entry of entries) {\n if (entry === 'stop' || entry === undefined) {\n return [collect(), true];\n }\n\n const key = `${entry.connCtx.clientID}:${entry.connCtx.wsID}:${entry.connCtx.revision}`;\n getOrInsertComputed(pushesByConnection, key, newArray).push(entry);\n }\n\n return [collect(), false] as const;\n}\n\n// These invariants should always be true for a given clientID.\n// If they are not, we have a bug in the code somewhere.\nfunction assertAreCompatiblePushes(left: PusherEntry, right: PusherEntry) {\n assert(\n left.connCtx.clientID === right.connCtx.clientID,\n 'clientID must be the same for all pushes',\n );\n assert(\n left.connCtx.wsID === right.connCtx.wsID,\n 'wsID must be the same for all pushes',\n );\n assert(\n left.connCtx.revision === right.connCtx.revision,\n 'revision must be the same for all pushes',\n );\n assert(\n authEquals(left.connCtx.auth, right.connCtx.auth),\n 'auth must be the same for all pushes with the same clientID',\n );\n assert(\n left.push.schemaVersion === right.push.schemaVersion,\n 'schemaVersion must be the same for all pushes with the same clientID',\n );\n assert(\n left.push.pushVersion === right.push.pushVersion,\n 'pushVersion must be the same for all pushes with the same clientID',\n );\n assert(\n left.connCtx.mutateContext.headerOptions.cookie ===\n right.connCtx.mutateContext.headerOptions.cookie,\n 'httpCookie must be the same for all pushes with the same clientID',\n );\n assert(\n left.connCtx.mutateContext.headerOptions.origin ===\n right.connCtx.mutateContext.headerOptions.origin,\n 'origin must be the same for all pushes with the same clientID',\n );\n assert(\n left.connCtx.user.id === right.connCtx.user.id,\n 'userID must be the same for all pushes with the same clientID',\n );\n assert(\n left.connCtx.mutateContext.url === right.connCtx.mutateContext.url,\n 'userPushURL must be the same for all pushes with the same clientID',\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoEA,IAAa,gBAAb,MAAsD;CACpD;CACA;CACA;CACA;CACA;CACA;CACA;CACA,YAAY;CACZ,aAAa;CAEb,YACE,WACA,IACA,eACA,oBACA;EACA,KAAK,sBAAsB;EAC3B,KAAK,UAAU;EACf,KAAK,MAAM,GAAG,YAAY,aAAa,eAAe;EACtD,KAAK,SAAS,IAAI,MAAM;EACxB,KAAK,UAAU,IAAI,WACjB,WACA,IACA,KAAK,qBACL,KAAK,MACP;EACA,KAAK,KAAK;CACZ;CAEA,eAAe,UAAkD;EAC/D,OAAO,KAAK,QAAQ,eAAe,QAAQ;CAC7C;CAEA,YACE,UACA,MACsC;EACtC,MAAM,UAAU,KAAK,oBAAoB,qBAAqB,QAAQ;EACtE,IAAI,SACF,KAAK,QAAQ,YAAY,SAAS,IAAI;EAGxC,OAAO,EACL,MAAM,KACR;CACF;CAEA,MAAM,qBACJ,WACA,QACe;EACf,MAAM,UAAU,KAAK,oBAAoB,qBAAqB,SAAS;EACvE,IAAI,CAAC,SAAS,eAAe,KAE3B;EAGF,MAAM,cAAwB;GAC5B,eAAe,KAAK;GACpB,WAAW,CACT;IACE,MAAMA;IACN,IAAI;IACJ,UAAU,OAAO;IACjB,MAAM;IACN,MAAM,CACJ;KACE,MAAM;KACN,eAAe,KAAK;KACpB,UAAU,OAAO;KACjB,gBAAgB,OAAO;IACzB,CACF;IACA,WAAW,KAAK,IAAI;GACtB,CACF;GACA,aAAa;GACb,WAAW,KAAK,IAAI;GACpB,WAAW,WAAW,KAAK,GAAG,GAAG,OAAO,SAAS,GAAG,OAAO;EAC7D;EAEA,IAAI;GACF,MAAM,mBACJ,sBACA,QACA,KAAK,KACL,SACA;IAAC,OAAO,KAAK,QAAQ,IAAI;IAAI,UAAU,KAAK,QAAQ,MAAM;GAAG,GAC7D,aACA;IAAC,WAAW;IAAW,aAAa;GAAQ,CAC9C;EACF,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,mCAAmC,EACjD,OAAO,gBAAgB,CAAC,EAC1B,CAAC;EACH;CACF;;;;;;;CAQA,MAAM,sBACJ,WACA,WACe;EACf,IAAI,UAAU,WAAW,GACvB;EAGF,MAAM,UAAU,KAAK,oBAAoB,qBAAqB,SAAS;EACvE,IAAI,CAAC,SAAS,eAAe,KAE3B;EAGF,MAAM,cAAwB;GAC5B,eAAe,KAAK;GACpB,WAAW,CACT;IACE,MAAMA;IACN,IAAI;IACJ,UAAU,UAAU;IACpB,MAAM;IACN,MAAM,CACJ;KACE,MAAM;KACN,eAAe,KAAK;KACpB;IACF,CACF;IACA,WAAW,KAAK,IAAI;GACtB,CACF;GACA,aAAa;GACb,WAAW,KAAK,IAAI;GACpB,WAAW,gBAAgB,KAAK,GAAG,GAAG,KAAK,IAAI;EACjD;EAEA,IAAI;GACF,MAAM,mBACJ,sBACA,QACA,KAAK,KACL,SACA;IAAC,OAAO,KAAK,QAAQ,IAAI;IAAI,UAAU,KAAK,QAAQ,MAAM;GAAG,GAC7D,aACA;IAAC,WAAW;IAAW,aAAa;GAAM,CAC5C;EACF,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,wCAAwC,EACtD,OAAO,gBAAgB,CAAC,EAC1B,CAAC;EACH;CACF;CAEA,MAAM;EACJ,OAAO,CAAC,KAAK,YAAY,kCAAkC;EAC3D,EAAE,KAAK;CACT;CAEA,QAAQ;EACN,OAAO,CAAC,KAAK,YAAY,kCAAkC;EAC3D,EAAE,KAAK;EACP,IAAI,KAAK,aAAa,GACpB,KAAU,KAAK;CAEnB;CAEA,UAAmB;EACjB,OAAO,KAAK,YAAY;CAC1B;CAEA,MAAqB;EACnB,KAAK,WAAW,KAAK,QAAQ,IAAI;EACjC,OAAO,KAAK;CACd;CAEA,OAAsB;EACpB,IAAI,KAAK,YACP,OAAO,KAAK,KAAK,UAAU,8BAA8B;EAE3D,KAAK,aAAa;EAClB,KAAK,OAAO,QAAQ,MAAM;EAC1B,OAAO,KAAK,KAAK,UAAU,8BAA8B;CAC3D;AACF;;;;;AAYA,IAAM,aAAN,MAAiB;CACf;CACA;CACA;CACA;CACA;;;;;;;;;;;;CAeA,iCAA0B,IAAI,IAAoB;CAElD,mBAA4B,mBAC1B,YACA,UACA,sCACF;CACA,UAAmB,mBACjB,YACA,UACA,0CACF;CAEA,YACE,QACA,IACA,oBACA,OACA;EACA,KAAK,MAAM,GAAG,YAAY,aAAa,QAAQ;EAC/C,KAAK,sBAAsB;EAC3B,KAAK,SAAS;EACd,KAAK,UAAU;EACf,KAAK,2BAAW,IAAI,IAAI;CAC1B;;;;;CAMA,eAAe,UAAwD;EACrE,MAAM,WAAW,KAAK,SAAS,IAAI,SAAS,QAAQ;EACpD,IAAI,YAAY,SAAS,SAAS,SAAS,MAEzC,MAAM,IAAI,MAAM,oCAAoC;EAItD,IAAI,UACF,SAAS,WAAW,OAAO;EAG7B,MAAM,aAAa,aAAa,OAAmB,EACjD,eAAe;GACb,KAAK,SAAS,OAAO,SAAS,QAAQ;EACxC,EACF,CAAC;EACD,KAAK,SAAS,IAAI,SAAS,UAAU;GACnC,MAAM,SAAS;GACf;EACF,CAAC;EACD,OAAO;CACT;CAEA,YAAY,SAA4B,MAAgB;EACtD,KAAK,OAAO,QAAQ;GAClB;GACA;EACF,CAAC;CACH;CAEA,MAAM,MAAM;EACV,SAAS;GAGP,MAAM,gBAAgB,CAAC,GAAG,KAAK,cAAc,CAAC,CAAC,QAC5C,CAAC,UAAU,UACV,CAAC,KAAK,oBAAoB,qBAAqB;IAAC;IAAU;GAAI,CAAC,CACnE;GAGA,MAAM,CAAC,QAAQ,aAAa,cAAc,CAAC,MAFxB,KAAK,OAAO,QAAQ,GAEU,GADpC,KAAK,OAAO,MAC2B,CAAI,CAAC;GACzD,KAAK,MAAM,QAAQ,QAAQ;IACzB,MAAM,EAAC,UAAU,SAAQ,KAAK;IAC9B,IAAI,KAAK,eAAe,IAAI,QAAQ,MAAM,MAAM;KAC9C,KAAK,IAAI,QACP,8GACA;MAAC;MAAU;MAAM,WAAW,KAAK,KAAK,UAAU;KAAM,CACxD;KACA;IACF;IACA,MAAM,gBAAgB,KAAK,KAAK,cAC5B,YAAY,QAAQ,cAAc,EAChC,aAAa,KAAK,KAAK,YACzB,CAAC,IACD,QAAQ,OAAO;IACnB,MAAM,WAAW,MAAM,QAAQ,KAAK,qBAClC,KAAK,aAAa,IAAI,CACxB;IACA,MAAM,KAAK,iBAAiB,KAAK,SAAS,QAAQ;GACpD;GAEA,KAAK,MAAM,CAAC,UAAU,SAAS,eAC7B,IAAI,KAAK,eAAe,IAAI,QAAQ,MAAM,MACxC,KAAK,eAAe,OAAO,QAAQ;GAIvC,IAAI,WACF;EAEJ;CACF;;;;;;;;;;;;;CAcA,iBAAiB,QAA2B,UAA0B;EACpE,MAAM,yBAAyC,CAAC;EAGhD,IACG,UAAU,YAAY,SAAS,SAAS,gBACzC,WAAW,UACX;GACA,KAAK,IAAI,OACP,4DACA,QACF;GACA,MAAM,cAAc,SAAS,eAAe,CAAC;GAC7C,IAAI,YAAY,SAAS,GAAG;IAG1B,IAAI,WAAW,UAAU;KAGvB,MAAM,iBACJ,SAAS,UAAU,SACf;MACE,MAAMC;MACN,QAAQC;MACR,QAAQC;MACR,QAAQ,SAAS;MACjB,aAAa,SAAS;MACtB;MACA,SAAS,gDAAgD,SAAS;KACpE,IACA,SAAS,UAAU,2BACjB;MACE,MAAMF;MACN,QAAQG;MACR,QAAQC;MACR;MACA,SAAS;KACX,IACA;MACE,MAAMJ;MACN,QAAQG;MACR,QAAQE;MACR;MACA,SACE,SAAS,UAAU,eACf,SAAS,UACT,SAAS,UAAU,6BACjB,+BACA;KACV;KAER,KAAK,YAAY,QAAQ,cAAc;IACzC,OACE,KAAK,YAAY,QAAQ,QAAQ;GAErC;EACF,OAAO;GAEL,MAAM,mBAAmB,QAAQ,SAAS,YAAW,MAAK,EAAE,GAAG,QAAQ;GACvE,KAAK,MAAM,aAAa,iBAAiB,OAAO,GAAG;IACjD,IAAI;IACJ,IAAI,IAAI;IACR,OAAO,IAAI,UAAU,QAAQ,KAAK;KAChC,MAAM,IAAI,UAAU;KACpB,IAAI,WAAW,EAAE,QACf,KAAK,IAAI,OACP,gEACA,EAAE,MACJ;KAKF,IAAI,WAAW,EAAE,UAAU,EAAE,OAAO,UAAU,eAAe;MAC3D,UAAU;OACR,MAAML;OACN,QAAQG;OACR,QAAQG;OACR,SAAS;OACT,SAAS,EAAE,OAAO;OAClB,aAAa,UAAU,KAAI,OAAM;QAC/B,UAAU,EAAE,GAAG;QACf,IAAI,EAAE,GAAG;OACX,EAAE;MACJ;MACA;KACF;IACF;IAEA,IAAI,WAAW,IAAI,UAAU,SAAS,GACpC,KAAK,IAAI,OACP,qFACF;IAGF,IAAI,SACF,uBAAuB,WAAW,KAAK,YAAY,QAAQ,OAAO,CAAC;GAEvE;EACF;EAEA,uBAAuB,SAAQ,OAAM,GAAG,CAAC;CAC3C;;;;;;;;;;CAWA,YAAY,QAA2B,WAAiC;EACtE,KAAK,eAAe,IAAI,OAAO,UAAU,OAAO,IAAI;EACpD,MAAM,SAAS,KAAK,SAAS,IAAI,OAAO,QAAQ;EAChD,IAAI,QAAQ,SAAS,OAAO,MAC1B,KAAK,gBAAgB,OAAO,YAAY,SAAS;CAErD;CAEA,MAAM,aAAa,OAA6C;EAC9D,KAAK,iBAAiB,IAAI,MAAM,KAAK,UAAU,QAAQ,EACrD,eAAe,MAAM,KAAK,cAC5B,CAAC;EACD,KAAK,QAAQ,IAAI,GAAG,EAClB,eAAe,MAAM,KAAK,cAC5B,CAAC;EAGD,eAAe,UAAU,MAAM,KAAK,UAAU,MAAM;EAEpD,MAAM,MAAM,KACV,MAAM,QAAQ,cAAc,KAC5B,4BACF;EAEA,KAAK,IAAI,QACP,cACA,KACA,QACA,MAAM,KAAK,UAAU,QACrB,WACF;EAEA,IAAI,cAA4B,CAAC;EAEjC,IAAI;GACF,cAAc,MAAM,KAAK,UAAU,KAAI,OAAM;IAC3C,IAAI,EAAE;IACN,UAAU,EAAE;GACd,EAAE;GAEF,MAAM,WAAW,MAAM,mBACrB,sBACA,QACA,KAAK,KACL,MAAM,SACN;IACE,OAAO,KAAK,QAAQ,IAAI;IACxB,UAAU,KAAK,QAAQ,MAAM;GAC/B,GACA,MAAM,MACN,EAAC,WAAW,SAAQ,CACtB;GACA,IACG,UAAU,YAAY,SAAS,SAAS,gBACzC,WAAW,UACX;IACA,IAAI,gBAAgB,QAAQ,GAAG;KAC7B,KAAK,IAAI,OAAO,6CAA6C;MAC3D,UAAU,MAAM,QAAQ;MACxB,UAAU,UAAU,WAAW,SAAS,UAAU,KAAA;KACpD,CAAC;KACD,KAAK,oBAAoB,eACvB,MAAM,SACN,MAAM,QAAQ,QAChB;IACF;IACA,OAAO;GACT;GAGA,KAAK,oBAAoB,mBACvB,MAAM,SACN,MAAM,QAAQ,UACd,UAAU,YACR,SAAS,SAAS,oBAClB,UAAU,WAAW,KAAA,IACnB;IAAC,MAAM;IAAoB,iBAAiB,SAAS;GAAM,IAC3D,EAAC,MAAM,kBAAiB,CAC9B;GACA,OAAO;EACT,SAAS,GAAG;GACV,IAAI,gBAAgB,CAAC,KAAK,EAAE,UAAU,SAAS,cAAsB;IACnE,MAAM,WAAW;KACf,GAAG,EAAE;KACL;IACF;IACA,IAAI,gBAAgB,QAAQ,GAAG;KAC7B,KAAK,IAAI,OAAO,6CAA6C;MAC3D,UAAU,MAAM,QAAQ;MACxB,UAAU,UAAU,WAAW,SAAS,UAAU,KAAA;KACpD,CAAC;KACD,KAAK,oBAAoB,eACvB,MAAM,SACN,MAAM,QAAQ,QAChB;IACF;IACA,OAAO;GACT;GAEA,IAAI,gBAAgB,CAAC,KAAK,gBAAgB,EAAE,SAAS,GAAG;IAGtD,KAAK,IAAI,OAAO,mDAAmD;KACjE,UAAU,MAAM,QAAQ;KACxB,UAAU,EAAE;IACd,CAAC;IACD,KAAK,oBAAoB,eACvB,MAAM,SACN,MAAM,QAAQ,QAChB;IACA,OAAO;KACL,MAAMN;KACN,QAAQC;KACR,QAAQC;KACR,SAAS,EAAE;KACX,QAAQ;KACR;IACF;GACF;GAEA,OAAO;IACL,MAAMF;IACN,QAAQC;IACR,QAAQI;IACR,SAAS,mBAAmB,gBAAgB,CAAC;IAC7C;GACF;EACF;CACF;CAEA,gBACE,YACA,WACM;EACN,WAAW,KAAK,IAAI,uBAAuB,WAAW,MAAM,CAAC;CAC/D;AACF;;;;;;;AAQA,SAAgB,cACd,SAC0B;CAC1B,MAAM,qCAAqB,IAAI,IAA2B;CAE1D,SAAS,UAAU;EACjB,MAAM,MAAqB,CAAC;EAC5B,KAAK,MAAM,WAAW,mBAAmB,OAAO,GAAG;GACjD,MAAM,YAAyB;IAC7B,GAAG,QAAQ;IACX,MAAM;KACJ,GAAG,QAAQ,EAAE,CAAC;KACd,WAAW,CAAC;IACd;GACF;GACA,IAAI,KAAK,SAAS;GAClB,KAAK,MAAM,SAAS,SAAS;IAC3B,0BAA0B,WAAW,KAAK;IAC1C,UAAU,KAAK,UAAU,KAAK,GAAG,MAAM,KAAK,SAAS;GACvD;EACF;EACA,OAAO;CACT;CAEA,KAAK,MAAM,SAAS,SAAS;EAC3B,IAAI,UAAU,UAAU,UAAU,KAAA,GAChC,OAAO,CAAC,QAAQ,GAAG,IAAI;EAGzB,MAAM,MAAM,GAAG,MAAM,QAAQ,SAAS,GAAG,MAAM,QAAQ,KAAK,GAAG,MAAM,QAAQ;EAC7E,oBAAoB,oBAAoB,KAAK,QAAQ,CAAC,CAAC,KAAK,KAAK;CACnE;CAEA,OAAO,CAAC,QAAQ,GAAG,KAAK;AAC1B;AAIA,SAAS,0BAA0B,MAAmB,OAAoB;CACxE,OACE,KAAK,QAAQ,aAAa,MAAM,QAAQ,UACxC,0CACF;CACA,OACE,KAAK,QAAQ,SAAS,MAAM,QAAQ,MACpC,sCACF;CACA,OACE,KAAK,QAAQ,aAAa,MAAM,QAAQ,UACxC,0CACF;CACA,OACE,WAAW,KAAK,QAAQ,MAAM,MAAM,QAAQ,IAAI,GAChD,6DACF;CACA,OACE,KAAK,KAAK,kBAAkB,MAAM,KAAK,eACvC,sEACF;CACA,OACE,KAAK,KAAK,gBAAgB,MAAM,KAAK,aACrC,oEACF;CACA,OACE,KAAK,QAAQ,cAAc,cAAc,WACvC,MAAM,QAAQ,cAAc,cAAc,QAC5C,mEACF;CACA,OACE,KAAK,QAAQ,cAAc,cAAc,WACvC,MAAM,QAAQ,cAAc,cAAc,QAC5C,+DACF;CACA,OACE,KAAK,QAAQ,KAAK,OAAO,MAAM,QAAQ,KAAK,IAC5C,+DACF;CACA,OACE,KAAK,QAAQ,cAAc,QAAQ,MAAM,QAAQ,cAAc,KAC/D,oEACF;AACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"drain-coordinator.js","names":[],"sources":["../../../../../../zero-cache/src/services/view-syncer/drain-coordinator.ts"],"sourcesContent":["import {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../shared/src/asserts.ts';\n\ntype DrainListener = () => void;\n\n// The target (additional) utilization to impose on the server\n// that receives the drained connections.\
|
|
1
|
+
{"version":3,"file":"drain-coordinator.js","names":[],"sources":["../../../../../../zero-cache/src/services/view-syncer/drain-coordinator.ts"],"sourcesContent":["import {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../shared/src/asserts.ts';\n\ntype DrainListener = () => void;\n\n// The target (additional) utilization to impose on the server\n// that receives the drained connections.\nexport const TARGET_UTILIZATION = 0.6;\n\n/**\n * There are two types of drains:\n * 1. Elective drains happen when a view-syncer is about to process\n * a replication event and drains instead because {@link shouldDrain()}\n * returned true. In this case, it exits its processing loop and\n * calls {@link drainNextIn()} immediately.\n * 2. Force drains are performed by the Syncer, which picks a random\n * view-syncer and calls {@link stop()}.\n *\n * In the case of a forced drain, the view-syncer may be queued up\n * behind a large amount of CPU-consuming events, and thus it may take\n * a long time before it actually stops. Elective drains are preferred because\n * they are less subject to that variance. However, elective drains only happen\n * if there is a task to be processed, so forced drains are still necessary\n * for draining servers with no work / tasks.\n *\n * The Syncer kicks off the drain process by calling {@link drainNextIn drainNextIn(0)},\n * which sets off a short {@link forceDrainTimeout} but starts returning `true` for\n * {@link shouldDrain()} for elective drains. In the latter case, the drained\n * view-syncer immediately exiting its processing loop and calls\n * {@link drainNextIn drainNextIn(myHydrationTime)} to reset the timeout\n * for the next elective or forced drain.\n */\nexport class DrainCoordinator {\n #draining = false;\n readonly #drainListeners = new Set<DrainListener>();\n #nextDrainTime = 0;\n #timeout = resolver();\n #timeoutID: NodeJS.Timeout | undefined;\n\n /**\n * Calls `listener` once draining has started (immediately if it already\n * has) and returns a function that unsubscribes it.\n *\n * This is a subscription rather than a Promise so that a caller that stops\n * caring (e.g. a view-syncer that finished initializing) can let go of it.\n * A reaction attached to a Promise that may never settle would otherwise\n * be retained by the coordinator for the lifetime of the server, once per\n * view-syncer ever created.\n */\n onDraining(listener: DrainListener): () => void {\n if (this.#draining) {\n listener();\n return () => {};\n }\n this.#drainListeners.add(listener);\n return () => {\n this.#drainListeners.delete(listener);\n };\n }\n\n // Exposed for testing.\n get drainListenerCount() {\n return this.#drainListeners.size;\n }\n\n shouldDrain() {\n return this.#nextDrainTime && this.#nextDrainTime <= Date.now();\n }\n\n drainNextIn(interval: number) {\n if (!this.#draining) {\n this.#draining = true;\n const listeners = [...this.#drainListeners];\n this.#drainListeners.clear();\n for (const listener of listeners) {\n listener();\n }\n }\n // Increase the timeout between drains to give the receiving\n // server space to perform normal processing.\n interval /= TARGET_UTILIZATION;\n\n const now = Date.now();\n assert(\n this.#nextDrainTime <= now,\n `drainNextIn() should only be called if shouldDrain()`,\n );\n this.#nextDrainTime = now + interval;\n\n // Push the forceDrainTimeout forward.\n clearTimeout(this.#timeoutID);\n this.#timeoutID = setTimeout(() => {\n this.#timeout.resolve();\n this.#timeout = resolver();\n }, interval + FORCE_DRAIN_PADDING);\n }\n\n get forceDrainTimeout() {\n return this.#timeout.promise;\n }\n\n // Exposed for testing.\n get nextDrainTime() {\n return this.#nextDrainTime;\n }\n}\n\nconst FORCE_DRAIN_PADDING = 2;\n"],"mappings":";;;AAOA,IAAa,qBAAqB;;;;;;;;;;;;;;;;;;;;;;;;AAyBlC,IAAa,mBAAb,MAA8B;CAC5B,YAAY;CACZ,kCAA2B,IAAI,IAAmB;CAClD,iBAAiB;CACjB,WAAW,SAAS;CACpB;;;;;;;;;;;CAYA,WAAW,UAAqC;EAC9C,IAAI,KAAK,WAAW;GAClB,SAAS;GACT,aAAa,CAAC;EAChB;EACA,KAAK,gBAAgB,IAAI,QAAQ;EACjC,aAAa;GACX,KAAK,gBAAgB,OAAO,QAAQ;EACtC;CACF;CAGA,IAAI,qBAAqB;EACvB,OAAO,KAAK,gBAAgB;CAC9B;CAEA,cAAc;EACZ,OAAO,KAAK,kBAAkB,KAAK,kBAAkB,KAAK,IAAI;CAChE;CAEA,YAAY,UAAkB;EAC5B,IAAI,CAAC,KAAK,WAAW;GACnB,KAAK,YAAY;GACjB,MAAM,YAAY,CAAC,GAAG,KAAK,eAAe;GAC1C,KAAK,gBAAgB,MAAM;GAC3B,KAAK,MAAM,YAAY,WACrB,SAAS;EAEb;EAGA,YAAY;EAEZ,MAAM,MAAM,KAAK,IAAI;EACrB,OACE,KAAK,kBAAkB,KACvB,sDACF;EACA,KAAK,iBAAiB,MAAM;EAG5B,aAAa,KAAK,UAAU;EAC5B,KAAK,aAAa,iBAAiB;GACjC,KAAK,SAAS,QAAQ;GACtB,KAAK,WAAW,SAAS;EAC3B,GAAG,WAAW,mBAAmB;CACnC;CAEA,IAAI,oBAAoB;EACtB,OAAO,KAAK,SAAS;CACvB;CAGA,IAAI,gBAAgB;EAClB,OAAO,KAAK;CACd;AACF;AAEA,IAAM,sBAAsB"}
|
|
@@ -537,6 +537,9 @@ var ViewSyncerService = class {
|
|
|
537
537
|
get queryCount() {
|
|
538
538
|
return this.#pipelines.initialized() ? this.#pipelines.queries().size : 0;
|
|
539
539
|
}
|
|
540
|
+
get hydrationTimeMs() {
|
|
541
|
+
return this.#totalHydrationTimeMs();
|
|
542
|
+
}
|
|
540
543
|
pipelineHashes() {
|
|
541
544
|
if (!this.#pipelines.initialized()) return [];
|
|
542
545
|
const cvrQueries = this.#cvr?.queries;
|