@rocicorp/zero 1.8.0-canary.5 → 1.8.0-canary.7

package/out/zero-cache/src/workers/connection.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"connection.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/workers/connection.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAW,MAAM,kBAAkB,CAAC;AAE3D,OAAO,SAAkC,MAAM,IAAI,CAAC;AAIpD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,oCAAoC,CAAC;AAGnE,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,qCAAqC,CAAC;AACnE,OAAO,EAEL,KAAK,aAAa,EACnB,MAAM,qCAAqC,CAAC;AAK7C,OAAO,EAAiB,KAAK,QAAQ,EAAC,MAAM,kCAAkC,CAAC;AAM/E,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,qBAAqB,CAAC;AAChD,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,qBAAqB,CAAC;AAEvD,MAAM,MAAM,aAAa,GACrB;IACE,IAAI,EAAE,IAAI,CAAC;CACZ,GACD;IACE,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,SAAS,CAAC;CAClB,GACD;IACE,IAAI,EAAE,WAAW,CAAC;IAClB,MAAM,EAAE,SAAS,EAAE,CAAC;CACrB,GACD,YAAY,CAAC;AAEjB,MAAM,MAAM,YAAY,GAAG;IACzB,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,YAAY,GAAG,QAAQ,CAAC;IAChC,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;CAC5B,CAAC;AAEF,MAAM,WAAW,cAAc;IAC7B,aAAa,CAAC,GAAG,EAAE,QAAQ,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC;CACxD;AAeD;;;;;;;GAOG;AACH,qBAAa,UAAU;;gBAcnB,EAAE,EAAE,UAAU,EACd,aAAa,EAAE,aAAa,EAC5B,EAAE,EAAE,SAAS,EACb,cAAc,EAAE,cAAc,EAC9B,OAAO,EAAE,MAAM,IAAI;IA2BrB;;;;;;OAMG;IACH,IAAI,IAAI,OAAO;IAyBf,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE;IAsBxC,oBAAoB,CAAC,iBAAiB,EAAE,MAAM;IAmJ9C,IAAI,CACF,IAAI,EAAE,UAAU,EAChB,QAAQ,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,KAAK,IAAI,CAAC,GAAG,qBAAqB;IAMlE,SAAS,CAAC,SAAS,EAAE,SAAS,EAAE,MAAM,CAAC,EAAE,OAAO;CAGjD;AAED,MAAM,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,GAAG;IAC1D,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,KAAK,IAAI,GAAG,IAAI,CAAC;CACtD,CAAC;AAGF,wBAAgB,IAAI,CAClB,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,aAAa,EACjB,IAAI,EAAE,UAAU,EAChB,QAAQ,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,KAAK,IAAI,CAAC,GAAG,qBAAqB,QAwBjE;AAED,wBAAgB,SAAS,CACvB,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,SAAS,EACb,SAAS,EAAE,SAAS,EACpB,MAAM,CAAC,EAAE,OAAO,QA+BjB;AAED,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,OAAO,GAAG,aAAa,GAAG,SAAS,CAQ3E"}
+{"version":3,"file":"connection.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/workers/connection.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAW,MAAM,kBAAkB,CAAC;AAE3D,OAAO,SAAkC,MAAM,IAAI,CAAC;AAIpD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,oCAAoC,CAAC;AAGnE,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,qCAAqC,CAAC;AACnE,OAAO,EAEL,KAAK,aAAa,EACnB,MAAM,qCAAqC,CAAC;AAK7C,OAAO,EAAiB,KAAK,QAAQ,EAAC,MAAM,kCAAkC,CAAC;AAO/E,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,qBAAqB,CAAC;AAChD,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,qBAAqB,CAAC;AAEvD,MAAM,MAAM,aAAa,GACrB;IACE,IAAI,EAAE,IAAI,CAAC;CACZ,GACD;IACE,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,SAAS,CAAC;CAClB,GACD;IACE,IAAI,EAAE,WAAW,CAAC;IAClB,MAAM,EAAE,SAAS,EAAE,CAAC;CACrB,GACD,YAAY,CAAC;AAEjB,MAAM,MAAM,YAAY,GAAG;IACzB,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,YAAY,GAAG,QAAQ,CAAC;IAChC,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;CAC5B,CAAC;AAEF,MAAM,WAAW,cAAc;IAC7B,aAAa,CAAC,GAAG,EAAE,QAAQ,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC;CACxD;AAiBD;;;;;;;GAOG;AACH,qBAAa,UAAU;;gBAmBnB,EAAE,EAAE,UAAU,EACd,aAAa,EAAE,aAAa,EAC5B,EAAE,EAAE,SAAS,EACb,cAAc,EAAE,cAAc,EAC9B,OAAO,EAAE,MAAM,IAAI;IA2BrB;;;;;;OAMG;IACH,IAAI,IAAI,OAAO;IAyBf,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE;IAsBxC,oBAAoB,CAAC,iBAAiB,EAAE,MAAM;IA8J9C,IAAI,CACF,IAAI,EAAE,UAAU,EAChB,QAAQ,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,KAAK,IAAI,CAAC,GAAG,qBAAqB;IAMlE,SAAS,CAAC,SAAS,EAAE,SAAS,EAAE,MAAM,CAAC,EAAE,OAAO;CAGjD;AAED,MAAM,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,GAAG;IAC1D,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,KAAK,IAAI,GAAG,IAAI,CAAC;CACtD,CAAC;AAGF,wBAAgB,IAAI,CAClB,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,aAAa,EACjB,IAAI,EAAE,UAAU,EAChB,QAAQ,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,KAAK,IAAI,CAAC,GAAG,qBAAqB,QAwBjE;AAED,wBAAgB,SAAS,CACvB,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,SAAS,EACb,SAAS,EAAE,SAAS,EACpB,MAAM,CAAC,EAAE,OAAO,QA+BjB;AAED,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,OAAO,GAAG,aAAa,GAAG,SAAS,CAQ3E"}

package/out/zero-cache/src/workers/connection.js

@@ -4,12 +4,15 @@ import { Internal, InvalidMessage, VersionNotSupported } from "../../../zero-pro
 import { ZeroCache } from "../../../zero-protocol/src/error-origin-enum.js";
 import { isProtocolError } from "../../../zero-protocol/src/error.js";
 import "../../../zero-protocol/src/protocol-version.js";
+import { getOrCreateCounter } from "../observability/metrics.js";
 import { ProtocolErrorWithLevel, getLogLevel, wrapWithProtocolError } from "../types/error-with-level.js";
 import { upstreamSchema } from "../../../zero-protocol/src/up.js";
 import WebSocket$1, { createWebSocketStream } from "ws";
 import { Readable, Writable, pipeline } from "node:stream";
 //#region ../zero-cache/src/workers/connection.ts
 var DOWNSTREAM_MSG_INTERVAL_MS = 6e3;
+var PROTOCOL_VERSION_ATTRIBUTE = "protocol.version";
+var EVENT_TYPE_ATTRIBUTE = "event.type";
 /**
 * Represents a connection between the client and server.
 *
@@ -26,6 +29,7 @@ var Connection = class {
 	#onClose;
 	#messageHandler;
 	#downstreamMsgTimer;
+	#webSocketErrors = getOrCreateCounter("sync", "websocket.errors", "Client WebSocket error events.");
 	#viewSyncerOutboundStream;
 	#pusherOutboundStream;
 	#closed = false;
@@ -136,6 +140,7 @@ var Connection = class {
 	}
 	#handleClose = (e) => {
 		const { code, reason, wasClean } = e;
+		if (!wasClean) this.#recordWebSocketError("unclean_close");
 		this.close("WebSocket close event", {
 			code,
 			reason,
@@ -143,8 +148,15 @@ var Connection = class {
 		});
 	};
 	#handleError = (e) => {
+		this.#recordWebSocketError("error_event");
 		this.#lc.warn?.("WebSocket error event", e.message, e.error);
 	};
+	#recordWebSocketError(eventType) {
+		this.#webSocketErrors.add(1, {
+			[PROTOCOL_VERSION_ATTRIBUTE]: this.#protocolVersion,
+			[EVENT_TYPE_ATTRIBUTE]: eventType
+		});
+	}
 	#proxyInbound() {
 		pipeline(createWebSocketStream(this.#ws), new Writable({ write: (data, _encoding, callback) => {
 			this.#handleMessage({ data }).then(() => callback(), callback);

package/out/zero-cache/src/workers/connection.js.map

@@ -1 +1 @@
-{"version":3,"file":"connection.js","names":["#ws","#wsID","#protocolVersion","#lc","#onClose","#messageHandler","#downstreamMsgTimer","#handleClose","#handleError","#proxyInbound","#maybeSendPong","#closeWithError","#closed","#viewSyncerOutboundStream","#pusherOutboundStream","#handleMessage","#handleMessageResult","#closeWithThrown","#proxyOutbound","#lastDownstreamMsgTime"],"sources":["../../../../../zero-cache/src/workers/connection.ts"],"sourcesContent":["import {pipeline, Readable, Writable} from 'node:stream';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport type {CloseEvent, Data, ErrorEvent} from 'ws';\nimport WebSocket, {createWebSocketStream} from 'ws';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport * as valita from '../../../shared/src/valita.ts';\nimport type {ConnectedMessage} from '../../../zero-protocol/src/connect.ts';\nimport type {Downstream} from '../../../zero-protocol/src/down.ts';\nimport {ErrorKind} from '../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../zero-protocol/src/error-origin.ts';\nimport type {ErrorBody} from '../../../zero-protocol/src/error.ts';\nimport {\n  isProtocolError,\n  type ProtocolError,\n} from '../../../zero-protocol/src/error.ts';\nimport {\n  MIN_SERVER_SUPPORTED_SYNC_PROTOCOL,\n  PROTOCOL_VERSION,\n} from '../../../zero-protocol/src/protocol-version.ts';\nimport {upstreamSchema, type Upstream} from '../../../zero-protocol/src/up.ts';\nimport {\n  ProtocolErrorWithLevel,\n  getLogLevel,\n  wrapWithProtocolError,\n} from '../types/error-with-level.ts';\nimport type {Source} from '../types/streams.ts';\nimport type {ConnectParams} from './connect-params.ts';\n\nexport type HandlerResult =\n  | {\n      type: 'ok';\n    }\n  | {\n      type: 'fatal';\n      error: ErrorBody;\n    }\n  | {\n      type: 'transient';\n      errors: ErrorBody[];\n    }\n  | StreamResult;\n\nexport type StreamResult = {\n  type: 'stream';\n  source: 'viewSyncer' | 'pusher';\n  stream: Source<Downstream>;\n};\n\nexport interface MessageHandler {\n  handleMessage(msg: Upstream): Promise<HandlerResult[]>;\n}\n\n// Ensures that a downstream message is sent at least every interval, sending a\n// 'pong' if necessary. This is set to be slightly longer than the client-side\n// PING_INTERVAL of 5 seconds, so that in the common case, 'pong's are sent in\n// response to client-initiated 'ping's. However, if the inbound stream is\n// backed up because a command is taking a long time to process, the pings\n// will be stuck in the queue (i.e. back-pressured), in which case pongs will\n// be manually sent to notify the client of server liveness.\n//\n// This is equivalent to what is done for Postgres keepalives on the\n// replication stream (which can similarly be back-pressured):\n// https://github.com/rocicorp/mono/blob/f98cb369a2dbb15650328859c732db358f187ef0/packages/zero-cache/src/services/change-source/pg/logical-replication/stream.ts#L21\nconst DOWNSTREAM_MSG_INTERVAL_MS = 6_000;\n\n/**\n * Represents a connection between the client and server.\n *\n * Handles incoming messages on the connection and dispatches\n * them to the correct service.\n *\n * Listens to the ViewSyncer and sends messages to the client.\n */\nexport class Connection {\n  readonly #ws: WebSocket;\n  readonly #wsID: string;\n  readonly #protocolVersion: number;\n  readonly #lc: LogContext;\n  readonly #onClose: () => void;\n  readonly #messageHandler: MessageHandler;\n  readonly #downstreamMsgTimer: NodeJS.Timeout | undefined;\n\n  #viewSyncerOutboundStream: Source<Downstream> | undefined;\n  #pusherOutboundStream: Source<Downstream> | undefined;\n  #closed = false;\n\n  constructor(\n    lc: LogContext,\n    connectParams: ConnectParams,\n    ws: WebSocket,\n    messageHandler: MessageHandler,\n    onClose: () => void,\n  ) {\n    const {clientGroupID, clientID, wsID, protocolVersion} = connectParams;\n    this.#messageHandler = messageHandler;\n\n    this.#ws = ws;\n    this.#wsID = wsID;\n    this.#protocolVersion = protocolVersion;\n\n    this.#lc = lc\n      .withContext('connection')\n      .withContext('clientID', clientID)\n      .withContext('clientGroupID', clientGroupID)\n      .withContext('wsID', wsID);\n    this.#lc.debug?.('new connection');\n    this.#onClose = onClose;\n\n    this.#ws.addEventListener('close', this.#handleClose);\n    this.#ws.addEventListener('error', this.#handleError);\n\n    this.#proxyInbound();\n    this.#downstreamMsgTimer = setInterval(\n      this.#maybeSendPong,\n      DOWNSTREAM_MSG_INTERVAL_MS / 2,\n    );\n  }\n\n  /**\n   * Checks the protocol version and errors for unsupported protocols,\n   * sending the initial `connected` response on success.\n   *\n   * This is early in the connection lifecycle because {@link #handleMessage}\n   * will only parse messages with schema(s) of supported protocol versions.\n   */\n  init(): boolean {\n    if (\n      this.#protocolVersion > PROTOCOL_VERSION ||\n      this.#protocolVersion < MIN_SERVER_SUPPORTED_SYNC_PROTOCOL\n    ) {\n      this.#closeWithError({\n        kind: ErrorKind.VersionNotSupported,\n        message: `server is at sync protocol v${PROTOCOL_VERSION} and does not support v${\n          this.#protocolVersion\n        }. The ${\n          this.#protocolVersion > PROTOCOL_VERSION ? 'server' : 'client'\n        } must be updated to a newer release.`,\n        origin: ErrorOrigin.ZeroCache,\n      });\n    } else {\n      const connectedMessage: ConnectedMessage = [\n        'connected',\n        {wsid: this.#wsID, timestamp: Date.now()},\n      ];\n      this.send(connectedMessage, 'ignore-backpressure');\n      return true;\n    }\n    return false;\n  }\n\n  close(reason: string, ...args: unknown[]) {\n    if (this.#closed) {\n      return;\n    }\n    this.#closed = true;\n    this.#lc.info?.(`closing connection: ${reason}`, ...args);\n    this.#ws.removeEventListener('close', this.#handleClose);\n    this.#ws.removeEventListener('error', this.#handleError);\n    this.#viewSyncerOutboundStream?.cancel();\n    this.#viewSyncerOutboundStream = undefined;\n    this.#pusherOutboundStream?.cancel();\n    this.#pusherOutboundStream = undefined;\n    this.#onClose();\n    if (this.#ws.readyState !== this.#ws.CLOSED) {\n      this.#ws.close();\n    }\n    clearTimeout(this.#downstreamMsgTimer);\n\n    // spin down services if we have\n    // no more client connections for the client group?\n  }\n\n  handleInitConnection(initConnectionMsg: string) {\n    return this.#handleMessage({data: initConnectionMsg});\n  }\n\n  #handleMessage = async (event: {data: Data}) => {\n    const data = event.data.toString();\n    if (this.#closed) {\n      this.#lc.debug?.('Ignoring message received after closed', data);\n      return;\n    }\n\n    let msg;\n    try {\n      const value = JSON.parse(data);\n      msg = valita.parse(value, upstreamSchema);\n    } catch (e) {\n      const errorBody = {\n        kind: ErrorKind.InvalidMessage,\n        message: String(e),\n        origin: ErrorOrigin.ZeroCache,\n      } as const;\n      this.#closeWithError(\n        errorBody,\n        new ProtocolErrorWithLevel(errorBody, 'warn'),\n      );\n      return;\n    }\n\n    try {\n      const msgType = msg[0];\n      if (msgType === 'ping') {\n        this.send(['pong', {}], 'ignore-backpressure');\n        return;\n      }\n\n      const result = await this.#messageHandler.handleMessage(msg);\n      for (const r of result) {\n        this.#handleMessageResult(r);\n      }\n    } catch (e) {\n      this.#closeWithThrown(e);\n    }\n  };\n\n  #handleMessageResult(result: HandlerResult): void {\n    switch (result.type) {\n      case 'fatal':\n        this.#closeWithError(result.error);\n        break;\n      case 'ok':\n        break;\n      case 'stream': {\n        switch (result.source) {\n          case 'viewSyncer':\n            assert(\n              this.#viewSyncerOutboundStream === undefined,\n              'Outbound stream already set for this connection!',\n            );\n            this.#viewSyncerOutboundStream = result.stream;\n            break;\n          case 'pusher':\n            assert(\n              this.#pusherOutboundStream === undefined,\n              'Outbound stream already set for this connection!',\n            );\n            this.#pusherOutboundStream = result.stream;\n            break;\n        }\n        this.#proxyOutbound(result.stream);\n        break;\n      }\n      case 'transient': {\n        for (const error of result.errors) {\n          this.sendError(error);\n        }\n      }\n    }\n  }\n\n  #handleClose = (e: CloseEvent) => {\n    const {code, reason, wasClean} = e;\n    this.close('WebSocket close event', {code, reason, wasClean});\n  };\n\n  #handleError = (e: ErrorEvent) => {\n    this.#lc.warn?.('WebSocket error event', e.message, e.error);\n  };\n\n  #proxyInbound() {\n    pipeline(\n      createWebSocketStream(this.#ws),\n      new Writable({\n        write: (data, _encoding, callback) => {\n          this.#handleMessage({data}).then(() => callback(), callback);\n        },\n      }),\n      // The done callback is not used, as #handleClose and #handleError,\n      // configured on the underlying WebSocket, provide more complete\n      // information.\n      () => {},\n    );\n  }\n\n  #proxyOutbound(outboundStream: Source<Downstream>) {\n    // Note: createWebSocketStream() is avoided here in order to control\n    //       exception handling with #closeWithThrown(). If the Writable\n    //       from createWebSocketStream() were instead used, exceptions\n    //       from the outboundStream result in the Writable closing the\n    //       the websocket before the error message can be sent.\n    pipeline(\n      Readable.from(outboundStream),\n      new Writable({\n        objectMode: true,\n        write: (downstream: Downstream, _encoding, callback) =>\n          this.send(downstream, callback),\n      }),\n      e =>\n        e\n          ? this.#closeWithThrown(e)\n          : this.close(`downstream closed by ViewSyncer`),\n    );\n  }\n\n  #closeWithThrown(e: unknown) {\n    const errorBody =\n      findProtocolError(e)?.errorBody ?? wrapWithProtocolError(e).errorBody;\n\n    this.#closeWithError(errorBody, e);\n  }\n\n  #closeWithError(errorBody: ErrorBody, thrown?: unknown) {\n    this.sendError(errorBody, thrown);\n    this.close(\n      `${errorBody.kind} (${errorBody.origin}): ${errorBody.message}`,\n      errorBody,\n    );\n  }\n\n  #lastDownstreamMsgTime = Date.now();\n\n  #maybeSendPong = () => {\n    if (Date.now() - this.#lastDownstreamMsgTime > DOWNSTREAM_MSG_INTERVAL_MS) {\n      this.#lc.debug?.('manually sending pong');\n      this.send(['pong', {}], 'ignore-backpressure');\n    }\n  };\n\n  send(\n    data: Downstream,\n    callback: ((err?: Error | null) => void) | 'ignore-backpressure',\n  ) {\n    this.#lastDownstreamMsgTime = Date.now();\n    return send(this.#lc, this.#ws, data, callback);\n  }\n\n  sendError(errorBody: ErrorBody, thrown?: unknown) {\n    sendError(this.#lc, this.#ws, errorBody, thrown);\n  }\n}\n\nexport type WebSocketLike = Pick<WebSocket, 'readyState'> & {\n  send(data: string, cb?: (err?: Error) => void): void;\n};\n\n// Exported for testing purposes.\nexport function send(\n  lc: LogContext,\n  ws: WebSocketLike,\n  data: Downstream,\n  callback: ((err?: Error | null) => void) | 'ignore-backpressure',\n) {\n  if (ws.readyState === WebSocket.OPEN) {\n    ws.send(\n      JSON.stringify(data),\n      callback === 'ignore-backpressure' ? undefined : callback,\n    );\n  } else {\n    lc.debug?.(`Dropping outbound message on ws (state: ${ws.readyState})`, {\n      dropped: data,\n    });\n    if (callback !== 'ignore-backpressure') {\n      callback(\n        new ProtocolErrorWithLevel(\n          {\n            kind: ErrorKind.Internal,\n            message: 'WebSocket closed',\n            origin: ErrorOrigin.ZeroCache,\n          },\n          'info',\n        ),\n      );\n    }\n  }\n}\n\nexport function sendError(\n  lc: LogContext,\n  ws: WebSocket,\n  errorBody: ErrorBody,\n  thrown?: unknown,\n) {\n  lc = lc.withContext('errorKind', errorBody.kind);\n\n  let logLevel: LogLevel;\n\n  // If the thrown error is a ProtocolErrorWithLevel, its explicit logLevel takes precedence\n  if (thrown instanceof ProtocolErrorWithLevel) {\n    logLevel = thrown.logLevel;\n  }\n  // Errors with errno or transient socket codes are low-level, transient I/O issues\n  // (e.g., EPIPE, ECONNRESET) and should be warnings, not errors\n  else if (\n    hasErrno(thrown) ||\n    hasTransientSocketCode(thrown) ||\n    isTransientSocketMessage(errorBody.message)\n  ) {\n    logLevel = 'warn';\n  }\n  // Fallback: check errorBody.kind for errors that weren't thrown as ProtocolErrorWithLevel\n  else if (\n    errorBody.kind === ErrorKind.ClientNotFound ||\n    errorBody.kind === ErrorKind.TransformFailed\n  ) {\n    logLevel = 'warn';\n  } else {\n    logLevel = thrown ? getLogLevel(thrown) : 'info';\n  }\n\n  lc[logLevel]?.('Sending error on WebSocket', errorBody, thrown ?? '');\n  send(lc, ws, ['error', errorBody], 'ignore-backpressure');\n}\n\nexport function findProtocolError(error: unknown): ProtocolError | undefined {\n  if (isProtocolError(error)) {\n    return error;\n  }\n  if (error instanceof Error && error.cause) {\n    return findProtocolError(error.cause);\n  }\n  return undefined;\n}\n\nfunction hasErrno(error: unknown): boolean {\n  return Boolean(\n    error &&\n    typeof error === 'object' &&\n    'errno' in error &&\n    typeof (error as {errno: unknown}).errno !== 'undefined',\n  );\n}\n\n// System error codes that indicate transient socket conditions.\n// These are checked via the `code` property on errors.\nconst TRANSIENT_SOCKET_ERROR_CODES = new Set([\n  'EPIPE',\n  'ECONNRESET',\n  'ECANCELED',\n]);\n\n// Error messages that indicate transient socket conditions but don't have\n// standard error codes (e.g., WebSocket library errors).\nconst TRANSIENT_SOCKET_MESSAGE_PATTERNS = [\n  'socket was closed while data was being compressed',\n];\n\nfunction hasTransientSocketCode(error: unknown): boolean {\n  if (!error || typeof error !== 'object') {\n    return false;\n  }\n  const maybeCode =\n    'code' in error ? String((error as {code?: unknown}).code) : undefined;\n  return Boolean(\n    maybeCode && TRANSIENT_SOCKET_ERROR_CODES.has(maybeCode.toUpperCase()),\n  );\n}\n\nfunction isTransientSocketMessage(message: string | undefined): boolean {\n  if (!message) {\n    return false;\n  }\n  const lower = message.toLowerCase();\n  return TRANSIENT_SOCKET_MESSAGE_PATTERNS.some(pattern =>\n    lower.includes(pattern),\n  );\n}\n"],"mappings":";;;;;;;;;;;AA+DA,IAAM,6BAA6B;;;;;;;;;AAUnC,IAAa,aAAb,MAAwB;CACtB;CACA;CACA;CACA;CACA;CACA;CACA;CAEA;CACA;CACA,UAAU;CAEV,YACE,IACA,eACA,IACA,gBACA,SACA;EACA,MAAM,EAAC,eAAe,UAAU,MAAM,oBAAmB;EACzD,KAAKK,kBAAkB;EAEvB,KAAKL,MAAM;EACX,KAAKC,QAAQ;EACb,KAAKC,mBAAmB;EAExB,KAAKC,MAAM,GACR,YAAY,YAAY,EACxB,YAAY,YAAY,QAAQ,EAChC,YAAY,iBAAiB,aAAa,EAC1C,YAAY,QAAQ,IAAI;EAC3B,KAAKA,IAAI,QAAQ,gBAAgB;EACjC,KAAKC,WAAW;EAEhB,KAAKJ,IAAI,iBAAiB,SAAS,KAAKO,YAAY;EACpD,KAAKP,IAAI,iBAAiB,SAAS,KAAKQ,YAAY;EAEpD,KAAKC,cAAc;EACnB,KAAKH,sBAAsB,YACzB,KAAKI,gBACL,6BAA6B,CAC/B;CACF;;;;;;;;CASA,OAAgB;EACd,IACE,KAAKR,mBAAAA,MACL,KAAKA,mBAAAA,IAEL,KAAKS,gBAAgB;GACnB,MAAM;GACN,SAAS,wDACP,KAAKT,iBACN,QACC,KAAKA,mBAAAA,KAAsC,WAAW,SACvD;GACD,QAAQ;EACV,CAAC;OACI;GACL,MAAM,mBAAqC,CACzC,aACA;IAAC,MAAM,KAAKD;IAAO,WAAW,KAAK,IAAI;GAAC,CAC1C;GACA,KAAK,KAAK,kBAAkB,qBAAqB;GACjD,OAAO;EACT;EACA,OAAO;CACT;CAEA,MAAM,QAAgB,GAAG,MAAiB;EACxC,IAAI,KAAKW,SACP;EAEF,KAAKA,UAAU;EACf,KAAKT,IAAI,OAAO,uBAAuB,UAAU,GAAG,IAAI;EACxD,KAAKH,IAAI,oBAAoB,SAAS,KAAKO,YAAY;EACvD,KAAKP,IAAI,oBAAoB,SAAS,KAAKQ,YAAY;EACvD,KAAKK,2BAA2B,OAAO;EACvC,KAAKA,4BAA4B,KAAA;EACjC,KAAKC,uBAAuB,OAAO;EACnC,KAAKA,wBAAwB,KAAA;EAC7B,KAAKV,SAAS;EACd,IAAI,KAAKJ,IAAI,eAAe,KAAKA,IAAI,QACnC,KAAKA,IAAI,MAAM;EAEjB,aAAa,KAAKM,mBAAmB;CAIvC;CAEA,qBAAqB,mBAA2B;EAC9C,OAAO,KAAKS,eAAe,EAAC,MAAM,kBAAiB,CAAC;CACtD;CAEA,iBAAiB,OAAO,UAAwB;EAC9C,MAAM,OAAO,MAAM,KAAK,SAAS;EACjC,IAAI,KAAKH,SAAS;GAChB,KAAKT,IAAI,QAAQ,0CAA0C,IAAI;GAC/D;EACF;EAEA,IAAI;EACJ,IAAI;GAEF,MAAM,MADQ,KAAK,MAAM,IACN,GAAO,cAAc;EAC1C,SAAS,GAAG;GACV,MAAM,YAAY;IAChB,MAAM;IACN,SAAS,OAAO,CAAC;IACjB,QAAQ;GACV;GACA,KAAKQ,gBACH,WACA,IAAI,uBAAuB,WAAW,MAAM,CAC9C;GACA;EACF;EAEA,IAAI;GAEF,IADgB,IAAI,OACJ,QAAQ;IACtB,KAAK,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,qBAAqB;IAC7C;GACF;GAEA,MAAM,SAAS,MAAM,KAAKN,gBAAgB,cAAc,GAAG;GAC3D,KAAK,MAAM,KAAK,QACd,KAAKW,qBAAqB,CAAC;EAE/B,SAAS,GAAG;GACV,KAAKC,iBAAiB,CAAC;EACzB;CACF;CAEA,qBAAqB,QAA6B;EAChD,QAAQ,OAAO,MAAf;GACE,KAAK;IACH,KAAKN,gBAAgB,OAAO,KAAK;IACjC;GACF,KAAK,MACH;GACF,KAAK;IACH,QAAQ,OAAO,QAAf;KACE,KAAK;MACH,OACE,KAAKE,8BAA8B,KAAA,GACnC,kDACF;MACA,KAAKA,4BAA4B,OAAO;MACxC;KACF,KAAK;MACH,OACE,KAAKC,0BAA0B,KAAA,GAC/B,kDACF;MACA,KAAKA,wBAAwB,OAAO;MACpC;IACJ;IACA,KAAKI,eAAe,OAAO,MAAM;IACjC;GAEF,KAAK,aACH,KAAK,MAAM,SAAS,OAAO,QACzB,KAAK,UAAU,KAAK;EAG1B;CACF;CAEA,gBAAgB,MAAkB;EAChC,MAAM,EAAC,MAAM,QAAQ,aAAY;EACjC,KAAK,MAAM,yBAAyB;GAAC;GAAM;GAAQ;EAAQ,CAAC;CAC9D;CAEA,gBAAgB,MAAkB;EAChC,KAAKf,IAAI,OAAO,yBAAyB,EAAE,SAAS,EAAE,KAAK;CAC7D;CAEA,gBAAgB;EACd,SACE,sBAAsB,KAAKH,GAAG,GAC9B,IAAI,SAAS,EACX,QAAQ,MAAM,WAAW,aAAa;GACpC,KAAKe,eAAe,EAAC,KAAI,CAAC,EAAE,WAAW,SAAS,GAAG,QAAQ;EAC7D,EACF,CAAC,SAIK,CAAC,CACT;CACF;CAEA,eAAe,gBAAoC;EAMjD,SACE,SAAS,KAAK,cAAc,GAC5B,IAAI,SAAS;GACX,YAAY;GACZ,QAAQ,YAAwB,WAAW,aACzC,KAAK,KAAK,YAAY,QAAQ;EAClC,CAAC,IACD,MACE,IACI,KAAKE,iBAAiB,CAAC,IACvB,KAAK,MAAM,iCAAiC,CACpD;CACF;CAEA,iBAAiB,GAAY;EAC3B,MAAM,YACJ,kBAAkB,CAAC,GAAG,aAAa,sBAAsB,CAAC,EAAE;EAE9D,KAAKN,gBAAgB,WAAW,CAAC;CACnC;CAEA,gBAAgB,WAAsB,QAAkB;EACtD,KAAK,UAAU,WAAW,MAAM;EAChC,KAAK,MACH,GAAG,UAAU,KAAK,IAAI,UAAU,OAAO,KAAK,UAAU,WACtD,SACF;CACF;CAEA,yBAAyB,KAAK,IAAI;CAElC,uBAAuB;EACrB,IAAI,KAAK,IAAI,IAAI,KAAKQ,yBAAyB,4BAA4B;GACzE,KAAKhB,IAAI,QAAQ,uBAAuB;GACxC,KAAK,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,qBAAqB;EAC/C;CACF;CAEA,KACE,MACA,UACA;EACA,KAAKgB,yBAAyB,KAAK,IAAI;EACvC,OAAO,KAAK,KAAKhB,KAAK,KAAKH,KAAK,MAAM,QAAQ;CAChD;CAEA,UAAU,WAAsB,QAAkB;EAChD,UAAU,KAAKG,KAAK,KAAKH,KAAK,WAAW,MAAM;CACjD;AACF;AAOA,SAAgB,KACd,IACA,IACA,MACA,UACA;CACA,IAAI,GAAG,eAAe,YAAU,MAC9B,GAAG,KACD,KAAK,UAAU,IAAI,GACnB,aAAa,wBAAwB,KAAA,IAAY,QACnD;MACK;EACL,GAAG,QAAQ,2CAA2C,GAAG,WAAW,IAAI,EACtE,SAAS,KACX,CAAC;EACD,IAAI,aAAa,uBACf,SACE,IAAI,uBACF;GACE,MAAM;GACN,SAAS;GACT,QAAQ;EACV,GACA,MACF,CACF;CAEJ;AACF;AAEA,SAAgB,UACd,IACA,IACA,WACA,QACA;CACA,KAAK,GAAG,YAAY,aAAa,UAAU,IAAI;CAE/C,IAAI;CAGJ,IAAI,kBAAkB,wBACpB,WAAW,OAAO;MAIf,IACH,SAAS,MAAM,KACf,uBAAuB,MAAM,KAC7B,yBAAyB,UAAU,OAAO,GAE1C,WAAW;MAGR,IACH,UAAU,SAAS,oBACnB,UAAU,SAAS,mBAEnB,WAAW;MAEX,WAAW,SAAS,YAAY,MAAM,IAAI;CAG5C,GAAG,YAAY,8BAA8B,WAAW,UAAU,EAAE;CACpE,KAAK,IAAI,IAAI,CAAC,SAAS,SAAS,GAAG,qBAAqB;AAC1D;AAEA,SAAgB,kBAAkB,OAA2C;CAC3E,IAAI,gBAAgB,KAAK,GACvB,OAAO;CAET,IAAI,iBAAiB,SAAS,MAAM,OAClC,OAAO,kBAAkB,MAAM,KAAK;AAGxC;AAEA,SAAS,SAAS,OAAyB;CACzC,OAAO,QACL,SACA,OAAO,UAAU,YACjB,WAAW,SACX,OAAQ,MAA2B,UAAU,WAC/C;AACF;AAIA,IAAM,+BAA+B,IAAI,IAAI;CAC3C;CACA;CACA;AACF,CAAC;AAID,IAAM,oCAAoC,CACxC,mDACF;AAEA,SAAS,uBAAuB,OAAyB;CACvD,IAAI,CAAC,SAAS,OAAO,UAAU,UAC7B,OAAO;CAET,MAAM,YACJ,UAAU,QAAQ,OAAQ,MAA2B,IAAI,IAAI,KAAA;CAC/D,OAAO,QACL,aAAa,6BAA6B,IAAI,UAAU,YAAY,CAAC,CACvE;AACF;AAEA,SAAS,yBAAyB,SAAsC;CACtE,IAAI,CAAC,SACH,OAAO;CAET,MAAM,QAAQ,QAAQ,YAAY;CAClC,OAAO,kCAAkC,MAAK,YAC5C,MAAM,SAAS,OAAO,CACxB;AACF"}
+{"version":3,"file":"connection.js","names":["#ws","#wsID","#protocolVersion","#lc","#onClose","#messageHandler","#downstreamMsgTimer","#webSocketErrors","#handleClose","#handleError","#proxyInbound","#maybeSendPong","#closeWithError","#closed","#viewSyncerOutboundStream","#pusherOutboundStream","#handleMessage","#handleMessageResult","#closeWithThrown","#proxyOutbound","#recordWebSocketError","#lastDownstreamMsgTime"],"sources":["../../../../../zero-cache/src/workers/connection.ts"],"sourcesContent":["import {pipeline, Readable, Writable} from 'node:stream';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport type {CloseEvent, Data, ErrorEvent} from 'ws';\nimport WebSocket, {createWebSocketStream} from 'ws';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport * as valita from '../../../shared/src/valita.ts';\nimport type {ConnectedMessage} from '../../../zero-protocol/src/connect.ts';\nimport type {Downstream} from '../../../zero-protocol/src/down.ts';\nimport {ErrorKind} from '../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../zero-protocol/src/error-origin.ts';\nimport type {ErrorBody} from '../../../zero-protocol/src/error.ts';\nimport {\n  isProtocolError,\n  type ProtocolError,\n} from '../../../zero-protocol/src/error.ts';\nimport {\n  MIN_SERVER_SUPPORTED_SYNC_PROTOCOL,\n  PROTOCOL_VERSION,\n} from '../../../zero-protocol/src/protocol-version.ts';\nimport {upstreamSchema, type Upstream} from '../../../zero-protocol/src/up.ts';\nimport {getOrCreateCounter} from '../observability/metrics.ts';\nimport {\n  ProtocolErrorWithLevel,\n  getLogLevel,\n  wrapWithProtocolError,\n} from '../types/error-with-level.ts';\nimport type {Source} from '../types/streams.ts';\nimport type {ConnectParams} from './connect-params.ts';\n\nexport type HandlerResult =\n  | {\n      type: 'ok';\n    }\n  | {\n      type: 'fatal';\n      error: ErrorBody;\n    }\n  | {\n      type: 'transient';\n      errors: ErrorBody[];\n    }\n  | StreamResult;\n\nexport type StreamResult = {\n  type: 'stream';\n  source: 'viewSyncer' | 'pusher';\n  stream: Source<Downstream>;\n};\n\nexport interface MessageHandler {\n  handleMessage(msg: Upstream): Promise<HandlerResult[]>;\n}\n\n// Ensures that a downstream message is sent at least every interval, sending a\n// 'pong' if necessary. This is set to be slightly longer than the client-side\n// PING_INTERVAL of 5 seconds, so that in the common case, 'pong's are sent in\n// response to client-initiated 'ping's. However, if the inbound stream is\n// backed up because a command is taking a long time to process, the pings\n// will be stuck in the queue (i.e. back-pressured), in which case pongs will\n// be manually sent to notify the client of server liveness.\n//\n// This is equivalent to what is done for Postgres keepalives on the\n// replication stream (which can similarly be back-pressured):\n// https://github.com/rocicorp/mono/blob/f98cb369a2dbb15650328859c732db358f187ef0/packages/zero-cache/src/services/change-source/pg/logical-replication/stream.ts#L21\nconst DOWNSTREAM_MSG_INTERVAL_MS = 6_000;\nconst PROTOCOL_VERSION_ATTRIBUTE = 'protocol.version';\nconst EVENT_TYPE_ATTRIBUTE = 'event.type';\n\n/**\n * Represents a connection between the client and server.\n *\n * Handles incoming messages on the connection and dispatches\n * them to the correct service.\n *\n * Listens to the ViewSyncer and sends messages to the client.\n */\nexport class Connection {\n  readonly #ws: WebSocket;\n  readonly #wsID: string;\n  readonly #protocolVersion: number;\n  readonly #lc: LogContext;\n  readonly #onClose: () => void;\n  readonly #messageHandler: MessageHandler;\n  readonly #downstreamMsgTimer: NodeJS.Timeout | undefined;\n  readonly #webSocketErrors = getOrCreateCounter(\n    'sync',\n    'websocket.errors',\n    'Client WebSocket error events.',\n  );\n\n  #viewSyncerOutboundStream: Source<Downstream> | undefined;\n  #pusherOutboundStream: Source<Downstream> | undefined;\n  #closed = false;\n\n  constructor(\n    lc: LogContext,\n    connectParams: ConnectParams,\n    ws: WebSocket,\n    messageHandler: MessageHandler,\n    onClose: () => void,\n  ) {\n    const {clientGroupID, clientID, wsID, protocolVersion} = connectParams;\n    this.#messageHandler = messageHandler;\n\n    this.#ws = ws;\n    this.#wsID = wsID;\n    this.#protocolVersion = protocolVersion;\n\n    this.#lc = lc\n      .withContext('connection')\n      .withContext('clientID', clientID)\n      .withContext('clientGroupID', clientGroupID)\n      .withContext('wsID', wsID);\n    this.#lc.debug?.('new connection');\n    this.#onClose = onClose;\n\n    this.#ws.addEventListener('close', this.#handleClose);\n    this.#ws.addEventListener('error', this.#handleError);\n\n    this.#proxyInbound();\n    this.#downstreamMsgTimer = setInterval(\n      this.#maybeSendPong,\n      DOWNSTREAM_MSG_INTERVAL_MS / 2,\n    );\n  }\n\n  /**\n   * Checks the protocol version and errors for unsupported protocols,\n   * sending the initial `connected` response on success.\n   *\n   * This is early in the connection lifecycle because {@link #handleMessage}\n   * will only parse messages with schema(s) of supported protocol versions.\n   */\n  init(): boolean {\n    if (\n      this.#protocolVersion > PROTOCOL_VERSION ||\n      this.#protocolVersion < MIN_SERVER_SUPPORTED_SYNC_PROTOCOL\n    ) {\n      this.#closeWithError({\n        kind: ErrorKind.VersionNotSupported,\n        message: `server is at sync protocol v${PROTOCOL_VERSION} and does not support v${\n          this.#protocolVersion\n        }. The ${\n          this.#protocolVersion > PROTOCOL_VERSION ? 'server' : 'client'\n        } must be updated to a newer release.`,\n        origin: ErrorOrigin.ZeroCache,\n      });\n    } else {\n      const connectedMessage: ConnectedMessage = [\n        'connected',\n        {wsid: this.#wsID, timestamp: Date.now()},\n      ];\n      this.send(connectedMessage, 'ignore-backpressure');\n      return true;\n    }\n    return false;\n  }\n\n  close(reason: string, ...args: unknown[]) {\n    if (this.#closed) {\n      return;\n    }\n    this.#closed = true;\n    this.#lc.info?.(`closing connection: ${reason}`, ...args);\n    this.#ws.removeEventListener('close', this.#handleClose);\n    this.#ws.removeEventListener('error', this.#handleError);\n    this.#viewSyncerOutboundStream?.cancel();\n    this.#viewSyncerOutboundStream = undefined;\n    this.#pusherOutboundStream?.cancel();\n    this.#pusherOutboundStream = undefined;\n    this.#onClose();\n    if (this.#ws.readyState !== this.#ws.CLOSED) {\n      this.#ws.close();\n    }\n    clearTimeout(this.#downstreamMsgTimer);\n\n    // spin down services if we have\n    // no more client connections for the client group?\n  }\n\n  handleInitConnection(initConnectionMsg: string) {\n    return this.#handleMessage({data: initConnectionMsg});\n  }\n\n  #handleMessage = async (event: {data: Data}) => {\n    const data = event.data.toString();\n    if (this.#closed) {\n      this.#lc.debug?.('Ignoring message received after closed', data);\n      return;\n    }\n\n    let msg;\n    try {\n      const value = JSON.parse(data);\n      msg = valita.parse(value, upstreamSchema);\n    } catch (e) {\n      const errorBody = {\n        kind: ErrorKind.InvalidMessage,\n        message: String(e),\n        origin: ErrorOrigin.ZeroCache,\n      } as const;\n      this.#closeWithError(\n        errorBody,\n        new ProtocolErrorWithLevel(errorBody, 'warn'),\n      );\n      return;\n    }\n\n    try {\n      const msgType = msg[0];\n      if (msgType === 'ping') {\n        this.send(['pong', {}], 'ignore-backpressure');\n        return;\n      }\n\n      const result = await this.#messageHandler.handleMessage(msg);\n      for (const r of result) {\n        this.#handleMessageResult(r);\n      }\n    } catch (e) {\n      this.#closeWithThrown(e);\n    }\n  };\n\n  #handleMessageResult(result: HandlerResult): void {\n    switch (result.type) {\n      case 'fatal':\n        this.#closeWithError(result.error);\n        break;\n      case 'ok':\n        break;\n      case 'stream': {\n        switch (result.source) {\n          case 'viewSyncer':\n            assert(\n              this.#viewSyncerOutboundStream === undefined,\n              'Outbound stream already set for this connection!',\n            );\n            this.#viewSyncerOutboundStream = result.stream;\n            break;\n          case 'pusher':\n            assert(\n              this.#pusherOutboundStream === undefined,\n              'Outbound stream already set for this connection!',\n            );\n            this.#pusherOutboundStream = result.stream;\n            break;\n        }\n        this.#proxyOutbound(result.stream);\n        break;\n      }\n      case 'transient': {\n        for (const error of result.errors) {\n          this.sendError(error);\n        }\n      }\n    }\n  }\n\n  #handleClose = (e: CloseEvent) => {\n    const {code, reason, wasClean} = e;\n    if (!wasClean) {\n      this.#recordWebSocketError('unclean_close');\n    }\n    this.close('WebSocket close event', {code, reason, wasClean});\n  };\n\n  #handleError = (e: ErrorEvent) => {\n    this.#recordWebSocketError('error_event');\n    this.#lc.warn?.('WebSocket error event', e.message, e.error);\n  };\n\n  #recordWebSocketError(eventType: string) {\n    this.#webSocketErrors.add(1, {\n      [PROTOCOL_VERSION_ATTRIBUTE]: this.#protocolVersion,\n      [EVENT_TYPE_ATTRIBUTE]: eventType,\n    });\n  }\n\n  #proxyInbound() {\n    pipeline(\n      createWebSocketStream(this.#ws),\n      new Writable({\n        write: (data, _encoding, callback) => {\n          this.#handleMessage({data}).then(() => callback(), callback);\n        },\n      }),\n      // The done callback is not used, as #handleClose and #handleError,\n      // configured on the underlying WebSocket, provide more complete\n      // information.\n      () => {},\n    );\n  }\n\n  #proxyOutbound(outboundStream: Source<Downstream>) {\n    // Note: createWebSocketStream() is avoided here in order to control\n    //       exception handling with #closeWithThrown(). If the Writable\n    //       from createWebSocketStream() were instead used, exceptions\n    //       from the outboundStream result in the Writable closing the\n    //       the websocket before the error message can be sent.\n    pipeline(\n      Readable.from(outboundStream),\n      new Writable({\n        objectMode: true,\n        write: (downstream: Downstream, _encoding, callback) =>\n          this.send(downstream, callback),\n      }),\n      e =>\n        e\n          ? this.#closeWithThrown(e)\n          : this.close(`downstream closed by ViewSyncer`),\n    );\n  }\n\n  #closeWithThrown(e: unknown) {\n    const errorBody =\n      findProtocolError(e)?.errorBody ?? wrapWithProtocolError(e).errorBody;\n\n    this.#closeWithError(errorBody, e);\n  }\n\n  #closeWithError(errorBody: ErrorBody, thrown?: unknown) {\n    this.sendError(errorBody, thrown);\n    this.close(\n      `${errorBody.kind} (${errorBody.origin}): ${errorBody.message}`,\n      errorBody,\n    );\n  }\n\n  #lastDownstreamMsgTime = Date.now();\n\n  #maybeSendPong = () => {\n    if (Date.now() - this.#lastDownstreamMsgTime > DOWNSTREAM_MSG_INTERVAL_MS) {\n      this.#lc.debug?.('manually sending pong');\n      this.send(['pong', {}], 'ignore-backpressure');\n    }\n  };\n\n  send(\n    data: Downstream,\n    callback: ((err?: Error | null) => void) | 'ignore-backpressure',\n  ) {\n    this.#lastDownstreamMsgTime = Date.now();\n    return send(this.#lc, this.#ws, data, callback);\n  }\n\n  sendError(errorBody: ErrorBody, thrown?: unknown) {\n    sendError(this.#lc, this.#ws, errorBody, thrown);\n  }\n}\n\nexport type WebSocketLike = Pick<WebSocket, 'readyState'> & {\n  send(data: string, cb?: (err?: Error) => void): void;\n};\n\n// Exported for testing purposes.\nexport function send(\n  lc: LogContext,\n  ws: WebSocketLike,\n  data: Downstream,\n  callback: ((err?: Error | null) => void) | 'ignore-backpressure',\n) {\n  if (ws.readyState === WebSocket.OPEN) {\n    ws.send(\n      JSON.stringify(data),\n      callback === 'ignore-backpressure' ? undefined : callback,\n    );\n  } else {\n    lc.debug?.(`Dropping outbound message on ws (state: ${ws.readyState})`, {\n      dropped: data,\n    });\n    if (callback !== 'ignore-backpressure') {\n      callback(\n        new ProtocolErrorWithLevel(\n          {\n            kind: ErrorKind.Internal,\n            message: 'WebSocket closed',\n            origin: ErrorOrigin.ZeroCache,\n          },\n          'info',\n        ),\n      );\n    }\n  }\n}\n\nexport function sendError(\n  lc: LogContext,\n  ws: WebSocket,\n  errorBody: ErrorBody,\n  thrown?: unknown,\n) {\n  lc = lc.withContext('errorKind', errorBody.kind);\n\n  let logLevel: LogLevel;\n\n  // If the thrown error is a ProtocolErrorWithLevel, its explicit logLevel takes precedence\n  if (thrown instanceof ProtocolErrorWithLevel) {\n    logLevel = thrown.logLevel;\n  }\n  // Errors with errno or transient socket codes are low-level, transient I/O issues\n  // (e.g., EPIPE, ECONNRESET) and should be warnings, not errors\n  else if (\n    hasErrno(thrown) ||\n    hasTransientSocketCode(thrown) ||\n    isTransientSocketMessage(errorBody.message)\n  ) {\n    logLevel = 'warn';\n  }\n  // Fallback: check errorBody.kind for errors that weren't thrown as ProtocolErrorWithLevel\n  else if (\n    errorBody.kind === ErrorKind.ClientNotFound ||\n    errorBody.kind === ErrorKind.TransformFailed\n  ) {\n    logLevel = 'warn';\n  } else {\n    logLevel = thrown ? getLogLevel(thrown) : 'info';\n  }\n\n  lc[logLevel]?.('Sending error on WebSocket', errorBody, thrown ?? '');\n  send(lc, ws, ['error', errorBody], 'ignore-backpressure');\n}\n\nexport function findProtocolError(error: unknown): ProtocolError | undefined {\n  if (isProtocolError(error)) {\n    return error;\n  }\n  if (error instanceof Error && error.cause) {\n    return findProtocolError(error.cause);\n  }\n  return undefined;\n}\n\nfunction hasErrno(error: unknown): boolean {\n  return Boolean(\n    error &&\n    typeof error === 'object' &&\n    'errno' in error &&\n    typeof (error as {errno: unknown}).errno !== 'undefined',\n  );\n}\n\n// System error codes that indicate transient socket conditions.\n// These are checked via the `code` property on errors.\nconst TRANSIENT_SOCKET_ERROR_CODES = new Set([\n  'EPIPE',\n  'ECONNRESET',\n  'ECANCELED',\n]);\n\n// Error messages that indicate transient socket conditions but don't have\n// standard error codes (e.g., WebSocket library errors).\nconst TRANSIENT_SOCKET_MESSAGE_PATTERNS = [\n  'socket was closed while data was being compressed',\n];\n\nfunction hasTransientSocketCode(error: unknown): boolean {\n  if (!error || typeof error !== 'object') {\n    return false;\n  }\n  const maybeCode =\n    'code' in error ? String((error as {code?: unknown}).code) : undefined;\n  return Boolean(\n    maybeCode && TRANSIENT_SOCKET_ERROR_CODES.has(maybeCode.toUpperCase()),\n  );\n}\n\nfunction isTransientSocketMessage(message: string | undefined): boolean {\n  if (!message) {\n    return false;\n  }\n  const lower = message.toLowerCase();\n  return TRANSIENT_SOCKET_MESSAGE_PATTERNS.some(pattern =>\n    lower.includes(pattern),\n  );\n}\n"],"mappings":";;;;;;;;;;;;AAgEA,IAAM,6BAA6B;AACnC,IAAM,6BAA6B;AACnC,IAAM,uBAAuB;;;;;;;;;AAU7B,IAAa,aAAb,MAAwB;CACtB;CACA;CACA;CACA;CACA;CACA;CACA;CACA,mBAA4B,mBAC1B,QACA,oBACA,gCACF;CAEA;CACA;CACA,UAAU;CAEV,YACE,IACA,eACA,IACA,gBACA,SACA;EACA,MAAM,EAAC,eAAe,UAAU,MAAM,oBAAmB;EACzD,KAAKK,kBAAkB;EAEvB,KAAKL,MAAM;EACX,KAAKC,QAAQ;EACb,KAAKC,mBAAmB;EAExB,KAAKC,MAAM,GACR,YAAY,YAAY,EACxB,YAAY,YAAY,QAAQ,EAChC,YAAY,iBAAiB,aAAa,EAC1C,YAAY,QAAQ,IAAI;EAC3B,KAAKA,IAAI,QAAQ,gBAAgB;EACjC,KAAKC,WAAW;EAEhB,KAAKJ,IAAI,iBAAiB,SAAS,KAAKQ,YAAY;EACpD,KAAKR,IAAI,iBAAiB,SAAS,KAAKS,YAAY;EAEpD,KAAKC,cAAc;EACnB,KAAKJ,sBAAsB,YACzB,KAAKK,gBACL,6BAA6B,CAC/B;CACF;;;;;;;;CASA,OAAgB;EACd,IACE,KAAKT,mBAAAA,MACL,KAAKA,mBAAAA,IAEL,KAAKU,gBAAgB;GACnB,MAAM;GACN,SAAS,wDACP,KAAKV,iBACN,QACC,KAAKA,mBAAAA,KAAsC,WAAW,SACvD;GACD,QAAQ;EACV,CAAC;OACI;GACL,MAAM,mBAAqC,CACzC,aACA;IAAC,MAAM,KAAKD;IAAO,WAAW,KAAK,IAAI;GAAC,CAC1C;GACA,KAAK,KAAK,kBAAkB,qBAAqB;GACjD,OAAO;EACT;EACA,OAAO;CACT;CAEA,MAAM,QAAgB,GAAG,MAAiB;EACxC,IAAI,KAAKY,SACP;EAEF,KAAKA,UAAU;EACf,KAAKV,IAAI,OAAO,uBAAuB,UAAU,GAAG,IAAI;EACxD,KAAKH,IAAI,oBAAoB,SAAS,KAAKQ,YAAY;EACvD,KAAKR,IAAI,oBAAoB,SAAS,KAAKS,YAAY;EACvD,KAAKK,2BAA2B,OAAO;EACvC,KAAKA,4BAA4B,KAAA;EACjC,KAAKC,uBAAuB,OAAO;EACnC,KAAKA,wBAAwB,KAAA;EAC7B,KAAKX,SAAS;EACd,IAAI,KAAKJ,IAAI,eAAe,KAAKA,IAAI,QACnC,KAAKA,IAAI,MAAM;EAEjB,aAAa,KAAKM,mBAAmB;CAIvC;CAEA,qBAAqB,mBAA2B;EAC9C,OAAO,KAAKU,eAAe,EAAC,MAAM,kBAAiB,CAAC;CACtD;CAEA,iBAAiB,OAAO,UAAwB;EAC9C,MAAM,OAAO,MAAM,KAAK,SAAS;EACjC,IAAI,KAAKH,SAAS;GAChB,KAAKV,IAAI,QAAQ,0CAA0C,IAAI;GAC/D;EACF;EAEA,IAAI;EACJ,IAAI;GAEF,MAAM,MADQ,KAAK,MAAM,IACN,GAAO,cAAc;EAC1C,SAAS,GAAG;GACV,MAAM,YAAY;IAChB,MAAM;IACN,SAAS,OAAO,CAAC;IACjB,QAAQ;GACV;GACA,KAAKS,gBACH,WACA,IAAI,uBAAuB,WAAW,MAAM,CAC9C;GACA;EACF;EAEA,IAAI;GAEF,IADgB,IAAI,OACJ,QAAQ;IACtB,KAAK,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,qBAAqB;IAC7C;GACF;GAEA,MAAM,SAAS,MAAM,KAAKP,gBAAgB,cAAc,GAAG;GAC3D,KAAK,MAAM,KAAK,QACd,KAAKY,qBAAqB,CAAC;EAE/B,SAAS,GAAG;GACV,KAAKC,iBAAiB,CAAC;EACzB;CACF;CAEA,qBAAqB,QAA6B;EAChD,QAAQ,OAAO,MAAf;GACE,KAAK;IACH,KAAKN,gBAAgB,OAAO,KAAK;IACjC;GACF,KAAK,MACH;GACF,KAAK;IACH,QAAQ,OAAO,QAAf;KACE,KAAK;MACH,OACE,KAAKE,8BAA8B,KAAA,GACnC,kDACF;MACA,KAAKA,4BAA4B,OAAO;MACxC;KACF,KAAK;MACH,OACE,KAAKC,0BAA0B,KAAA,GAC/B,kDACF;MACA,KAAKA,wBAAwB,OAAO;MACpC;IACJ;IACA,KAAKI,eAAe,OAAO,MAAM;IACjC;GAEF,KAAK,aACH,KAAK,MAAM,SAAS,OAAO,QACzB,KAAK,UAAU,KAAK;EAG1B;CACF;CAEA,gBAAgB,MAAkB;EAChC,MAAM,EAAC,MAAM,QAAQ,aAAY;EACjC,IAAI,CAAC,UACH,KAAKC,sBAAsB,eAAe;EAE5C,KAAK,MAAM,yBAAyB;GAAC;GAAM;GAAQ;EAAQ,CAAC;CAC9D;CAEA,gBAAgB,MAAkB;EAChC,KAAKA,sBAAsB,aAAa;EACxC,KAAKjB,IAAI,OAAO,yBAAyB,EAAE,SAAS,EAAE,KAAK;CAC7D;CAEA,sBAAsB,WAAmB;EACvC,KAAKI,iBAAiB,IAAI,GAAG;IAC1B,6BAA6B,KAAKL;IAClC,uBAAuB;EAC1B,CAAC;CACH;CAEA,gBAAgB;EACd,SACE,sBAAsB,KAAKF,GAAG,GAC9B,IAAI,SAAS,EACX,QAAQ,MAAM,WAAW,aAAa;GACpC,KAAKgB,eAAe,EAAC,KAAI,CAAC,EAAE,WAAW,SAAS,GAAG,QAAQ;EAC7D,EACF,CAAC,SAIK,CAAC,CACT;CACF;CAEA,eAAe,gBAAoC;EAMjD,SACE,SAAS,KAAK,cAAc,GAC5B,IAAI,SAAS;GACX,YAAY;GACZ,QAAQ,YAAwB,WAAW,aACzC,KAAK,KAAK,YAAY,QAAQ;EAClC,CAAC,IACD,MACE,IACI,KAAKE,iBAAiB,CAAC,IACvB,KAAK,MAAM,iCAAiC,CACpD;CACF;CAEA,iBAAiB,GAAY;EAC3B,MAAM,YACJ,kBAAkB,CAAC,GAAG,aAAa,sBAAsB,CAAC,EAAE;EAE9D,KAAKN,gBAAgB,WAAW,CAAC;CACnC;CAEA,gBAAgB,WAAsB,QAAkB;EACtD,KAAK,UAAU,WAAW,MAAM;EAChC,KAAK,MACH,GAAG,UAAU,KAAK,IAAI,UAAU,OAAO,KAAK,UAAU,WACtD,SACF;CACF;CAEA,yBAAyB,KAAK,IAAI;CAElC,uBAAuB;EACrB,IAAI,KAAK,IAAI,IAAI,KAAKS,yBAAyB,4BAA4B;GACzE,KAAKlB,IAAI,QAAQ,uBAAuB;GACxC,KAAK,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,qBAAqB;EAC/C;CACF;CAEA,KACE,MACA,UACA;EACA,KAAKkB,yBAAyB,KAAK,IAAI;EACvC,OAAO,KAAK,KAAKlB,KAAK,KAAKH,KAAK,MAAM,QAAQ;CAChD;CAEA,UAAU,WAAsB,QAAkB;EAChD,UAAU,KAAKG,KAAK,KAAKH,KAAK,WAAW,MAAM;CACjD;AACF;AAOA,SAAgB,KACd,IACA,IACA,MACA,UACA;CACA,IAAI,GAAG,eAAe,YAAU,MAC9B,GAAG,KACD,KAAK,UAAU,IAAI,GACnB,aAAa,wBAAwB,KAAA,IAAY,QACnD;MACK;EACL,GAAG,QAAQ,2CAA2C,GAAG,WAAW,IAAI,EACtE,SAAS,KACX,CAAC;EACD,IAAI,aAAa,uBACf,SACE,IAAI,uBACF;GACE,MAAM;GACN,SAAS;GACT,QAAQ;EACV,GACA,MACF,CACF;CAEJ;AACF;AAEA,SAAgB,UACd,IACA,IACA,WACA,QACA;CACA,KAAK,GAAG,YAAY,aAAa,UAAU,IAAI;CAE/C,IAAI;CAGJ,IAAI,kBAAkB,wBACpB,WAAW,OAAO;MAIf,IACH,SAAS,MAAM,KACf,uBAAuB,MAAM,KAC7B,yBAAyB,UAAU,OAAO,GAE1C,WAAW;MAGR,IACH,UAAU,SAAS,oBACnB,UAAU,SAAS,mBAEnB,WAAW;MAEX,WAAW,SAAS,YAAY,MAAM,IAAI;CAG5C,GAAG,YAAY,8BAA8B,WAAW,UAAU,EAAE;CACpE,KAAK,IAAI,IAAI,CAAC,SAAS,SAAS,GAAG,qBAAqB;AAC1D;AAEA,SAAgB,kBAAkB,OAA2C;CAC3E,IAAI,gBAAgB,KAAK,GACvB,OAAO;CAET,IAAI,iBAAiB,SAAS,MAAM,OAClC,OAAO,kBAAkB,MAAM,KAAK;AAGxC;AAEA,SAAS,SAAS,OAAyB;CACzC,OAAO,QACL,SACA,OAAO,UAAU,YACjB,WAAW,SACX,OAAQ,MAA2B,UAAU,WAC/C;AACF;AAIA,IAAM,+BAA+B,IAAI,IAAI;CAC3C;CACA;CACA;AACF,CAAC;AAID,IAAM,oCAAoC,CACxC,mDACF;AAEA,SAAS,uBAAuB,OAAyB;CACvD,IAAI,CAAC,SAAS,OAAO,UAAU,UAC7B,OAAO;CAET,MAAM,YACJ,UAAU,QAAQ,OAAQ,MAA2B,IAAI,IAAI,KAAA;CAC/D,OAAO,QACL,aAAa,6BAA6B,IAAI,UAAU,YAAY,CAAC,CACvE;AACF;AAEA,SAAS,yBAAyB,SAAsC;CACtE,IAAI,CAAC,SACH,OAAO;CAET,MAAM,QAAQ,QAAQ,YAAY;CAClC,OAAO,kCAAkC,MAAK,YAC5C,MAAM,SAAS,OAAO,CACxB;AACF"}

package/out/zero-cache/src/workers/syncer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"syncer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/workers/syncer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,qBAAqB,CAAC;AACrD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAUjD,OAAO,EAAyB,KAAK,iBAAiB,EAAC,MAAM,iBAAiB,CAAC;AAE/E,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,0BAA0B,CAAC;AAOzD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,gCAAgC,CAAC;AAC5D,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,+BAA+B,CAAC;AAC1D,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,sCAAsC,CAAC;AAEvE,OAAO,KAAK,EACV,oBAAoB,EACpB,OAAO,EACP,gBAAgB,EACjB,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAC,wBAAwB,EAAC,MAAM,uDAAuD,CAAC;AACpG,OAAO,EAAC,gBAAgB,EAAC,MAAM,8CAA8C,CAAC;AAC9E,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,wCAAwC,CAAC;AACvE,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,uBAAuB,CAAC;AAClD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAO3D,MAAM,MAAM,gBAAgB,GAAG;IAC7B,cAAc,EAAE,WAAW,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACrC,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG,IAAI,CACrC,UAAU,EACV,aAAa,GAAG,eAAe,CAChC,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,eAAO,MAAM,wBAAwB,QAAS,CAAC;AA8F/C,wBAAgB,wBAAwB,CACtC,GAAG,EAAE,MAAM,EACX,kBAAkB,EAAE,iBAAiB,EAAE,EACvC,WAAW,EAAE,QAAQ,CAAC,oBAAoB,CAAC,GAC1C,eAAe,CAsCjB;AAED,wBAAgB,sBAAsB,CACpC,GAAG,EAAE,MAAM,EACX,kBAAkB,EAAE,iBAAiB,EAAE,EACvC,WAAW,EAAE,QAAQ,CAAC,oBAAoB,CAAC,GAC1C,MAAM,CAER;AA4BD;;;;;;GAMG;AACH,qBAAa,MAAO,YAAW,gBAAgB;;IAC7C,QAAQ,CAAC,EAAE,SAAmB;gBAiB5B,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,EAClB,iBAAiB,EAAE,CACjB,EAAE,EAAE,MAAM,EACV,GAAG,EAAE,YAAY,CAAC,YAAY,CAAC,EAC/B,gBAAgB,EAAE,gBAAgB,KAC/B,UAAU,GAAG,oBAAoB,EACtC,cAAc,EAAE,CAAC,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,GAAG,OAAO,CAAC,GAAG,SAAS,EAC/D,aAAa,EACT,CAAC,CACC,EAAE,EAAE,MAAM,EACV,kBAAkB,EAAE,wBAAwB,KACzC,MAAM,GAAG,OAAO,CAAC,GACtB,SAAS,EACb,MAAM,EAAE,MAAM,EACd,iBAAiB,EAAE,iBAAiB,GAAG,SAAS;IA2TlD,GAAG;IAIH;;;;;OAKG;IACG,KAAK;IAqBX,IAAI;CAKL"}
+{"version":3,"file":"syncer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/workers/syncer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,qBAAqB,CAAC;AACrD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAUjD,OAAO,EAAyB,KAAK,iBAAiB,EAAC,MAAM,iBAAiB,CAAC;AAE/E,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,0BAA0B,CAAC;AAYzD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,gCAAgC,CAAC;AAC5D,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,+BAA+B,CAAC;AAC1D,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,sCAAsC,CAAC;AAEvE,OAAO,KAAK,EACV,oBAAoB,EACpB,OAAO,EACP,gBAAgB,EACjB,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAC,wBAAwB,EAAC,MAAM,uDAAuD,CAAC;AACpG,OAAO,EAAC,gBAAgB,EAAC,MAAM,8CAA8C,CAAC;AAC9E,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,wCAAwC,CAAC;AACvE,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,uBAAuB,CAAC;AAClD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAO3D,MAAM,MAAM,gBAAgB,GAAG;IAC7B,cAAc,EAAE,WAAW,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACrC,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG,IAAI,CACrC,UAAU,EACV,aAAa,GAAG,eAAe,CAChC,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB,CAAC;AAMF,eAAO,MAAM,wBAAwB,QAAS,CAAC;AAkJ/C,wBAAgB,wBAAwB,CACtC,GAAG,EAAE,MAAM,EACX,kBAAkB,EAAE,iBAAiB,EAAE,EACvC,WAAW,EAAE,QAAQ,CAAC,oBAAoB,CAAC,GAC1C,eAAe,CAejB;AAED,wBAAgB,sBAAsB,CACpC,GAAG,EAAE,MAAM,EACX,kBAAkB,EAAE,iBAAiB,EAAE,EACvC,WAAW,EAAE,QAAQ,CAAC,oBAAoB,CAAC,GAC1C,MAAM,CAER;AA4BD;;;;;;GAMG;AACH,qBAAa,MAAO,YAAW,gBAAgB;;IAC7C,QAAQ,CAAC,EAAE,SAAmB;gBAiD5B,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,EAClB,iBAAiB,EAAE,CACjB,EAAE,EAAE,MAAM,EACV,GAAG,EAAE,YAAY,CAAC,YAAY,CAAC,EAC/B,gBAAgB,EAAE,gBAAgB,KAC/B,UAAU,GAAG,oBAAoB,EACtC,cAAc,EAAE,CAAC,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,GAAG,OAAO,CAAC,GAAG,SAAS,EAC/D,aAAa,EACT,CAAC,CACC,EAAE,EAAE,MAAM,EACV,kBAAkB,EAAE,wBAAwB,KACzC,MAAM,GAAG,OAAO,CAAC,GACtB,SAAS,EACb,MAAM,EAAE,MAAM,EACd,iBAAiB,EAAE,iBAAiB,GAAG,SAAS;IA8WlD,GAAG;IAIH;;;;;OAKG;IACG,KAAK;IAqBX,IAAI;CAML"}

package/out/zero-cache/src/workers/syncer.js

@@ -3,7 +3,7 @@ import { Unauthorized } from "../../../zero-protocol/src/error-kind-enum.js";
 import { ZeroCache } from "../../../zero-protocol/src/error-origin-enum.js";
 import { ProtocolError, isProtocolError } from "../../../zero-protocol/src/error.js";
 import "../config/zero-config.js";
-import { getOrCreateGauge } from "../observability/metrics.js";
+import { getOrCreateCounter, getOrCreateGauge, getOrCreateNativeHistogram, getOrCreateUpDownCounter } from "../observability/metrics.js";
 import { installWebSocketReceiver } from "../types/websocket-handoff.js";
 import { createNotifierFrom, subscribeTo } from "./replicator.js";
 import { recordConnectionAttempted, recordConnectionSuccess, setActiveClientGroupsGetter } from "../server/anonymous-otel-start.js";
@@ -18,6 +18,9 @@ import { WebSocketServer } from "ws";
 import { pid } from "node:process";
 //#region ../zero-cache/src/workers/syncer.ts
 var MAX_REPLICA_READY_STATES = 1e4;
+var VIEW_SYNCER_LAG_SAMPLE_INTERVAL_MS = 6e4;
+var PROTOCOL_VERSION_ATTRIBUTE = "protocol.version";
+var FAILURE_REASON_ATTRIBUTE = "reason";
 function boundReplicaReadyStates(replicaReadyStates) {
 	if (replicaReadyStates.length > 1e4) replicaReadyStates.splice(0, replicaReadyStates.length - MAX_REPLICA_READY_STATES);
 }
@@ -55,7 +58,7 @@ function percentileNearestRank(sortedValues, percentile) {
 	if (sortedValues.length === 0) return 0;
 	return sortedValues[Math.min(sortedValues.length - 1, Math.max(0, Math.ceil(percentile / 100 * sortedValues.length) - 1))];
 }
-function computeServingLagStatsMs(now, replicaReadyStates, viewSyncers) {
+function computeServingLagDistributionMs(now, replicaReadyStates, viewSyncers) {
 	const lags = [];
 	let laggingClientGroups = 0;
 	let firstNeededIndex = replicaReadyStates.length;
@@ -79,7 +82,8 @@ function computeServingLagStatsMs(now, replicaReadyStates, viewSyncers) {
 		p50Ms: percentileNearestRank(lags, 50),
 		p75Ms: percentileNearestRank(lags, 75),
 		p99Ms: percentileNearestRank(lags, 99),
-		maxMs: lags.at(-1) ?? 0
+		maxMs: lags.at(-1) ?? 0,
+		lagsMs: lags
 	};
 }
 function getWebSocketServerOptions(config) {
@@ -118,8 +122,17 @@ var Syncer = class {
 	#config;
 	#validateLegacyJWT;
 	#replicaReadyStates = [];
-	#servingLagStatsCache;
-	#servingLagStatsCacheClearQueued = false;
+	#webSocketOpenConnections = getOrCreateUpDownCounter("sync", "websocket.open_connections", "Open client WebSocket connections.");
+	#webSocketConnectionAttempts = getOrCreateCounter("sync", "websocket.connection_attempts", "Client WebSocket connection attempts.");
+	#webSocketConnectionSuccesses = getOrCreateCounter("sync", "websocket.connection_successes", "Client WebSocket connections successfully initialized.");
+	#webSocketConnectionFailures = getOrCreateCounter("sync", "websocket.connection_failures", "Client WebSocket connection attempts that failed before initialization.");
+	#viewSyncerLag = getOrCreateNativeHistogram("sync", "view_syncer_lag", {
+		description: "Lag from replica-ready change to ViewSyncer output for active client groups. A change is output after IVM advancement, CVR flush, and pokeEnd.",
+		unit: "s"
+	});
+	#servingLagDistributionCache;
+	#servingLagDistributionCacheClearQueued = false;
+	#viewSyncerLagSampleInterval;
 	constructor(lc, config, viewSyncerFactory, mutagenFactory, pusherFactory, parent, validateLegacyJWT) {
 		this.#config = config;
 		this.#validateLegacyJWT = validateLegacyJWT;
@@ -144,39 +157,45 @@ var Syncer = class {
 			for (const vs of this.#viewSyncers.getServices()) total += vs.rowCount;
 			result.observe(total);
 		});
-		getOrCreateGauge("sync", "serving-lag", {
+		getOrCreateGauge("sync", "serving_lag", {
 			description: "Maximum time active ViewSyncer client groups have had unserved replica changes. A change is served after IVM advancement, CVR flush, and pokeEnd.",
 			unit: "millisecond"
 		}).addCallback((result) => {
-			result.observe(this.#computeServingLagStats().maxMs);
+			result.observe(this.#computeServingLagDistribution().maxMs);
 		});
-		getOrCreateGauge("sync", "serving-lag-stats", {
+		getOrCreateGauge("sync", "serving_lag_stats", {
 			description: "Distribution of time active ViewSyncer client groups have had unserved replica changes. A change is served after IVM advancement, CVR flush, and pokeEnd.",
 			unit: "millisecond"
 		}).addCallback((result) => {
-			const stats = this.#computeServingLagStats();
+			const stats = this.#computeServingLagDistribution();
 			result.observe(stats.minMs, { stat: "min" });
 			result.observe(stats.p50Ms, { stat: "p50" });
 			result.observe(stats.p75Ms, { stat: "p75" });
 			result.observe(stats.p99Ms, { stat: "p99" });
 			result.observe(stats.maxMs, { stat: "max" });
 		});
-		getOrCreateGauge("sync", "serving-lagging-client-groups", "Number of active client groups with unserved replica changes").addCallback((result) => {
-			result.observe(this.#computeServingLagStats().laggingClientGroups);
+		getOrCreateGauge("sync", "serving_lagging_client_groups", "Number of active client groups with unserved replica changes").addCallback((result) => {
+			result.observe(this.#computeServingLagDistribution().laggingClientGroups);
 		});
+		this.#viewSyncerLagSampleInterval = setInterval(() => this.#recordViewSyncerLagSamples(), VIEW_SYNCER_LAG_SAMPLE_INTERVAL_MS);
+		this.#viewSyncerLagSampleInterval.unref?.();
 	}
-	#computeServingLagStats() {
-		if (this.#servingLagStatsCache) return this.#servingLagStatsCache;
-		const stats = computeServingLagStatsMs(Date.now(), this.#replicaReadyStates, this.#viewSyncers.getServices());
-		this.#servingLagStatsCache = stats;
-		if (!this.#servingLagStatsCacheClearQueued) {
-			this.#servingLagStatsCacheClearQueued = true;
+	#computeServingLagDistribution() {
+		if (this.#servingLagDistributionCache) return this.#servingLagDistributionCache;
+		const distribution = computeServingLagDistributionMs(Date.now(), this.#replicaReadyStates, this.#viewSyncers.getServices());
+		this.#servingLagDistributionCache = distribution;
+		if (!this.#servingLagDistributionCacheClearQueued) {
+			this.#servingLagDistributionCacheClearQueued = true;
 			queueMicrotask(() => {
-				this.#servingLagStatsCache = void 0;
-				this.#servingLagStatsCacheClearQueued = false;
+				this.#servingLagDistributionCache = void 0;
+				this.#servingLagDistributionCacheClearQueued = false;
 			});
 		}
-		return stats;
+		return distribution;
+	}
+	#recordViewSyncerLagSamples() {
+		const { lagsMs } = this.#computeServingLagDistribution();
+		for (const lagMs of lagsMs) this.#viewSyncerLag.recordMs(lagMs);
 	}
 	#recordReplicaReadyState(state) {
 		if (state.watermark === void 0 || state.replicaReadyTimeMs === void 0) return;
@@ -193,6 +212,26 @@ var Syncer = class {
 		boundReplicaReadyStates(this.#replicaReadyStates);
 	}
 	#createConnection = async (ws, params) => {
+		const metricAttributes = { [PROTOCOL_VERSION_ATTRIBUTE]: params.protocolVersion };
+		let connectionResultRecorded = false;
+		const recordConnectionFailure = (reason) => {
+			if (connectionResultRecorded) return;
+			connectionResultRecorded = true;
+			this.#webSocketConnectionFailures.add(1, {
+				...metricAttributes,
+				[FAILURE_REASON_ATTRIBUTE]: reason
+			});
+		};
+		const recordConnectionSuccessMetric = () => {
+			if (connectionResultRecorded) return;
+			connectionResultRecorded = true;
+			this.#webSocketConnectionSuccesses.add(1, metricAttributes);
+		};
+		this.#webSocketConnectionAttempts.add(1, metricAttributes);
+		this.#webSocketOpenConnections.add(1, metricAttributes);
+		ws.once("close", () => {
+			this.#webSocketOpenConnections.add(-1, metricAttributes);
+		});
 		this.#lc.debug?.("creating connection", params.clientGroupID, params.clientID);
 		recordConnectionAttempted();
 		const { clientID, clientGroupID, auth, userID } = params;
@@ -202,7 +241,10 @@ var Syncer = class {
 			const tokenOptions = tokenConfigOptions(this.#config.auth ?? {});
 			const hasPushOrMutate = this.#config?.push?.url !== void 0 || this.#config?.mutate?.url !== void 0;
 			const hasQueries = this.#config?.query?.url !== void 0 || this.#config?.getQueries?.url !== void 0;
-			if (!(tokenOptions.length === 1) && !(hasPushOrMutate && hasQueries)) throw new Error("Exactly one of jwk, secret, or jwksUrl must be set in order to verify tokens but actually the following were set: " + JSON.stringify(tokenOptions) + ". You may also set both ZERO_MUTATE_URL and ZERO_QUERY_URL to enable custom mutations and queries without passing token verification options.");
+			if (!(tokenOptions.length === 1) && !(hasPushOrMutate && hasQueries)) {
+				recordConnectionFailure("configuration");
+				throw new Error("Exactly one of jwk, secret, or jwksUrl must be set in order to verify tokens but actually the following were set: " + JSON.stringify(tokenOptions) + ". You may also set both ZERO_MUTATE_URL and ZERO_QUERY_URL to enable custom mutations and queries without passing token verification options.");
+			}
 		}
 		let initialAuth;
 		try {
@@ -216,10 +258,12 @@ var Syncer = class {
 					hasProvidedAuth,
 					errorKind: e.message
 				});
+				recordConnectionFailure("auth");
 				sendError(this.#lc, ws, e.errorBody);
 				ws.close(3e3, e.errorBody.message);
 				return;
 			}
+			recordConnectionFailure("internal");
 			throw e;
 		}
 		const viewSyncer = this.#viewSyncers.getService(clientGroupID);
@@ -231,6 +275,7 @@ var Syncer = class {
 				message: "Client groups are pinned to a single userID. Connection userID does not match existing client group userID.",
 				origin: ZeroCache
 			});
+			recordConnectionFailure("user_mismatch");
 			sendError(this.#lc, ws, error.errorBody);
 			ws.close(3e3, error.message);
 			return;
@@ -260,6 +305,7 @@ var Syncer = class {
 				pusher?.unref();
 			});
 		} catch (e) {
+			recordConnectionFailure("internal");
 			connContextManager.closeConnection({
 				clientID,
 				wsID: params.wsID
@@ -269,7 +315,10 @@ var Syncer = class {
 			throw e;
 		}
 		this.#connections.set(clientID, connection);
-		connection.init() && recordConnectionSuccess();
+		if (connection.init()) {
+			recordConnectionSuccessMetric();
+			recordConnectionSuccess();
+		} else recordConnectionFailure("protocol_version");
 		if (params.initConnectionMsg) {
 			this.#lc.debug?.("handling init connection message from sec header", params.clientGroupID, params.clientID);
 			await connection.handleInitConnection(JSON.stringify(params.initConnectionMsg));
@@ -299,6 +348,7 @@ var Syncer = class {
 		this.#lc.info?.(`finished draining (${Date.now() - start} ms)`);
 	}
 	stop() {
+		clearInterval(this.#viewSyncerLagSampleInterval);
 		this.#wss.close();
 		this.#stopped.resolve();
 		return promiseVoid;

package/out/zero-cache/src/workers/syncer.js.map

@@ -1 +1 @@
-{"version":3,"file":"syncer.js","names":["#lc","#viewSyncers","#mutagens","#pushers","#connections","#drainCoordinator","#parent","#wss","#stopped","#config","#validateLegacyJWT","#replicaReadyStates","#recordReplicaReadyState","#createConnection","#computeServingLagStats","#servingLagStatsCache","#servingLagStatsCacheClearQueued"],"sources":["../../../../../zero-cache/src/workers/syncer.ts"],"sourcesContent":["import {pid} from 'node:process';\nimport type {MessagePort} from 'node:worker_threads';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {WebSocketServer, type ServerOptions, type WebSocket} from 'ws';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {ErrorKind} from '../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../zero-protocol/src/error-origin.ts';\nimport {\n  isProtocolError,\n  ProtocolError,\n} from '../../../zero-protocol/src/error.ts';\nimport {resolveAuth, type Auth, type ValidateLegacyJWT} from '../auth/auth.ts';\nimport {tokenConfigOptions} from '../auth/jwt.ts';\nimport {type ZeroConfig} from '../config/zero-config.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {\n  recordConnectionAttempted,\n  recordConnectionSuccess,\n  setActiveClientGroupsGetter,\n} from '../server/anonymous-otel-start.ts';\nimport type {Mutagen} from '../services/mutagen/mutagen.ts';\nimport type {Pusher} from '../services/mutagen/pusher.ts';\nimport type {ReplicaState} from '../services/replicator/replicator.ts';\nimport {ServiceRunner} from '../services/runner.ts';\nimport type {\n  ActivityBasedService,\n  Service,\n  SingletonService,\n} from '../services/service.ts';\nimport type {ConnectionContextManager} from '../services/view-syncer/connection-context-manager.ts';\nimport {DrainCoordinator} from '../services/view-syncer/drain-coordinator.ts';\nimport type {ViewSyncer} from '../services/view-syncer/view-syncer.ts';\nimport type {Worker} from '../types/processes.ts';\nimport type {Subscription} from '../types/subscription.ts';\nimport {installWebSocketReceiver} from '../types/websocket-handoff.ts';\nimport type {ConnectParams} from './connect-params.ts';\nimport {Connection, sendError} from './connection.ts';\nimport {createNotifierFrom, subscribeTo} from './replicator.ts';\nimport {SyncerWsMessageHandler} from './syncer-ws-message-handler.ts';\n\nexport type SyncerWorkerData = {\n  replicatorPort: MessagePort;\n};\n\nexport type ReplicaReadyState = {\n  readonly watermark: string;\n  readonly replicaReadyTimeMs: number;\n};\n\nexport type ServingLagViewSyncer = Pick<\n  ViewSyncer,\n  'createdAtMs' | 'servedVersion'\n>;\n\nexport type ServingLagStats = {\n  readonly activeClientGroups: number;\n  readonly laggingClientGroups: number;\n  readonly minMs: number;\n  readonly p50Ms: number;\n  readonly p75Ms: number;\n  readonly p99Ms: number;\n  readonly maxMs: number;\n};\n\nexport const MAX_REPLICA_READY_STATES = 10_000;\n\nfunction boundReplicaReadyStates(\n  replicaReadyStates: ReplicaReadyState[],\n): void {\n  if (replicaReadyStates.length > MAX_REPLICA_READY_STATES) {\n    replicaReadyStates.splice(\n      0,\n      replicaReadyStates.length - MAX_REPLICA_READY_STATES,\n    );\n  }\n}\n\nfunction pruneReplicaReadyStates(\n  replicaReadyStates: ReplicaReadyState[],\n  firstNeededIndex: number,\n): void {\n  if (firstNeededIndex > 0) {\n    replicaReadyStates.splice(0, firstNeededIndex);\n  }\n\n  boundReplicaReadyStates(replicaReadyStates);\n}\n\nfunction lowerBoundReplicaReadyTimeMs(\n  replicaReadyStates: readonly ReplicaReadyState[],\n  replicaReadyTimeMs: number,\n): number {\n  let low = 0;\n  let high = replicaReadyStates.length;\n  while (low < high) {\n    const mid = low + Math.floor((high - low) / 2);\n    if (replicaReadyStates[mid].replicaReadyTimeMs < replicaReadyTimeMs) {\n      low = mid + 1;\n    } else {\n      high = mid;\n    }\n  }\n  return low;\n}\n\nfunction upperBoundWatermark(\n  replicaReadyStates: readonly ReplicaReadyState[],\n  watermark: string,\n): number {\n  let low = 0;\n  let high = replicaReadyStates.length;\n  while (low < high) {\n    const mid = low + Math.floor((high - low) / 2);\n    if (replicaReadyStates[mid].watermark <= watermark) {\n      low = mid + 1;\n    } else {\n      high = mid;\n    }\n  }\n  return low;\n}\n\nfunction findFirstUnservedIndex(\n  replicaReadyStates: readonly ReplicaReadyState[],\n  viewSyncer: ServingLagViewSyncer,\n): number {\n  const firstReadyAfterCreation = lowerBoundReplicaReadyTimeMs(\n    replicaReadyStates,\n    viewSyncer.createdAtMs,\n  );\n  const firstAfterServedVersion =\n    viewSyncer.servedVersion === null\n      ? 0\n      : upperBoundWatermark(replicaReadyStates, viewSyncer.servedVersion);\n\n  const firstUnservedIndex = Math.max(\n    firstReadyAfterCreation,\n    firstAfterServedVersion,\n  );\n  return firstUnservedIndex < replicaReadyStates.length\n    ? firstUnservedIndex\n    : -1;\n}\n\nfunction percentileNearestRank(\n  sortedValues: readonly number[],\n  percentile: number,\n): number {\n  if (sortedValues.length === 0) {\n    return 0;\n  }\n  const index = Math.min(\n    sortedValues.length - 1,\n    Math.max(0, Math.ceil((percentile / 100) * sortedValues.length) - 1),\n  );\n  return sortedValues[index];\n}\n\nexport function computeServingLagStatsMs(\n  now: number,\n  replicaReadyStates: ReplicaReadyState[],\n  viewSyncers: Iterable<ServingLagViewSyncer>,\n): ServingLagStats {\n  const lags: number[] = [];\n  let laggingClientGroups = 0;\n  let firstNeededIndex = replicaReadyStates.length;\n\n  for (const viewSyncer of viewSyncers) {\n    const firstUnservedIndex = findFirstUnservedIndex(\n      replicaReadyStates,\n      viewSyncer,\n    );\n    if (firstUnservedIndex === -1) {\n      lags.push(0);\n      continue;\n    }\n\n    firstNeededIndex = Math.min(firstNeededIndex, firstUnservedIndex);\n    const lagMs = Math.max(\n      0,\n      now - replicaReadyStates[firstUnservedIndex].replicaReadyTimeMs,\n    );\n    lags.push(lagMs);\n    if (lagMs > 0) {\n      laggingClientGroups++;\n    }\n  }\n\n  pruneReplicaReadyStates(replicaReadyStates, firstNeededIndex);\n\n  lags.sort((a, b) => a - b);\n  return {\n    activeClientGroups: lags.length,\n    laggingClientGroups,\n    minMs: lags[0] ?? 0,\n    p50Ms: percentileNearestRank(lags, 50),\n    p75Ms: percentileNearestRank(lags, 75),\n    p99Ms: percentileNearestRank(lags, 99),\n    maxMs: lags.at(-1) ?? 0,\n  };\n}\n\nexport function computeMaxServingLagMs(\n  now: number,\n  replicaReadyStates: ReplicaReadyState[],\n  viewSyncers: Iterable<ServingLagViewSyncer>,\n): number {\n  return computeServingLagStatsMs(now, replicaReadyStates, viewSyncers).maxMs;\n}\n\nfunction getWebSocketServerOptions(config: ZeroConfig): ServerOptions {\n  const options: ServerOptions = {\n    noServer: true,\n    maxPayload: config.websocketMaxPayloadBytes,\n  };\n\n  if (config.websocketCompression) {\n    options.perMessageDeflate = true;\n\n    if (config.websocketCompressionOptions) {\n      try {\n        const compressionOptions = JSON.parse(\n          config.websocketCompressionOptions,\n        );\n        options.perMessageDeflate = compressionOptions;\n      } catch (e) {\n        throw new Error(\n          `Failed to parse ZERO_WEBSOCKET_COMPRESSION_OPTIONS: ${String(e)}. Expected valid JSON.`,\n        );\n      }\n    }\n  }\n\n  return options;\n}\n\n/**\n * The Syncer worker receives websocket handoffs for \"/sync\" connections\n * from the Dispatcher in the main thread, and creates websocket\n * {@link Connection}s with a corresponding {@link ViewSyncer}, {@link Mutagen},\n * and {@link Subscription} to version notifications from the Replicator\n * worker.\n */\nexport class Syncer implements SingletonService {\n  readonly id = `syncer-${pid}`;\n  readonly #lc: LogContext;\n  readonly #viewSyncers: ServiceRunner<ViewSyncer & ActivityBasedService>;\n  readonly #mutagens: ServiceRunner<Mutagen & Service> | undefined;\n  readonly #pushers: ServiceRunner<Pusher & Service> | undefined;\n  readonly #connections = new Map<string, Connection>();\n  readonly #drainCoordinator = new DrainCoordinator();\n  readonly #parent: Worker;\n  readonly #wss: WebSocketServer;\n  readonly #stopped = resolver();\n  readonly #config: ZeroConfig;\n  readonly #validateLegacyJWT: ValidateLegacyJWT | undefined;\n  readonly #replicaReadyStates: ReplicaReadyState[] = [];\n  #servingLagStatsCache: ServingLagStats | undefined;\n  #servingLagStatsCacheClearQueued = false;\n\n  constructor(\n    lc: LogContext,\n    config: ZeroConfig,\n    viewSyncerFactory: (\n      id: string,\n      sub: Subscription<ReplicaState>,\n      drainCoordinator: DrainCoordinator,\n    ) => ViewSyncer & ActivityBasedService,\n    mutagenFactory: ((id: string) => Mutagen & Service) | undefined,\n    pusherFactory:\n      | ((\n          id: string,\n          connContextManager: ConnectionContextManager,\n        ) => Pusher & Service)\n      | undefined,\n    parent: Worker,\n    validateLegacyJWT: ValidateLegacyJWT | undefined,\n  ) {\n    this.#config = config;\n    this.#validateLegacyJWT = validateLegacyJWT;\n    // Relays notifications from the parent thread subscription\n    // to ViewSyncers within this thread.\n    const notifier = createNotifierFrom(lc, parent, state =>\n      this.#recordReplicaReadyState(state),\n    );\n    subscribeTo(lc, parent);\n\n    this.#lc = lc;\n    this.#viewSyncers = new ServiceRunner(\n      lc,\n      id => viewSyncerFactory(id, notifier.subscribe(), this.#drainCoordinator),\n      v => v.keepalive(),\n    );\n    if (mutagenFactory) {\n      this.#mutagens = new ServiceRunner(lc, mutagenFactory, m => m.hasRefs());\n    }\n    if (pusherFactory) {\n      this.#pushers = new ServiceRunner(\n        lc,\n        id =>\n          pusherFactory(\n            id,\n            this.#viewSyncers.getService(id).connContextManager,\n          ),\n        p => p.hasRefs(),\n      );\n    }\n    this.#parent = parent;\n    this.#wss = new WebSocketServer(getWebSocketServerOptions(config));\n\n    installWebSocketReceiver(\n      lc,\n      this.#wss,\n      this.#createConnection,\n      this.#parent,\n    );\n\n    setActiveClientGroupsGetter(() => this.#viewSyncers.size);\n\n    getOrCreateGauge(\n      'sync',\n      'active-client-groups',\n      'Number of active client groups',\n    ).addCallback(result => result.observe(this.#viewSyncers.size));\n\n    getOrCreateGauge(\n      'sync',\n      'queries',\n      'Active queries (pipelines) across all client groups',\n    ).addCallback(result => {\n      let total = 0;\n      for (const vs of this.#viewSyncers.getServices()) {\n        total += vs.queryCount;\n      }\n      result.observe(total);\n    });\n\n    getOrCreateGauge(\n      'sync',\n      'rows',\n      'Tracked rows across all client groups',\n    ).addCallback(result => {\n      let total = 0;\n      for (const vs of this.#viewSyncers.getServices()) {\n        total += vs.rowCount;\n      }\n      result.observe(total);\n    });\n\n    getOrCreateGauge('sync', 'serving-lag', {\n      description:\n        'Maximum time active ViewSyncer client groups have had unserved ' +\n        'replica changes. A change is served after IVM advancement, CVR flush, ' +\n        'and pokeEnd.',\n      unit: 'millisecond',\n    }).addCallback(result => {\n      result.observe(this.#computeServingLagStats().maxMs);\n    });\n\n    getOrCreateGauge('sync', 'serving-lag-stats', {\n      description:\n        'Distribution of time active ViewSyncer client groups have had ' +\n        'unserved replica changes. A change is served after IVM advancement, ' +\n        'CVR flush, and pokeEnd.',\n      unit: 'millisecond',\n    }).addCallback(result => {\n      const stats = this.#computeServingLagStats();\n      result.observe(stats.minMs, {stat: 'min'});\n      result.observe(stats.p50Ms, {stat: 'p50'});\n      result.observe(stats.p75Ms, {stat: 'p75'});\n      result.observe(stats.p99Ms, {stat: 'p99'});\n      result.observe(stats.maxMs, {stat: 'max'});\n    });\n\n    getOrCreateGauge(\n      'sync',\n      'serving-lagging-client-groups',\n      'Number of active client groups with unserved replica changes',\n    ).addCallback(result => {\n      result.observe(this.#computeServingLagStats().laggingClientGroups);\n    });\n  }\n\n  #computeServingLagStats(): ServingLagStats {\n    if (this.#servingLagStatsCache) {\n      return this.#servingLagStatsCache;\n    }\n\n    const stats = computeServingLagStatsMs(\n      Date.now(),\n      this.#replicaReadyStates,\n      this.#viewSyncers.getServices(),\n    );\n    this.#servingLagStatsCache = stats;\n    if (!this.#servingLagStatsCacheClearQueued) {\n      this.#servingLagStatsCacheClearQueued = true;\n      queueMicrotask(() => {\n        this.#servingLagStatsCache = undefined;\n        this.#servingLagStatsCacheClearQueued = false;\n      });\n    }\n    return stats;\n  }\n\n  #recordReplicaReadyState(state: ReplicaState): void {\n    if (\n      state.watermark === undefined ||\n      state.replicaReadyTimeMs === undefined\n    ) {\n      return;\n    }\n    const last = this.#replicaReadyStates.at(-1);\n    if (last && last.watermark >= state.watermark) {\n      return;\n    }\n    this.#replicaReadyStates.push({\n      watermark: state.watermark,\n      replicaReadyTimeMs: state.replicaReadyTimeMs,\n    });\n    if (this.#viewSyncers.size === 0) {\n      this.#replicaReadyStates.length = 0;\n      return;\n    }\n    boundReplicaReadyStates(this.#replicaReadyStates);\n  }\n\n  readonly #createConnection = async (ws: WebSocket, params: ConnectParams) => {\n    this.#lc.debug?.(\n      'creating connection',\n      params.clientGroupID,\n      params.clientID,\n    );\n    recordConnectionAttempted();\n    const {clientID, clientGroupID, auth, userID} = params;\n    const hasProvidedAuth = auth !== undefined && auth !== '';\n    const incomingUserID = userID ?? null;\n\n    if (hasProvidedAuth) {\n      const tokenOptions = tokenConfigOptions(this.#config.auth ?? {});\n\n      const hasPushOrMutate =\n        this.#config?.push?.url !== undefined ||\n        this.#config?.mutate?.url !== undefined;\n      const hasQueries =\n        this.#config?.query?.url !== undefined ||\n        this.#config?.getQueries?.url !== undefined;\n\n      // must either have one of the token options set or have custom mutations & queries enabled\n      const hasExactlyOneTokenOption = tokenOptions.length === 1;\n      const hasCustomEndpoints = hasPushOrMutate && hasQueries;\n      if (!hasExactlyOneTokenOption && !hasCustomEndpoints) {\n        throw new Error(\n          'Exactly one of jwk, secret, or jwksUrl must be set in order to verify tokens but actually the following were set: ' +\n            JSON.stringify(tokenOptions) +\n            '. You may also set both ZERO_MUTATE_URL and ZERO_QUERY_URL to enable custom mutations and queries without passing token verification options.',\n        );\n      }\n    }\n\n    let initialAuth: Auth | undefined;\n\n    // Verify JWT BEFORE touching existing connections - prevents unauthenticated\n    // attackers from force-disconnecting legitimate users via DoS.\n    try {\n      initialAuth = await resolveAuth(\n        this.#lc\n          .withContext('clientGroupID', clientGroupID)\n          .withContext('clientID', clientID),\n        // no previous auth, since this is a new connection, and resolveAuth is\n        // connection scoped, not client group scoped\n        undefined,\n        incomingUserID,\n        auth,\n        this.#validateLegacyJWT,\n      );\n    } catch (e) {\n      if (isProtocolError(e)) {\n        this.#lc.warn?.(\n          'Rejecting sync connection during initial auth resolution',\n          {\n            clientGroupID,\n            clientID,\n            incomingUserID,\n            hasProvidedAuth,\n            errorKind: e.message,\n          },\n        );\n        sendError(this.#lc, ws, e.errorBody);\n        ws.close(3000, e.errorBody.message);\n        return;\n      }\n      throw e;\n    }\n\n    const viewSyncer = this.#viewSyncers.getService(clientGroupID);\n    const connContextManager = viewSyncer.connContextManager;\n    const group = connContextManager.getGroupState();\n\n    // TODO(0xcadams): we only check for user ID mismatch here if the group is\n    // already validated. This prevents wrong-user reconnects from evicting a\n    // healthy connection, but it does not protect against same-user reconnects\n    // with an invalid opaque token. The long-term fix is to keep the replacement\n    // connection pending until its auth is fully validated, and only then replace\n    // the existing socket.\n    if (\n      group.pinnedUser !== undefined &&\n      group.pinnedUser.id !== incomingUserID\n    ) {\n      const error = new ProtocolError({\n        kind: ErrorKind.Unauthorized,\n        message:\n          'Client groups are pinned to a single userID. Connection userID does not match existing client group userID.',\n        origin: ErrorOrigin.ZeroCache,\n      });\n      sendError(this.#lc, ws, error.errorBody);\n      ws.close(3000, error.message);\n      return;\n    }\n\n    // Check for and close existing connections AFTER auth is validated\n    const existing = this.#connections.get(clientID);\n    if (existing) {\n      this.#lc.debug?.(\n        `client ${clientID} already connected, closing existing connection`,\n      );\n      existing.close(`replaced by ${params.wsID}`);\n    }\n\n    connContextManager.registerConnection(\n      {clientID, wsID: params.wsID},\n      params,\n      initialAuth,\n    );\n\n    const mutagen = this.#mutagens?.getService(clientGroupID);\n    const pusher = this.#pushers?.getService(clientGroupID);\n    // a new connection is using the mutagen and pusher. Bump their ref counts.\n    mutagen?.ref();\n    pusher?.ref();\n\n    let connection: Connection;\n    try {\n      connection = new Connection(\n        this.#lc,\n        params,\n        ws,\n        new SyncerWsMessageHandler(\n          this.#lc,\n          params,\n          connContextManager,\n          viewSyncer,\n          mutagen,\n          pusher,\n        ),\n        () => {\n          connContextManager.closeConnection({\n            clientID,\n            wsID: params.wsID,\n          });\n          if (this.#connections.get(clientID) === connection) {\n            this.#connections.delete(clientID);\n          }\n          // Connection is closed. We can unref the mutagen and pusher.\n          // If their ref counts are zero, they will stop themselves and set themselves invalid.\n          mutagen?.unref();\n          pusher?.unref();\n        },\n      );\n    } catch (e) {\n      connContextManager.closeConnection({clientID, wsID: params.wsID});\n      mutagen?.unref();\n      pusher?.unref();\n      throw e;\n    }\n\n    this.#connections.set(clientID, connection);\n\n    connection.init() && recordConnectionSuccess();\n\n    if (params.initConnectionMsg) {\n      this.#lc.debug?.(\n        'handling init connection message from sec header',\n        params.clientGroupID,\n        params.clientID,\n      );\n      await connection.handleInitConnection(\n        JSON.stringify(params.initConnectionMsg),\n      );\n    }\n  };\n\n  run() {\n    return this.#stopped.promise;\n  }\n\n  /**\n   * Graceful shutdown involves shutting down view syncers one at a time, pausing\n   * for the duration of view syncer's hydration between each one. This paces the\n   * disconnects to avoid creating a backlog of hydrations in the receiving server\n   * when the clients reconnect.\n   */\n  async drain() {\n    const start = Date.now();\n    this.#lc.info?.(`draining ${this.#viewSyncers.size} view-syncers`);\n\n    this.#drainCoordinator.drainNextIn(0);\n\n    while (this.#viewSyncers.size) {\n      await this.#drainCoordinator.forceDrainTimeout;\n\n      // Pick an arbitrary view syncer to force drain.\n      for (const vs of this.#viewSyncers.getServices()) {\n        this.#lc.debug?.(`draining view-syncer ${vs.id} (forced)`);\n        // When this drain or an elective drain completes, the forceDrainTimeout will\n        // resolve after the next drain interval.\n        void vs.stop();\n        break;\n      }\n    }\n    this.#lc.info?.(`finished draining (${Date.now() - start} ms)`);\n  }\n\n  stop() {\n    this.#wss.close();\n    this.#stopped.resolve();\n    return promiseVoid;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;AAiEA,IAAa,2BAA2B;AAExC,SAAS,wBACP,oBACM;CACN,IAAI,mBAAmB,SAAA,KACrB,mBAAmB,OACjB,GACA,mBAAmB,SAAS,wBAC9B;AAEJ;AAEA,SAAS,wBACP,oBACA,kBACM;CACN,IAAI,mBAAmB,GACrB,mBAAmB,OAAO,GAAG,gBAAgB;CAG/C,wBAAwB,kBAAkB;AAC5C;AAEA,SAAS,6BACP,oBACA,oBACQ;CACR,IAAI,MAAM;CACV,IAAI,OAAO,mBAAmB;CAC9B,OAAO,MAAM,MAAM;EACjB,MAAM,MAAM,MAAM,KAAK,OAAO,OAAO,OAAO,CAAC;EAC7C,IAAI,mBAAmB,KAAK,qBAAqB,oBAC/C,MAAM,MAAM;OAEZ,OAAO;CAEX;CACA,OAAO;AACT;AAEA,SAAS,oBACP,oBACA,WACQ;CACR,IAAI,MAAM;CACV,IAAI,OAAO,mBAAmB;CAC9B,OAAO,MAAM,MAAM;EACjB,MAAM,MAAM,MAAM,KAAK,OAAO,OAAO,OAAO,CAAC;EAC7C,IAAI,mBAAmB,KAAK,aAAa,WACvC,MAAM,MAAM;OAEZ,OAAO;CAEX;CACA,OAAO;AACT;AAEA,SAAS,uBACP,oBACA,YACQ;CACR,MAAM,0BAA0B,6BAC9B,oBACA,WAAW,WACb;CACA,MAAM,0BACJ,WAAW,kBAAkB,OACzB,IACA,oBAAoB,oBAAoB,WAAW,aAAa;CAEtE,MAAM,qBAAqB,KAAK,IAC9B,yBACA,uBACF;CACA,OAAO,qBAAqB,mBAAmB,SAC3C,qBACA;AACN;AAEA,SAAS,sBACP,cACA,YACQ;CACR,IAAI,aAAa,WAAW,GAC1B,OAAO;CAMT,OAAO,aAJO,KAAK,IACjB,aAAa,SAAS,GACtB,KAAK,IAAI,GAAG,KAAK,KAAM,aAAa,MAAO,aAAa,MAAM,IAAI,CAAC,CAEjD;AACtB;AAEA,SAAgB,yBACd,KACA,oBACA,aACiB;CACjB,MAAM,OAAiB,CAAC;CACxB,IAAI,sBAAsB;CAC1B,IAAI,mBAAmB,mBAAmB;CAE1C,KAAK,MAAM,cAAc,aAAa;EACpC,MAAM,qBAAqB,uBACzB,oBACA,UACF;EACA,IAAI,uBAAuB,IAAI;GAC7B,KAAK,KAAK,CAAC;GACX;EACF;EAEA,mBAAmB,KAAK,IAAI,kBAAkB,kBAAkB;EAChE,MAAM,QAAQ,KAAK,IACjB,GACA,MAAM,mBAAmB,oBAAoB,kBAC/C;EACA,KAAK,KAAK,KAAK;EACf,IAAI,QAAQ,GACV;CAEJ;CAEA,wBAAwB,oBAAoB,gBAAgB;CAE5D,KAAK,MAAM,GAAG,MAAM,IAAI,CAAC;CACzB,OAAO;EACL,oBAAoB,KAAK;EACzB;EACA,OAAO,KAAK,MAAM;EAClB,OAAO,sBAAsB,MAAM,EAAE;EACrC,OAAO,sBAAsB,MAAM,EAAE;EACrC,OAAO,sBAAsB,MAAM,EAAE;EACrC,OAAO,KAAK,GAAG,EAAE,KAAK;CACxB;AACF;AAUA,SAAS,0BAA0B,QAAmC;CACpE,MAAM,UAAyB;EAC7B,UAAU;EACV,YAAY,OAAO;CACrB;CAEA,IAAI,OAAO,sBAAsB;EAC/B,QAAQ,oBAAoB;EAE5B,IAAI,OAAO,6BACT,IAAI;GAIF,QAAQ,oBAHmB,KAAK,MAC9B,OAAO,2BAEmB;EAC9B,SAAS,GAAG;GACV,MAAM,IAAI,MACR,uDAAuD,OAAO,CAAC,EAAE,uBACnE;EACF;CAEJ;CAEA,OAAO;AACT;;;;;;;;AASA,IAAa,SAAb,MAAgD;CAC9C,KAAc,UAAU;CACxB;CACA;CACA;CACA;CACA,+BAAwB,IAAI,IAAwB;CACpD,oBAA6B,IAAI,iBAAiB;CAClD;CACA;CACA,WAAoB,SAAS;CAC7B;CACA;CACA,sBAAoD,CAAC;CACrD;CACA,mCAAmC;CAEnC,YACE,IACA,QACA,mBAKA,gBACA,eAMA,QACA,mBACA;EACA,KAAKS,UAAU;EACf,KAAKC,qBAAqB;EAG1B,MAAM,WAAW,mBAAmB,IAAI,SAAQ,UAC9C,KAAKE,yBAAyB,KAAK,CACrC;EACA,YAAY,IAAI,MAAM;EAEtB,KAAKZ,MAAM;EACX,KAAKC,eAAe,IAAI,cACtB,KACA,OAAM,kBAAkB,IAAI,SAAS,UAAU,GAAG,KAAKI,iBAAiB,IACxE,MAAK,EAAE,UAAU,CACnB;EACA,IAAI,gBACF,KAAKH,YAAY,IAAI,cAAc,IAAI,iBAAgB,MAAK,EAAE,QAAQ,CAAC;EAEzE,IAAI,eACF,KAAKC,WAAW,IAAI,cAClB,KACA,OACE,cACE,IACA,KAAKF,aAAa,WAAW,EAAE,EAAE,kBACnC,IACF,MAAK,EAAE,QAAQ,CACjB;EAEF,KAAKK,UAAU;EACf,KAAKC,OAAO,IAAI,gBAAgB,0BAA0B,MAAM,CAAC;EAEjE,yBACE,IACA,KAAKA,MACL,KAAKM,mBACL,KAAKP,OACP;EAEA,kCAAkC,KAAKL,aAAa,IAAI;EAExD,iBACE,QACA,wBACA,gCACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKA,aAAa,IAAI,CAAC;EAE9D,iBACE,QACA,WACA,qDACF,EAAE,aAAY,WAAU;GACtB,IAAI,QAAQ;GACZ,KAAK,MAAM,MAAM,KAAKA,aAAa,YAAY,GAC7C,SAAS,GAAG;GAEd,OAAO,QAAQ,KAAK;EACtB,CAAC;EAED,iBACE,QACA,QACA,uCACF,EAAE,aAAY,WAAU;GACtB,IAAI,QAAQ;GACZ,KAAK,MAAM,MAAM,KAAKA,aAAa,YAAY,GAC7C,SAAS,GAAG;GAEd,OAAO,QAAQ,KAAK;EACtB,CAAC;EAED,iBAAiB,QAAQ,eAAe;GACtC,aACE;GAGF,MAAM;EACR,CAAC,EAAE,aAAY,WAAU;GACvB,OAAO,QAAQ,KAAKa,wBAAwB,EAAE,KAAK;EACrD,CAAC;EAED,iBAAiB,QAAQ,qBAAqB;GAC5C,aACE;GAGF,MAAM;EACR,CAAC,EAAE,aAAY,WAAU;GACvB,MAAM,QAAQ,KAAKA,wBAAwB;GAC3C,OAAO,QAAQ,MAAM,OAAO,EAAC,MAAM,MAAK,CAAC;GACzC,OAAO,QAAQ,MAAM,OAAO,EAAC,MAAM,MAAK,CAAC;GACzC,OAAO,QAAQ,MAAM,OAAO,EAAC,MAAM,MAAK,CAAC;GACzC,OAAO,QAAQ,MAAM,OAAO,EAAC,MAAM,MAAK,CAAC;GACzC,OAAO,QAAQ,MAAM,OAAO,EAAC,MAAM,MAAK,CAAC;EAC3C,CAAC;EAED,iBACE,QACA,iCACA,8DACF,EAAE,aAAY,WAAU;GACtB,OAAO,QAAQ,KAAKA,wBAAwB,EAAE,mBAAmB;EACnE,CAAC;CACH;CAEA,0BAA2C;EACzC,IAAI,KAAKC,uBACP,OAAO,KAAKA;EAGd,MAAM,QAAQ,yBACZ,KAAK,IAAI,GACT,KAAKJ,qBACL,KAAKV,aAAa,YAAY,CAChC;EACA,KAAKc,wBAAwB;EAC7B,IAAI,CAAC,KAAKC,kCAAkC;GAC1C,KAAKA,mCAAmC;GACxC,qBAAqB;IACnB,KAAKD,wBAAwB,KAAA;IAC7B,KAAKC,mCAAmC;GAC1C,CAAC;EACH;EACA,OAAO;CACT;CAEA,yBAAyB,OAA2B;EAClD,IACE,MAAM,cAAc,KAAA,KACpB,MAAM,uBAAuB,KAAA,GAE7B;EAEF,MAAM,OAAO,KAAKL,oBAAoB,GAAG,EAAE;EAC3C,IAAI,QAAQ,KAAK,aAAa,MAAM,WAClC;EAEF,KAAKA,oBAAoB,KAAK;GAC5B,WAAW,MAAM;GACjB,oBAAoB,MAAM;EAC5B,CAAC;EACD,IAAI,KAAKV,aAAa,SAAS,GAAG;GAChC,KAAKU,oBAAoB,SAAS;GAClC;EACF;EACA,wBAAwB,KAAKA,mBAAmB;CAClD;CAEA,oBAA6B,OAAO,IAAe,WAA0B;EAC3E,KAAKX,IAAI,QACP,uBACA,OAAO,eACP,OAAO,QACT;EACA,0BAA0B;EAC1B,MAAM,EAAC,UAAU,eAAe,MAAM,WAAU;EAChD,MAAM,kBAAkB,SAAS,KAAA,KAAa,SAAS;EACvD,MAAM,iBAAiB,UAAU;EAEjC,IAAI,iBAAiB;GACnB,MAAM,eAAe,mBAAmB,KAAKS,QAAQ,QAAQ,CAAC,CAAC;GAE/D,MAAM,kBACJ,KAAKA,SAAS,MAAM,QAAQ,KAAA,KAC5B,KAAKA,SAAS,QAAQ,QAAQ,KAAA;GAChC,MAAM,aACJ,KAAKA,SAAS,OAAO,QAAQ,KAAA,KAC7B,KAAKA,SAAS,YAAY,QAAQ,KAAA;GAKpC,IAAI,EAF6B,aAAa,WAAW,MAExB,EADN,mBAAmB,aAE5C,MAAM,IAAI,MACR,uHACE,KAAK,UAAU,YAAY,IAC3B,+IACJ;EAEJ;EAEA,IAAI;EAIJ,IAAI;GACF,cAAc,MAAM,YAClB,KAAKT,IACF,YAAY,iBAAiB,aAAa,EAC1C,YAAY,YAAY,QAAQ,GAGnC,KAAA,GACA,gBACA,MACA,KAAKU,kBACP;EACF,SAAS,GAAG;GACV,IAAI,gBAAgB,CAAC,GAAG;IACtB,KAAKV,IAAI,OACP,4DACA;KACE;KACA;KACA;KACA;KACA,WAAW,EAAE;IACf,CACF;IACA,UAAU,KAAKA,KAAK,IAAI,EAAE,SAAS;IACnC,GAAG,MAAM,KAAM,EAAE,UAAU,OAAO;IAClC;GACF;GACA,MAAM;EACR;EAEA,MAAM,aAAa,KAAKC,aAAa,WAAW,aAAa;EAC7D,MAAM,qBAAqB,WAAW;EACtC,MAAM,QAAQ,mBAAmB,cAAc;EAQ/C,IACE,MAAM,eAAe,KAAA,KACrB,MAAM,WAAW,OAAO,gBACxB;GACA,MAAM,QAAQ,IAAI,cAAc;IAC9B,MAAM;IACN,SACE;IACF,QAAQ;GACV,CAAC;GACD,UAAU,KAAKD,KAAK,IAAI,MAAM,SAAS;GACvC,GAAG,MAAM,KAAM,MAAM,OAAO;GAC5B;EACF;EAGA,MAAM,WAAW,KAAKI,aAAa,IAAI,QAAQ;EAC/C,IAAI,UAAU;GACZ,KAAKJ,IAAI,QACP,UAAU,SAAS,gDACrB;GACA,SAAS,MAAM,eAAe,OAAO,MAAM;EAC7C;EAEA,mBAAmB,mBACjB;GAAC;GAAU,MAAM,OAAO;EAAI,GAC5B,QACA,WACF;EAEA,MAAM,UAAU,KAAKE,WAAW,WAAW,aAAa;EACxD,MAAM,SAAS,KAAKC,UAAU,WAAW,aAAa;EAEtD,SAAS,IAAI;EACb,QAAQ,IAAI;EAEZ,IAAI;EACJ,IAAI;GACF,aAAa,IAAI,WACf,KAAKH,KACL,QACA,IACA,IAAI,uBACF,KAAKA,KACL,QACA,oBACA,YACA,SACA,MACF,SACM;IACJ,mBAAmB,gBAAgB;KACjC;KACA,MAAM,OAAO;IACf,CAAC;IACD,IAAI,KAAKI,aAAa,IAAI,QAAQ,MAAM,YACtC,KAAKA,aAAa,OAAO,QAAQ;IAInC,SAAS,MAAM;IACf,QAAQ,MAAM;GAChB,CACF;EACF,SAAS,GAAG;GACV,mBAAmB,gBAAgB;IAAC;IAAU,MAAM,OAAO;GAAI,CAAC;GAChE,SAAS,MAAM;GACf,QAAQ,MAAM;GACd,MAAM;EACR;EAEA,KAAKA,aAAa,IAAI,UAAU,UAAU;EAE1C,WAAW,KAAK,KAAK,wBAAwB;EAE7C,IAAI,OAAO,mBAAmB;GAC5B,KAAKJ,IAAI,QACP,oDACA,OAAO,eACP,OAAO,QACT;GACA,MAAM,WAAW,qBACf,KAAK,UAAU,OAAO,iBAAiB,CACzC;EACF;CACF;CAEA,MAAM;EACJ,OAAO,KAAKQ,SAAS;CACvB;;;;;;;CAQA,MAAM,QAAQ;EACZ,MAAM,QAAQ,KAAK,IAAI;EACvB,KAAKR,IAAI,OAAO,YAAY,KAAKC,aAAa,KAAK,cAAc;EAEjE,KAAKI,kBAAkB,YAAY,CAAC;EAEpC,OAAO,KAAKJ,aAAa,MAAM;GAC7B,MAAM,KAAKI,kBAAkB;GAG7B,KAAK,MAAM,MAAM,KAAKJ,aAAa,YAAY,GAAG;IAChD,KAAKD,IAAI,QAAQ,wBAAwB,GAAG,GAAG,UAAU;IAGzD,GAAQ,KAAK;IACb;GACF;EACF;EACA,KAAKA,IAAI,OAAO,sBAAsB,KAAK,IAAI,IAAI,MAAM,KAAK;CAChE;CAEA,OAAO;EACL,KAAKO,KAAK,MAAM;EAChB,KAAKC,SAAS,QAAQ;EACtB,OAAO;CACT;AACF"}
+{"version":3,"file":"syncer.js","names":["#lc","#viewSyncers","#mutagens","#pushers","#connections","#drainCoordinator","#parent","#wss","#stopped","#config","#validateLegacyJWT","#replicaReadyStates","#webSocketOpenConnections","#webSocketConnectionAttempts","#webSocketConnectionSuccesses","#webSocketConnectionFailures","#viewSyncerLag","#recordReplicaReadyState","#createConnection","#computeServingLagDistribution","#viewSyncerLagSampleInterval","#recordViewSyncerLagSamples","#servingLagDistributionCache","#servingLagDistributionCacheClearQueued"],"sources":["../../../../../zero-cache/src/workers/syncer.ts"],"sourcesContent":["import {pid} from 'node:process';\nimport type {MessagePort} from 'node:worker_threads';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {WebSocketServer, type ServerOptions, type WebSocket} from 'ws';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {ErrorKind} from '../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../zero-protocol/src/error-origin.ts';\nimport {\n  isProtocolError,\n  ProtocolError,\n} from '../../../zero-protocol/src/error.ts';\nimport {resolveAuth, type Auth, type ValidateLegacyJWT} from '../auth/auth.ts';\nimport {tokenConfigOptions} from '../auth/jwt.ts';\nimport {type ZeroConfig} from '../config/zero-config.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateGauge,\n  getOrCreateNativeHistogram,\n  getOrCreateUpDownCounter,\n} from '../observability/metrics.ts';\nimport {\n  recordConnectionAttempted,\n  recordConnectionSuccess,\n  setActiveClientGroupsGetter,\n} from '../server/anonymous-otel-start.ts';\nimport type {Mutagen} from '../services/mutagen/mutagen.ts';\nimport type {Pusher} from '../services/mutagen/pusher.ts';\nimport type {ReplicaState} from '../services/replicator/replicator.ts';\nimport {ServiceRunner} from '../services/runner.ts';\nimport type {\n  ActivityBasedService,\n  Service,\n  SingletonService,\n} from '../services/service.ts';\nimport type {ConnectionContextManager} from '../services/view-syncer/connection-context-manager.ts';\nimport {DrainCoordinator} from '../services/view-syncer/drain-coordinator.ts';\nimport type {ViewSyncer} from '../services/view-syncer/view-syncer.ts';\nimport type {Worker} from '../types/processes.ts';\nimport type {Subscription} from '../types/subscription.ts';\nimport {installWebSocketReceiver} from '../types/websocket-handoff.ts';\nimport type {ConnectParams} from './connect-params.ts';\nimport {Connection, sendError} from './connection.ts';\nimport {createNotifierFrom, subscribeTo} from './replicator.ts';\nimport {SyncerWsMessageHandler} from './syncer-ws-message-handler.ts';\n\nexport type SyncerWorkerData = {\n  replicatorPort: MessagePort;\n};\n\nexport type ReplicaReadyState = {\n  readonly watermark: string;\n  readonly replicaReadyTimeMs: number;\n};\n\nexport type ServingLagViewSyncer = Pick<\n  ViewSyncer,\n  'createdAtMs' | 'servedVersion'\n>;\n\nexport type ServingLagStats = {\n  readonly activeClientGroups: number;\n  readonly laggingClientGroups: number;\n  readonly minMs: number;\n  readonly p50Ms: number;\n  readonly p75Ms: number;\n  readonly p99Ms: number;\n  readonly maxMs: number;\n};\n\ntype ServingLagDistribution = ServingLagStats & {\n  readonly lagsMs: readonly number[];\n};\n\nexport const MAX_REPLICA_READY_STATES = 10_000;\nconst VIEW_SYNCER_LAG_SAMPLE_INTERVAL_MS = 60_000;\n\nconst PROTOCOL_VERSION_ATTRIBUTE = 'protocol.version';\nconst FAILURE_REASON_ATTRIBUTE = 'reason';\n\nfunction boundReplicaReadyStates(\n  replicaReadyStates: ReplicaReadyState[],\n): void {\n  if (replicaReadyStates.length > MAX_REPLICA_READY_STATES) {\n    replicaReadyStates.splice(\n      0,\n      replicaReadyStates.length - MAX_REPLICA_READY_STATES,\n    );\n  }\n}\n\nfunction pruneReplicaReadyStates(\n  replicaReadyStates: ReplicaReadyState[],\n  firstNeededIndex: number,\n): void {\n  if (firstNeededIndex > 0) {\n    replicaReadyStates.splice(0, firstNeededIndex);\n  }\n\n  boundReplicaReadyStates(replicaReadyStates);\n}\n\nfunction lowerBoundReplicaReadyTimeMs(\n  replicaReadyStates: readonly ReplicaReadyState[],\n  replicaReadyTimeMs: number,\n): number {\n  let low = 0;\n  let high = replicaReadyStates.length;\n  while (low < high) {\n    const mid = low + Math.floor((high - low) / 2);\n    if (replicaReadyStates[mid].replicaReadyTimeMs < replicaReadyTimeMs) {\n      low = mid + 1;\n    } else {\n      high = mid;\n    }\n  }\n  return low;\n}\n\nfunction upperBoundWatermark(\n  replicaReadyStates: readonly ReplicaReadyState[],\n  watermark: string,\n): number {\n  let low = 0;\n  let high = replicaReadyStates.length;\n  while (low < high) {\n    const mid = low + Math.floor((high - low) / 2);\n    if (replicaReadyStates[mid].watermark <= watermark) {\n      low = mid + 1;\n    } else {\n      high = mid;\n    }\n  }\n  return low;\n}\n\nfunction findFirstUnservedIndex(\n  replicaReadyStates: readonly ReplicaReadyState[],\n  viewSyncer: ServingLagViewSyncer,\n): number {\n  const firstReadyAfterCreation = lowerBoundReplicaReadyTimeMs(\n    replicaReadyStates,\n    viewSyncer.createdAtMs,\n  );\n  const firstAfterServedVersion =\n    viewSyncer.servedVersion === null\n      ? 0\n      : upperBoundWatermark(replicaReadyStates, viewSyncer.servedVersion);\n\n  const firstUnservedIndex = Math.max(\n    firstReadyAfterCreation,\n    firstAfterServedVersion,\n  );\n  return firstUnservedIndex < replicaReadyStates.length\n    ? firstUnservedIndex\n    : -1;\n}\n\nfunction percentileNearestRank(\n  sortedValues: readonly number[],\n  percentile: number,\n): number {\n  if (sortedValues.length === 0) {\n    return 0;\n  }\n  const index = Math.min(\n    sortedValues.length - 1,\n    Math.max(0, Math.ceil((percentile / 100) * sortedValues.length) - 1),\n  );\n  return sortedValues[index];\n}\n\nfunction computeServingLagDistributionMs(\n  now: number,\n  replicaReadyStates: ReplicaReadyState[],\n  viewSyncers: Iterable<ServingLagViewSyncer>,\n): ServingLagDistribution {\n  const lags: number[] = [];\n  let laggingClientGroups = 0;\n  let firstNeededIndex = replicaReadyStates.length;\n\n  for (const viewSyncer of viewSyncers) {\n    const firstUnservedIndex = findFirstUnservedIndex(\n      replicaReadyStates,\n      viewSyncer,\n    );\n    if (firstUnservedIndex === -1) {\n      lags.push(0);\n      continue;\n    }\n\n    firstNeededIndex = Math.min(firstNeededIndex, firstUnservedIndex);\n    const lagMs = Math.max(\n      0,\n      now - replicaReadyStates[firstUnservedIndex].replicaReadyTimeMs,\n    );\n    lags.push(lagMs);\n    if (lagMs > 0) {\n      laggingClientGroups++;\n    }\n  }\n\n  pruneReplicaReadyStates(replicaReadyStates, firstNeededIndex);\n\n  lags.sort((a, b) => a - b);\n  return {\n    activeClientGroups: lags.length,\n    laggingClientGroups,\n    // The percentile fields are for the legacy serving_lag_stats gauge.\n    // The native view_syncer_lag histogram records lagsMs and computes\n    // percentiles in Prometheus/AMP.\n    minMs: lags[0] ?? 0,\n    p50Ms: percentileNearestRank(lags, 50),\n    p75Ms: percentileNearestRank(lags, 75),\n    p99Ms: percentileNearestRank(lags, 99),\n    maxMs: lags.at(-1) ?? 0,\n    lagsMs: lags,\n  };\n}\n\nexport function computeServingLagStatsMs(\n  now: number,\n  replicaReadyStates: ReplicaReadyState[],\n  viewSyncers: Iterable<ServingLagViewSyncer>,\n): ServingLagStats {\n  const distribution = computeServingLagDistributionMs(\n    now,\n    replicaReadyStates,\n    viewSyncers,\n  );\n  return {\n    activeClientGroups: distribution.activeClientGroups,\n    laggingClientGroups: distribution.laggingClientGroups,\n    minMs: distribution.minMs,\n    p50Ms: distribution.p50Ms,\n    p75Ms: distribution.p75Ms,\n    p99Ms: distribution.p99Ms,\n    maxMs: distribution.maxMs,\n  };\n}\n\nexport function computeMaxServingLagMs(\n  now: number,\n  replicaReadyStates: ReplicaReadyState[],\n  viewSyncers: Iterable<ServingLagViewSyncer>,\n): number {\n  return computeServingLagStatsMs(now, replicaReadyStates, viewSyncers).maxMs;\n}\n\nfunction getWebSocketServerOptions(config: ZeroConfig): ServerOptions {\n  const options: ServerOptions = {\n    noServer: true,\n    maxPayload: config.websocketMaxPayloadBytes,\n  };\n\n  if (config.websocketCompression) {\n    options.perMessageDeflate = true;\n\n    if (config.websocketCompressionOptions) {\n      try {\n        const compressionOptions = JSON.parse(\n          config.websocketCompressionOptions,\n        );\n        options.perMessageDeflate = compressionOptions;\n      } catch (e) {\n        throw new Error(\n          `Failed to parse ZERO_WEBSOCKET_COMPRESSION_OPTIONS: ${String(e)}. Expected valid JSON.`,\n        );\n      }\n    }\n  }\n\n  return options;\n}\n\n/**\n * The Syncer worker receives websocket handoffs for \"/sync\" connections\n * from the Dispatcher in the main thread, and creates websocket\n * {@link Connection}s with a corresponding {@link ViewSyncer}, {@link Mutagen},\n * and {@link Subscription} to version notifications from the Replicator\n * worker.\n */\nexport class Syncer implements SingletonService {\n  readonly id = `syncer-${pid}`;\n  readonly #lc: LogContext;\n  readonly #viewSyncers: ServiceRunner<ViewSyncer & ActivityBasedService>;\n  readonly #mutagens: ServiceRunner<Mutagen & Service> | undefined;\n  readonly #pushers: ServiceRunner<Pusher & Service> | undefined;\n  readonly #connections = new Map<string, Connection>();\n  readonly #drainCoordinator = new DrainCoordinator();\n  readonly #parent: Worker;\n  readonly #wss: WebSocketServer;\n  readonly #stopped = resolver();\n  readonly #config: ZeroConfig;\n  readonly #validateLegacyJWT: ValidateLegacyJWT | undefined;\n  readonly #replicaReadyStates: ReplicaReadyState[] = [];\n  readonly #webSocketOpenConnections = getOrCreateUpDownCounter(\n    'sync',\n    'websocket.open_connections',\n    'Open client WebSocket connections.',\n  );\n  readonly #webSocketConnectionAttempts = getOrCreateCounter(\n    'sync',\n    'websocket.connection_attempts',\n    'Client WebSocket connection attempts.',\n  );\n  readonly #webSocketConnectionSuccesses = getOrCreateCounter(\n    'sync',\n    'websocket.connection_successes',\n    'Client WebSocket connections successfully initialized.',\n  );\n  readonly #webSocketConnectionFailures = getOrCreateCounter(\n    'sync',\n    'websocket.connection_failures',\n    'Client WebSocket connection attempts that failed before initialization.',\n  );\n  readonly #viewSyncerLag = getOrCreateNativeHistogram(\n    'sync',\n    'view_syncer_lag',\n    {\n      description:\n        'Lag from replica-ready change to ViewSyncer output for active ' +\n        'client groups. A change is output after IVM advancement, CVR flush, ' +\n        'and pokeEnd.',\n      unit: 's',\n    },\n  );\n  #servingLagDistributionCache: ServingLagDistribution | undefined;\n  #servingLagDistributionCacheClearQueued = false;\n  #viewSyncerLagSampleInterval: ReturnType<typeof setInterval> | undefined;\n\n  constructor(\n    lc: LogContext,\n    config: ZeroConfig,\n    viewSyncerFactory: (\n      id: string,\n      sub: Subscription<ReplicaState>,\n      drainCoordinator: DrainCoordinator,\n    ) => ViewSyncer & ActivityBasedService,\n    mutagenFactory: ((id: string) => Mutagen & Service) | undefined,\n    pusherFactory:\n      | ((\n          id: string,\n          connContextManager: ConnectionContextManager,\n        ) => Pusher & Service)\n      | undefined,\n    parent: Worker,\n    validateLegacyJWT: ValidateLegacyJWT | undefined,\n  ) {\n    this.#config = config;\n    this.#validateLegacyJWT = validateLegacyJWT;\n    // Relays notifications from the parent thread subscription\n    // to ViewSyncers within this thread.\n    const notifier = createNotifierFrom(lc, parent, state =>\n      this.#recordReplicaReadyState(state),\n    );\n    subscribeTo(lc, parent);\n\n    this.#lc = lc;\n    this.#viewSyncers = new ServiceRunner(\n      lc,\n      id => viewSyncerFactory(id, notifier.subscribe(), this.#drainCoordinator),\n      v => v.keepalive(),\n    );\n    if (mutagenFactory) {\n      this.#mutagens = new ServiceRunner(lc, mutagenFactory, m => m.hasRefs());\n    }\n    if (pusherFactory) {\n      this.#pushers = new ServiceRunner(\n        lc,\n        id =>\n          pusherFactory(\n            id,\n            this.#viewSyncers.getService(id).connContextManager,\n          ),\n        p => p.hasRefs(),\n      );\n    }\n    this.#parent = parent;\n    this.#wss = new WebSocketServer(getWebSocketServerOptions(config));\n\n    installWebSocketReceiver(\n      lc,\n      this.#wss,\n      this.#createConnection,\n      this.#parent,\n    );\n\n    setActiveClientGroupsGetter(() => this.#viewSyncers.size);\n\n    getOrCreateGauge(\n      'sync',\n      'active-client-groups',\n      'Number of active client groups',\n    ).addCallback(result => result.observe(this.#viewSyncers.size));\n\n    getOrCreateGauge(\n      'sync',\n      'queries',\n      'Active queries (pipelines) across all client groups',\n    ).addCallback(result => {\n      let total = 0;\n      for (const vs of this.#viewSyncers.getServices()) {\n        total += vs.queryCount;\n      }\n      result.observe(total);\n    });\n\n    getOrCreateGauge(\n      'sync',\n      'rows',\n      'Tracked rows across all client groups',\n    ).addCallback(result => {\n      let total = 0;\n      for (const vs of this.#viewSyncers.getServices()) {\n        total += vs.rowCount;\n      }\n      result.observe(total);\n    });\n\n    getOrCreateGauge('sync', 'serving_lag', {\n      description:\n        'Maximum time active ViewSyncer client groups have had unserved ' +\n        'replica changes. A change is served after IVM advancement, CVR flush, ' +\n        'and pokeEnd.',\n      unit: 'millisecond',\n    }).addCallback(result => {\n      result.observe(this.#computeServingLagDistribution().maxMs);\n    });\n\n    getOrCreateGauge('sync', 'serving_lag_stats', {\n      description:\n        'Distribution of time active ViewSyncer client groups have had ' +\n        'unserved replica changes. A change is served after IVM advancement, ' +\n        'CVR flush, and pokeEnd.',\n      unit: 'millisecond',\n    }).addCallback(result => {\n      const stats = this.#computeServingLagDistribution();\n      result.observe(stats.minMs, {stat: 'min'});\n      result.observe(stats.p50Ms, {stat: 'p50'});\n      result.observe(stats.p75Ms, {stat: 'p75'});\n      result.observe(stats.p99Ms, {stat: 'p99'});\n      result.observe(stats.maxMs, {stat: 'max'});\n    });\n\n    getOrCreateGauge(\n      'sync',\n      'serving_lagging_client_groups',\n      'Number of active client groups with unserved replica changes',\n    ).addCallback(result => {\n      result.observe(this.#computeServingLagDistribution().laggingClientGroups);\n    });\n\n    this.#viewSyncerLagSampleInterval = setInterval(\n      () => this.#recordViewSyncerLagSamples(),\n      VIEW_SYNCER_LAG_SAMPLE_INTERVAL_MS,\n    );\n    this.#viewSyncerLagSampleInterval.unref?.();\n  }\n\n  #computeServingLagDistribution(): ServingLagDistribution {\n    if (this.#servingLagDistributionCache) {\n      return this.#servingLagDistributionCache;\n    }\n\n    const distribution = computeServingLagDistributionMs(\n      Date.now(),\n      this.#replicaReadyStates,\n      this.#viewSyncers.getServices(),\n    );\n    this.#servingLagDistributionCache = distribution;\n    if (!this.#servingLagDistributionCacheClearQueued) {\n      this.#servingLagDistributionCacheClearQueued = true;\n      queueMicrotask(() => {\n        this.#servingLagDistributionCache = undefined;\n        this.#servingLagDistributionCacheClearQueued = false;\n      });\n    }\n    return distribution;\n  }\n\n  #recordViewSyncerLagSamples(): void {\n    const {lagsMs} = this.#computeServingLagDistribution();\n    for (const lagMs of lagsMs) {\n      this.#viewSyncerLag.recordMs(lagMs);\n    }\n  }\n\n  #recordReplicaReadyState(state: ReplicaState): void {\n    if (\n      state.watermark === undefined ||\n      state.replicaReadyTimeMs === undefined\n    ) {\n      return;\n    }\n    const last = this.#replicaReadyStates.at(-1);\n    if (last && last.watermark >= state.watermark) {\n      return;\n    }\n    this.#replicaReadyStates.push({\n      watermark: state.watermark,\n      replicaReadyTimeMs: state.replicaReadyTimeMs,\n    });\n    if (this.#viewSyncers.size === 0) {\n      this.#replicaReadyStates.length = 0;\n      return;\n    }\n    boundReplicaReadyStates(this.#replicaReadyStates);\n  }\n\n  readonly #createConnection = async (ws: WebSocket, params: ConnectParams) => {\n    const metricAttributes = {\n      [PROTOCOL_VERSION_ATTRIBUTE]: params.protocolVersion,\n    };\n    let connectionResultRecorded = false;\n    const recordConnectionFailure = (reason: string) => {\n      if (connectionResultRecorded) {\n        return;\n      }\n      connectionResultRecorded = true;\n      this.#webSocketConnectionFailures.add(1, {\n        ...metricAttributes,\n        [FAILURE_REASON_ATTRIBUTE]: reason,\n      });\n    };\n    const recordConnectionSuccessMetric = () => {\n      if (connectionResultRecorded) {\n        return;\n      }\n      connectionResultRecorded = true;\n      this.#webSocketConnectionSuccesses.add(1, metricAttributes);\n    };\n\n    this.#webSocketConnectionAttempts.add(1, metricAttributes);\n    this.#webSocketOpenConnections.add(1, metricAttributes);\n    ws.once('close', () => {\n      this.#webSocketOpenConnections.add(-1, metricAttributes);\n    });\n\n    this.#lc.debug?.(\n      'creating connection',\n      params.clientGroupID,\n      params.clientID,\n    );\n    recordConnectionAttempted();\n    const {clientID, clientGroupID, auth, userID} = params;\n    const hasProvidedAuth = auth !== undefined && auth !== '';\n    const incomingUserID = userID ?? null;\n\n    if (hasProvidedAuth) {\n      const tokenOptions = tokenConfigOptions(this.#config.auth ?? {});\n\n      const hasPushOrMutate =\n        this.#config?.push?.url !== undefined ||\n        this.#config?.mutate?.url !== undefined;\n      const hasQueries =\n        this.#config?.query?.url !== undefined ||\n        this.#config?.getQueries?.url !== undefined;\n\n      // must either have one of the token options set or have custom mutations & queries enabled\n      const hasExactlyOneTokenOption = tokenOptions.length === 1;\n      const hasCustomEndpoints = hasPushOrMutate && hasQueries;\n      if (!hasExactlyOneTokenOption && !hasCustomEndpoints) {\n        recordConnectionFailure('configuration');\n        throw new Error(\n          'Exactly one of jwk, secret, or jwksUrl must be set in order to verify tokens but actually the following were set: ' +\n            JSON.stringify(tokenOptions) +\n            '. You may also set both ZERO_MUTATE_URL and ZERO_QUERY_URL to enable custom mutations and queries without passing token verification options.',\n        );\n      }\n    }\n\n    let initialAuth: Auth | undefined;\n\n    // Verify JWT BEFORE touching existing connections - prevents unauthenticated\n    // attackers from force-disconnecting legitimate users via DoS.\n    try {\n      initialAuth = await resolveAuth(\n        this.#lc\n          .withContext('clientGroupID', clientGroupID)\n          .withContext('clientID', clientID),\n        // no previous auth, since this is a new connection, and resolveAuth is\n        // connection scoped, not client group scoped\n        undefined,\n        incomingUserID,\n        auth,\n        this.#validateLegacyJWT,\n      );\n    } catch (e) {\n      if (isProtocolError(e)) {\n        this.#lc.warn?.(\n          'Rejecting sync connection during initial auth resolution',\n          {\n            clientGroupID,\n            clientID,\n            incomingUserID,\n            hasProvidedAuth,\n            errorKind: e.message,\n          },\n        );\n        recordConnectionFailure('auth');\n        sendError(this.#lc, ws, e.errorBody);\n        ws.close(3000, e.errorBody.message);\n        return;\n      }\n      recordConnectionFailure('internal');\n      throw e;\n    }\n\n    const viewSyncer = this.#viewSyncers.getService(clientGroupID);\n    const connContextManager = viewSyncer.connContextManager;\n    const group = connContextManager.getGroupState();\n\n    // TODO(0xcadams): we only check for user ID mismatch here if the group is\n    // already validated. This prevents wrong-user reconnects from evicting a\n    // healthy connection, but it does not protect against same-user reconnects\n    // with an invalid opaque token. The long-term fix is to keep the replacement\n    // connection pending until its auth is fully validated, and only then replace\n    // the existing socket.\n    if (\n      group.pinnedUser !== undefined &&\n      group.pinnedUser.id !== incomingUserID\n    ) {\n      const error = new ProtocolError({\n        kind: ErrorKind.Unauthorized,\n        message:\n          'Client groups are pinned to a single userID. Connection userID does not match existing client group userID.',\n        origin: ErrorOrigin.ZeroCache,\n      });\n      recordConnectionFailure('user_mismatch');\n      sendError(this.#lc, ws, error.errorBody);\n      ws.close(3000, error.message);\n      return;\n    }\n\n    // Check for and close existing connections AFTER auth is validated\n    const existing = this.#connections.get(clientID);\n    if (existing) {\n      this.#lc.debug?.(\n        `client ${clientID} already connected, closing existing connection`,\n      );\n      existing.close(`replaced by ${params.wsID}`);\n    }\n\n    connContextManager.registerConnection(\n      {clientID, wsID: params.wsID},\n      params,\n      initialAuth,\n    );\n\n    const mutagen = this.#mutagens?.getService(clientGroupID);\n    const pusher = this.#pushers?.getService(clientGroupID);\n    // a new connection is using the mutagen and pusher. Bump their ref counts.\n    mutagen?.ref();\n    pusher?.ref();\n\n    let connection: Connection;\n    try {\n      connection = new Connection(\n        this.#lc,\n        params,\n        ws,\n        new SyncerWsMessageHandler(\n          this.#lc,\n          params,\n          connContextManager,\n          viewSyncer,\n          mutagen,\n          pusher,\n        ),\n        () => {\n          connContextManager.closeConnection({\n            clientID,\n            wsID: params.wsID,\n          });\n          if (this.#connections.get(clientID) === connection) {\n            this.#connections.delete(clientID);\n          }\n          // Connection is closed. We can unref the mutagen and pusher.\n          // If their ref counts are zero, they will stop themselves and set themselves invalid.\n          mutagen?.unref();\n          pusher?.unref();\n        },\n      );\n    } catch (e) {\n      recordConnectionFailure('internal');\n      connContextManager.closeConnection({clientID, wsID: params.wsID});\n      mutagen?.unref();\n      pusher?.unref();\n      throw e;\n    }\n\n    this.#connections.set(clientID, connection);\n\n    if (connection.init()) {\n      recordConnectionSuccessMetric();\n      recordConnectionSuccess();\n    } else {\n      recordConnectionFailure('protocol_version');\n    }\n\n    if (params.initConnectionMsg) {\n      this.#lc.debug?.(\n        'handling init connection message from sec header',\n        params.clientGroupID,\n        params.clientID,\n      );\n      await connection.handleInitConnection(\n        JSON.stringify(params.initConnectionMsg),\n      );\n    }\n  };\n\n  run() {\n    return this.#stopped.promise;\n  }\n\n  /**\n   * Graceful shutdown involves shutting down view syncers one at a time, pausing\n   * for the duration of view syncer's hydration between each one. This paces the\n   * disconnects to avoid creating a backlog of hydrations in the receiving server\n   * when the clients reconnect.\n   */\n  async drain() {\n    const start = Date.now();\n    this.#lc.info?.(`draining ${this.#viewSyncers.size} view-syncers`);\n\n    this.#drainCoordinator.drainNextIn(0);\n\n    while (this.#viewSyncers.size) {\n      await this.#drainCoordinator.forceDrainTimeout;\n\n      // Pick an arbitrary view syncer to force drain.\n      for (const vs of this.#viewSyncers.getServices()) {\n        this.#lc.debug?.(`draining view-syncer ${vs.id} (forced)`);\n        // When this drain or an elective drain completes, the forceDrainTimeout will\n        // resolve after the next drain interval.\n        void vs.stop();\n        break;\n      }\n    }\n    this.#lc.info?.(`finished draining (${Date.now() - start} ms)`);\n  }\n\n  stop() {\n    clearInterval(this.#viewSyncerLagSampleInterval);\n    this.#wss.close();\n    this.#stopped.resolve();\n    return promiseVoid;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;AA0EA,IAAa,2BAA2B;AACxC,IAAM,qCAAqC;AAE3C,IAAM,6BAA6B;AACnC,IAAM,2BAA2B;AAEjC,SAAS,wBACP,oBACM;CACN,IAAI,mBAAmB,SAAA,KACrB,mBAAmB,OACjB,GACA,mBAAmB,SAAS,wBAC9B;AAEJ;AAEA,SAAS,wBACP,oBACA,kBACM;CACN,IAAI,mBAAmB,GACrB,mBAAmB,OAAO,GAAG,gBAAgB;CAG/C,wBAAwB,kBAAkB;AAC5C;AAEA,SAAS,6BACP,oBACA,oBACQ;CACR,IAAI,MAAM;CACV,IAAI,OAAO,mBAAmB;CAC9B,OAAO,MAAM,MAAM;EACjB,MAAM,MAAM,MAAM,KAAK,OAAO,OAAO,OAAO,CAAC;EAC7C,IAAI,mBAAmB,KAAK,qBAAqB,oBAC/C,MAAM,MAAM;OAEZ,OAAO;CAEX;CACA,OAAO;AACT;AAEA,SAAS,oBACP,oBACA,WACQ;CACR,IAAI,MAAM;CACV,IAAI,OAAO,mBAAmB;CAC9B,OAAO,MAAM,MAAM;EACjB,MAAM,MAAM,MAAM,KAAK,OAAO,OAAO,OAAO,CAAC;EAC7C,IAAI,mBAAmB,KAAK,aAAa,WACvC,MAAM,MAAM;OAEZ,OAAO;CAEX;CACA,OAAO;AACT;AAEA,SAAS,uBACP,oBACA,YACQ;CACR,MAAM,0BAA0B,6BAC9B,oBACA,WAAW,WACb;CACA,MAAM,0BACJ,WAAW,kBAAkB,OACzB,IACA,oBAAoB,oBAAoB,WAAW,aAAa;CAEtE,MAAM,qBAAqB,KAAK,IAC9B,yBACA,uBACF;CACA,OAAO,qBAAqB,mBAAmB,SAC3C,qBACA;AACN;AAEA,SAAS,sBACP,cACA,YACQ;CACR,IAAI,aAAa,WAAW,GAC1B,OAAO;CAMT,OAAO,aAJO,KAAK,IACjB,aAAa,SAAS,GACtB,KAAK,IAAI,GAAG,KAAK,KAAM,aAAa,MAAO,aAAa,MAAM,IAAI,CAAC,CAEjD;AACtB;AAEA,SAAS,gCACP,KACA,oBACA,aACwB;CACxB,MAAM,OAAiB,CAAC;CACxB,IAAI,sBAAsB;CAC1B,IAAI,mBAAmB,mBAAmB;CAE1C,KAAK,MAAM,cAAc,aAAa;EACpC,MAAM,qBAAqB,uBACzB,oBACA,UACF;EACA,IAAI,uBAAuB,IAAI;GAC7B,KAAK,KAAK,CAAC;GACX;EACF;EAEA,mBAAmB,KAAK,IAAI,kBAAkB,kBAAkB;EAChE,MAAM,QAAQ,KAAK,IACjB,GACA,MAAM,mBAAmB,oBAAoB,kBAC/C;EACA,KAAK,KAAK,KAAK;EACf,IAAI,QAAQ,GACV;CAEJ;CAEA,wBAAwB,oBAAoB,gBAAgB;CAE5D,KAAK,MAAM,GAAG,MAAM,IAAI,CAAC;CACzB,OAAO;EACL,oBAAoB,KAAK;EACzB;EAIA,OAAO,KAAK,MAAM;EAClB,OAAO,sBAAsB,MAAM,EAAE;EACrC,OAAO,sBAAsB,MAAM,EAAE;EACrC,OAAO,sBAAsB,MAAM,EAAE;EACrC,OAAO,KAAK,GAAG,EAAE,KAAK;EACtB,QAAQ;CACV;AACF;AA+BA,SAAS,0BAA0B,QAAmC;CACpE,MAAM,UAAyB;EAC7B,UAAU;EACV,YAAY,OAAO;CACrB;CAEA,IAAI,OAAO,sBAAsB;EAC/B,QAAQ,oBAAoB;EAE5B,IAAI,OAAO,6BACT,IAAI;GAIF,QAAQ,oBAHmB,KAAK,MAC9B,OAAO,2BAEmB;EAC9B,SAAS,GAAG;GACV,MAAM,IAAI,MACR,uDAAuD,OAAO,CAAC,EAAE,uBACnE;EACF;CAEJ;CAEA,OAAO;AACT;;;;;;;;AASA,IAAa,SAAb,MAAgD;CAC9C,KAAc,UAAU;CACxB;CACA;CACA;CACA;CACA,+BAAwB,IAAI,IAAwB;CACpD,oBAA6B,IAAI,iBAAiB;CAClD;CACA;CACA,WAAoB,SAAS;CAC7B;CACA;CACA,sBAAoD,CAAC;CACrD,4BAAqC,yBACnC,QACA,8BACA,oCACF;CACA,+BAAwC,mBACtC,QACA,iCACA,uCACF;CACA,gCAAyC,mBACvC,QACA,kCACA,wDACF;CACA,+BAAwC,mBACtC,QACA,iCACA,yEACF;CACA,iBAA0B,2BACxB,QACA,mBACA;EACE,aACE;EAGF,MAAM;CACR,CACF;CACA;CACA,0CAA0C;CAC1C;CAEA,YACE,IACA,QACA,mBAKA,gBACA,eAMA,QACA,mBACA;EACA,KAAKS,UAAU;EACf,KAAKC,qBAAqB;EAG1B,MAAM,WAAW,mBAAmB,IAAI,SAAQ,UAC9C,KAAKO,yBAAyB,KAAK,CACrC;EACA,YAAY,IAAI,MAAM;EAEtB,KAAKjB,MAAM;EACX,KAAKC,eAAe,IAAI,cACtB,KACA,OAAM,kBAAkB,IAAI,SAAS,UAAU,GAAG,KAAKI,iBAAiB,IACxE,MAAK,EAAE,UAAU,CACnB;EACA,IAAI,gBACF,KAAKH,YAAY,IAAI,cAAc,IAAI,iBAAgB,MAAK,EAAE,QAAQ,CAAC;EAEzE,IAAI,eACF,KAAKC,WAAW,IAAI,cAClB,KACA,OACE,cACE,IACA,KAAKF,aAAa,WAAW,EAAE,EAAE,kBACnC,IACF,MAAK,EAAE,QAAQ,CACjB;EAEF,KAAKK,UAAU;EACf,KAAKC,OAAO,IAAI,gBAAgB,0BAA0B,MAAM,CAAC;EAEjE,yBACE,IACA,KAAKA,MACL,KAAKW,mBACL,KAAKZ,OACP;EAEA,kCAAkC,KAAKL,aAAa,IAAI;EAExD,iBACE,QACA,wBACA,gCACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKA,aAAa,IAAI,CAAC;EAE9D,iBACE,QACA,WACA,qDACF,EAAE,aAAY,WAAU;GACtB,IAAI,QAAQ;GACZ,KAAK,MAAM,MAAM,KAAKA,aAAa,YAAY,GAC7C,SAAS,GAAG;GAEd,OAAO,QAAQ,KAAK;EACtB,CAAC;EAED,iBACE,QACA,QACA,uCACF,EAAE,aAAY,WAAU;GACtB,IAAI,QAAQ;GACZ,KAAK,MAAM,MAAM,KAAKA,aAAa,YAAY,GAC7C,SAAS,GAAG;GAEd,OAAO,QAAQ,KAAK;EACtB,CAAC;EAED,iBAAiB,QAAQ,eAAe;GACtC,aACE;GAGF,MAAM;EACR,CAAC,EAAE,aAAY,WAAU;GACvB,OAAO,QAAQ,KAAKkB,+BAA+B,EAAE,KAAK;EAC5D,CAAC;EAED,iBAAiB,QAAQ,qBAAqB;GAC5C,aACE;GAGF,MAAM;EACR,CAAC,EAAE,aAAY,WAAU;GACvB,MAAM,QAAQ,KAAKA,+BAA+B;GAClD,OAAO,QAAQ,MAAM,OAAO,EAAC,MAAM,MAAK,CAAC;GACzC,OAAO,QAAQ,MAAM,OAAO,EAAC,MAAM,MAAK,CAAC;GACzC,OAAO,QAAQ,MAAM,OAAO,EAAC,MAAM,MAAK,CAAC;GACzC,OAAO,QAAQ,MAAM,OAAO,EAAC,MAAM,MAAK,CAAC;GACzC,OAAO,QAAQ,MAAM,OAAO,EAAC,MAAM,MAAK,CAAC;EAC3C,CAAC;EAED,iBACE,QACA,iCACA,8DACF,EAAE,aAAY,WAAU;GACtB,OAAO,QAAQ,KAAKA,+BAA+B,EAAE,mBAAmB;EAC1E,CAAC;EAED,KAAKC,+BAA+B,kBAC5B,KAAKC,4BAA4B,GACvC,kCACF;EACA,KAAKD,6BAA6B,QAAQ;CAC5C;CAEA,iCAAyD;EACvD,IAAI,KAAKE,8BACP,OAAO,KAAKA;EAGd,MAAM,eAAe,gCACnB,KAAK,IAAI,GACT,KAAKX,qBACL,KAAKV,aAAa,YAAY,CAChC;EACA,KAAKqB,+BAA+B;EACpC,IAAI,CAAC,KAAKC,yCAAyC;GACjD,KAAKA,0CAA0C;GAC/C,qBAAqB;IACnB,KAAKD,+BAA+B,KAAA;IACpC,KAAKC,0CAA0C;GACjD,CAAC;EACH;EACA,OAAO;CACT;CAEA,8BAAoC;EAClC,MAAM,EAAC,WAAU,KAAKJ,+BAA+B;EACrD,KAAK,MAAM,SAAS,QAClB,KAAKH,eAAe,SAAS,KAAK;CAEtC;CAEA,yBAAyB,OAA2B;EAClD,IACE,MAAM,cAAc,KAAA,KACpB,MAAM,uBAAuB,KAAA,GAE7B;EAEF,MAAM,OAAO,KAAKL,oBAAoB,GAAG,EAAE;EAC3C,IAAI,QAAQ,KAAK,aAAa,MAAM,WAClC;EAEF,KAAKA,oBAAoB,KAAK;GAC5B,WAAW,MAAM;GACjB,oBAAoB,MAAM;EAC5B,CAAC;EACD,IAAI,KAAKV,aAAa,SAAS,GAAG;GAChC,KAAKU,oBAAoB,SAAS;GAClC;EACF;EACA,wBAAwB,KAAKA,mBAAmB;CAClD;CAEA,oBAA6B,OAAO,IAAe,WAA0B;EAC3E,MAAM,mBAAmB,GACtB,6BAA6B,OAAO,gBACvC;EACA,IAAI,2BAA2B;EAC/B,MAAM,2BAA2B,WAAmB;GAClD,IAAI,0BACF;GAEF,2BAA2B;GAC3B,KAAKI,6BAA6B,IAAI,GAAG;IACvC,GAAG;KACF,2BAA2B;GAC9B,CAAC;EACH;EACA,MAAM,sCAAsC;GAC1C,IAAI,0BACF;GAEF,2BAA2B;GAC3B,KAAKD,8BAA8B,IAAI,GAAG,gBAAgB;EAC5D;EAEA,KAAKD,6BAA6B,IAAI,GAAG,gBAAgB;EACzD,KAAKD,0BAA0B,IAAI,GAAG,gBAAgB;EACtD,GAAG,KAAK,eAAe;GACrB,KAAKA,0BAA0B,IAAI,IAAI,gBAAgB;EACzD,CAAC;EAED,KAAKZ,IAAI,QACP,uBACA,OAAO,eACP,OAAO,QACT;EACA,0BAA0B;EAC1B,MAAM,EAAC,UAAU,eAAe,MAAM,WAAU;EAChD,MAAM,kBAAkB,SAAS,KAAA,KAAa,SAAS;EACvD,MAAM,iBAAiB,UAAU;EAEjC,IAAI,iBAAiB;GACnB,MAAM,eAAe,mBAAmB,KAAKS,QAAQ,QAAQ,CAAC,CAAC;GAE/D,MAAM,kBACJ,KAAKA,SAAS,MAAM,QAAQ,KAAA,KAC5B,KAAKA,SAAS,QAAQ,QAAQ,KAAA;GAChC,MAAM,aACJ,KAAKA,SAAS,OAAO,QAAQ,KAAA,KAC7B,KAAKA,SAAS,YAAY,QAAQ,KAAA;GAKpC,IAAI,EAF6B,aAAa,WAAW,MAExB,EADN,mBAAmB,aACQ;IACpD,wBAAwB,eAAe;IACvC,MAAM,IAAI,MACR,uHACE,KAAK,UAAU,YAAY,IAC3B,+IACJ;GACF;EACF;EAEA,IAAI;EAIJ,IAAI;GACF,cAAc,MAAM,YAClB,KAAKT,IACF,YAAY,iBAAiB,aAAa,EAC1C,YAAY,YAAY,QAAQ,GAGnC,KAAA,GACA,gBACA,MACA,KAAKU,kBACP;EACF,SAAS,GAAG;GACV,IAAI,gBAAgB,CAAC,GAAG;IACtB,KAAKV,IAAI,OACP,4DACA;KACE;KACA;KACA;KACA;KACA,WAAW,EAAE;IACf,CACF;IACA,wBAAwB,MAAM;IAC9B,UAAU,KAAKA,KAAK,IAAI,EAAE,SAAS;IACnC,GAAG,MAAM,KAAM,EAAE,UAAU,OAAO;IAClC;GACF;GACA,wBAAwB,UAAU;GAClC,MAAM;EACR;EAEA,MAAM,aAAa,KAAKC,aAAa,WAAW,aAAa;EAC7D,MAAM,qBAAqB,WAAW;EACtC,MAAM,QAAQ,mBAAmB,cAAc;EAQ/C,IACE,MAAM,eAAe,KAAA,KACrB,MAAM,WAAW,OAAO,gBACxB;GACA,MAAM,QAAQ,IAAI,cAAc;IAC9B,MAAM;IACN,SACE;IACF,QAAQ;GACV,CAAC;GACD,wBAAwB,eAAe;GACvC,UAAU,KAAKD,KAAK,IAAI,MAAM,SAAS;GACvC,GAAG,MAAM,KAAM,MAAM,OAAO;GAC5B;EACF;EAGA,MAAM,WAAW,KAAKI,aAAa,IAAI,QAAQ;EAC/C,IAAI,UAAU;GACZ,KAAKJ,IAAI,QACP,UAAU,SAAS,gDACrB;GACA,SAAS,MAAM,eAAe,OAAO,MAAM;EAC7C;EAEA,mBAAmB,mBACjB;GAAC;GAAU,MAAM,OAAO;EAAI,GAC5B,QACA,WACF;EAEA,MAAM,UAAU,KAAKE,WAAW,WAAW,aAAa;EACxD,MAAM,SAAS,KAAKC,UAAU,WAAW,aAAa;EAEtD,SAAS,IAAI;EACb,QAAQ,IAAI;EAEZ,IAAI;EACJ,IAAI;GACF,aAAa,IAAI,WACf,KAAKH,KACL,QACA,IACA,IAAI,uBACF,KAAKA,KACL,QACA,oBACA,YACA,SACA,MACF,SACM;IACJ,mBAAmB,gBAAgB;KACjC;KACA,MAAM,OAAO;IACf,CAAC;IACD,IAAI,KAAKI,aAAa,IAAI,QAAQ,MAAM,YACtC,KAAKA,aAAa,OAAO,QAAQ;IAInC,SAAS,MAAM;IACf,QAAQ,MAAM;GAChB,CACF;EACF,SAAS,GAAG;GACV,wBAAwB,UAAU;GAClC,mBAAmB,gBAAgB;IAAC;IAAU,MAAM,OAAO;GAAI,CAAC;GAChE,SAAS,MAAM;GACf,QAAQ,MAAM;GACd,MAAM;EACR;EAEA,KAAKA,aAAa,IAAI,UAAU,UAAU;EAE1C,IAAI,WAAW,KAAK,GAAG;GACrB,8BAA8B;GAC9B,wBAAwB;EAC1B,OACE,wBAAwB,kBAAkB;EAG5C,IAAI,OAAO,mBAAmB;GAC5B,KAAKJ,IAAI,QACP,oDACA,OAAO,eACP,OAAO,QACT;GACA,MAAM,WAAW,qBACf,KAAK,UAAU,OAAO,iBAAiB,CACzC;EACF;CACF;CAEA,MAAM;EACJ,OAAO,KAAKQ,SAAS;CACvB;;;;;;;CAQA,MAAM,QAAQ;EACZ,MAAM,QAAQ,KAAK,IAAI;EACvB,KAAKR,IAAI,OAAO,YAAY,KAAKC,aAAa,KAAK,cAAc;EAEjE,KAAKI,kBAAkB,YAAY,CAAC;EAEpC,OAAO,KAAKJ,aAAa,MAAM;GAC7B,MAAM,KAAKI,kBAAkB;GAG7B,KAAK,MAAM,MAAM,KAAKJ,aAAa,YAAY,GAAG;IAChD,KAAKD,IAAI,QAAQ,wBAAwB,GAAG,GAAG,UAAU;IAGzD,GAAQ,KAAK;IACb;GACF;EACF;EACA,KAAKA,IAAI,OAAO,sBAAsB,KAAK,IAAI,IAAI,MAAM,KAAK;CAChE;CAEA,OAAO;EACL,cAAc,KAAKoB,4BAA4B;EAC/C,KAAKb,KAAK,MAAM;EAChB,KAAKC,SAAS,QAAQ;EACtB,OAAO;CACT;AACF"}

package/out/zero-client/src/client/mutation-tracker.js

@@ -1,8 +1,8 @@
 import { assert, unreachable } from "../../../shared/src/asserts.js";
 import { parse } from "../../../shared/src/valita.js";
+import { must } from "../../../shared/src/must.js";
 import { getErrorDetails } from "../../../shared/src/error.js";
 import { ApplicationError, isApplicationError, wrapWithApplicationError } from "../../../zero-protocol/src/application-error.js";
-import { must } from "../../../shared/src/must.js";
 import { InvalidPush, PushFailed } from "../../../zero-protocol/src/error-kind-enum.js";
 import { Server, ZeroCache } from "../../../zero-protocol/src/error-origin-enum.js";
 import { HTTP, Internal } from "../../../zero-protocol/src/error-reason-enum.js";

package/out/zero-client/src/client/query-manager.js

@@ -1,12 +1,12 @@
 import { assert, unreachable } from "../../../shared/src/asserts.js";
 import { must } from "../../../shared/src/must.js";
 import { mapAST, normalizeAST } from "../../../zero-protocol/src/ast.js";
+import { clientToServer, serverToClient } from "../../../zero-schema/src/name-mapper.js";
+import { hashOfAST, hashOfNameAndArgs } from "../../../zero-protocol/src/query-hash.js";
 import { TransformFailed } from "../../../zero-protocol/src/error-kind-enum.js";
 import { ZeroCache } from "../../../zero-protocol/src/error-origin-enum.js";
 import { Internal } from "../../../zero-protocol/src/error-reason-enum.js";
 import { ProtocolError } from "../../../zero-protocol/src/error.js";
-import { clientToServer, serverToClient } from "../../../zero-schema/src/name-mapper.js";
-import { hashOfAST, hashOfNameAndArgs } from "../../../zero-protocol/src/query-hash.js";
 import "./error.js";
 import { clampTTL, compareTTL } from "../../../zql/src/query/ttl.js";
 import { desiredQueriesPrefixForClient } from "./keys.js";

package/out/zero-client/src/client/version.js

@@ -2,7 +2,7 @@
 /**
 * The current version of Zero.
 */
-var version = "1.8.0-canary.5";
+var version = "1.8.0-canary.7";
 //#endregion
 export { version };
 

package/out/zero-client/src/client/zero.js

@@ -8,14 +8,14 @@ import { localNavigator } from "../../../shared/src/navigator.js";
 import { getKVStoreProvider } from "../../../replicache/src/get-kv-store-provider.js";
 import { IDBDatabasesStore } from "../../../replicache/src/persist/idb-databases-store.js";
 import { dropDatabase } from "../../../replicache/src/persist/collect-idb-databases.js";
-import { getErrorMessage } from "../../../shared/src/error.js";
 import { must } from "../../../shared/src/must.js";
+import { clientToServer } from "../../../zero-schema/src/name-mapper.js";
+import { h64 } from "../../../shared/src/hash.js";
+import { getErrorMessage } from "../../../shared/src/error.js";
 import { ClientNotFound, InvalidMessage } from "../../../zero-protocol/src/error-kind-enum.js";
 import { ProtocolError } from "../../../zero-protocol/src/error.js";
-import { h64 } from "../../../shared/src/hash.js";
 import "../../../zero-protocol/src/client-schema.js";
 import { clientSchemaFrom } from "../../../zero-schema/src/builder/schema-builder.js";
-import { clientToServer } from "../../../zero-schema/src/name-mapper.js";
 import { isMutatorRegistry, iterateMutators } from "../../../zql/src/mutate/mutator-registry.js";
 import { createRunnableBuilder } from "../../../zql/src/query/create-builder.js";
 import "../../../zql/src/query/query.js";