@rocicorp/zero 1.11.0-canary.1 → 1.11.0-canary.11

package/out/zero-cache/src/types/websocket-handoff.js.map

@@ -1 +1 @@
-{"version":3,"file":"websocket-handoff.js","names":[],"sources":["../../../../../zero-cache/src/types/websocket-handoff.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport {Server} from 'node:http';\nimport type {Socket} from 'node:net';\nimport type {LogContext} from '@rocicorp/logger';\nimport {WebSocketServer, type ServerOptions, type WebSocket} from 'ws';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {serializableSubset, type IncomingMessageSubset} from './http.ts';\nimport type {MESSAGE_TYPES} from './processes.ts';\nimport {type Receiver, type Sender, type Worker} from './processes.ts';\nimport {closeWithError, PROTOCOL_ERROR} from './ws.ts';\n\nexport type HandoffSpec<P> = {\n  payload: P;\n  sender: Sender;\n};\n\n/**\n * The WebSocketHandoff is a function that either returns the payload and\n * receiver, or invokes the specified `callback` with the payload and\n * receiver. It must not do both.\n *\n * Similarly, an error can be handled by throwing synchronously from the\n * function, or invoking the `onerror` callback.\n */\nexport type WebSocketHandoff<P> = (\n  message: IncomingMessageSubset,\n  callback: (h: HandoffSpec<P>) => void,\n  onerror: (reason: unknown) => void,\n) => HandoffSpec<P> | void;\n\nexport type WebSocketReceiver<P> = (\n  ws: WebSocket,\n  payload: P,\n  msg: IncomingMessageSubset,\n) => void;\n\nexport type WebSocketHandoffHandler = (\n  message: IncomingMessageSubset,\n  socket: Socket,\n  head: ArrayBuffer,\n) => void;\n\n/**\n * Installs websocket handoff logic from either an http.Server\n * receiving requests, or a parent Worker process\n * that is handing off requests to this process.\n */\nexport function createWebSocketHandoffHandler<P>(\n  lc: LogContext,\n  handoff: WebSocketHandoff<P>,\n  serverOptions?: ServerOptions,\n): WebSocketHandoffHandler {\n  const wss = new WebSocketServer(\n    serverOptions ?? {\n      noServer: true,\n    },\n  );\n  return (\n    message: IncomingMessageSubset,\n    socket: Socket,\n    head: ArrayBuffer,\n  ) => {\n    let sent = false;\n\n    function send({payload, sender}: HandoffSpec<P>) {\n      assert(!sent, 'Handoff callback already invoked');\n      sent = true;\n\n      const data = [\n        'handoff',\n        {\n          message: serializableSubset(message),\n          head,\n          payload,\n        },\n      ] satisfies Handoff<P>;\n\n      // \"This event is guaranteed to be passed an instance of the <net.Socket> class\"\n      // https://nodejs.org/api/http.html#event-upgrade\n      sender.send(data, socket);\n    }\n\n    function onError(error: unknown) {\n      // Returning an error on the HTTP handshake looks like a hanging connection\n      // (at least from Chrome) and doesn't report any meaningful error in the browser.\n      // Instead, finish the upgrade to a websocket and then close it with an error.\n      wss.handleUpgrade(\n        message as IncomingMessage,\n        socket,\n        Buffer.from(head),\n        ws => closeWithError(lc, ws, error, PROTOCOL_ERROR),\n      );\n    }\n\n    try {\n      const spec = handoff(message, send, onError);\n      if (spec) {\n        send(spec);\n      }\n    } catch (error) {\n      onError(error);\n    }\n  };\n}\n\n/**\n * Installs websocket handoff logic from either an http.Server\n * receiving requests, or a parent Worker process\n * that is handing off requests to this process.\n */\nexport function installWebSocketHandoff<P>(\n  lc: LogContext,\n  handoff: WebSocketHandoff<P>,\n  source: Server | Worker,\n  serverOptions?: ServerOptions,\n) {\n  const handle = createWebSocketHandoffHandler(lc, handoff, serverOptions);\n\n  if (source instanceof Server) {\n    // handoff messages from an HTTP server\n    source.on('upgrade', handle);\n  } else {\n    // handoff messages from this worker's parent.\n    source.onMessageType<Handoff<P>>('handoff', (msg, socket) => {\n      const {message, head} = msg;\n      handle(message, socket as Socket, head);\n    });\n  }\n}\n\nexport function installWebSocketReceiver<P>(\n  lc: LogContext,\n  server: WebSocketServer,\n  receive: WebSocketReceiver<P>,\n  receiver: Receiver,\n) {\n  receiver.onMessageType<Handoff<P>>('handoff', (msg, socket) => {\n    // Per https://nodejs.org/api/child_process.html#subprocesssendmessage-sendhandle-options-callback\n    //\n    // > Any 'message' handlers in the subprocess should verify that socket\n    // > exists, as the connection may have been closed during the time it\n    // > takes to send the connection to the child.\n    if (!socket) {\n      lc.warn?.('websocket closed during handoff');\n      return;\n    }\n    const {message, head, payload} = msg;\n    server.handleUpgrade(\n      message as IncomingMessage,\n      socket as Socket,\n      Buffer.from(head),\n      ws => {\n        // Guard against WebSocket being closed during handoff.\n        // This can happen due to network issues or client disconnection\n        // between the time the socket was sent and when handleUpgrade completes.\n        if (ws.readyState === ws.CLOSED || ws.readyState === ws.CLOSING) {\n          lc.warn?.('websocket closed during upgrade, skipping receive');\n          return;\n        }\n        receive(ws, payload, message);\n      },\n    );\n  });\n}\n\nexport type Handoff<P> = [\n  typeof MESSAGE_TYPES.handoff,\n  {\n    message: IncomingMessageSubset;\n    head: ArrayBuffer;\n    payload: P;\n  },\n];\n"],"mappings":";;;;;;;;;;;;AA+CA,SAAgB,8BACd,IACA,SACA,eACyB;CACzB,MAAM,MAAM,IAAI,gBACd,iBAAiB,EACf,UAAU,KACZ,CACF;CACA,QACE,SACA,QACA,SACG;EACH,IAAI,OAAO;EAEX,SAAS,KAAK,EAAC,SAAS,UAAyB;GAC/C,OAAO,CAAC,MAAM,kCAAkC;GAChD,OAAO;GAEP,MAAM,OAAO,CACX,WACA;IACE,SAAS,mBAAmB,OAAO;IACnC;IACA;GACF,CACF;GAIA,OAAO,KAAK,MAAM,MAAM;EAC1B;EAEA,SAAS,QAAQ,OAAgB;GAI/B,IAAI,cACF,SACA,QACA,OAAO,KAAK,IAAI,IAChB,OAAM,eAAe,IAAI,IAAI,OAAO,cAAc,CACpD;EACF;EAEA,IAAI;GACF,MAAM,OAAO,QAAQ,SAAS,MAAM,OAAO;GAC3C,IAAI,MACF,KAAK,IAAI;EAEb,SAAS,OAAO;GACd,QAAQ,KAAK;EACf;CACF;AACF;;;;;;AAOA,SAAgB,wBACd,IACA,SACA,QACA,eACA;CACA,MAAM,SAAS,8BAA8B,IAAI,SAAS,aAAa;CAEvE,IAAI,kBAAkB,QAEpB,OAAO,GAAG,WAAW,MAAM;MAG3B,OAAO,cAA0B,YAAY,KAAK,WAAW;EAC3D,MAAM,EAAC,SAAS,SAAQ;EACxB,OAAO,SAAS,QAAkB,IAAI;CACxC,CAAC;AAEL;AAEA,SAAgB,yBACd,IACA,QACA,SACA,UACA;CACA,SAAS,cAA0B,YAAY,KAAK,WAAW;EAM7D,IAAI,CAAC,QAAQ;GACX,GAAG,OAAO,iCAAiC;GAC3C;EACF;EACA,MAAM,EAAC,SAAS,MAAM,YAAW;EACjC,OAAO,cACL,SACA,QACA,OAAO,KAAK,IAAI,IAChB,OAAM;GAIJ,IAAI,GAAG,eAAe,GAAG,UAAU,GAAG,eAAe,GAAG,SAAS;IAC/D,GAAG,OAAO,mDAAmD;IAC7D;GACF;GACA,QAAQ,IAAI,SAAS,OAAO;EAC9B,CACF;CACF,CAAC;AACH"}
+{"version":3,"file":"websocket-handoff.js","names":[],"sources":["../../../../../zero-cache/src/types/websocket-handoff.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport {Server} from 'node:http';\nimport type {Socket} from 'node:net';\nimport type {LogContext} from '@rocicorp/logger';\nimport {WebSocketServer, type ServerOptions, type WebSocket} from 'ws';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {serializableSubset, type IncomingMessageSubset} from './http.ts';\nimport type {MESSAGE_TYPES} from './processes.ts';\nimport {type Receiver, type Sender, type Worker} from './processes.ts';\nimport {closeWithError, PROTOCOL_ERROR} from './ws.ts';\n\nexport type HandoffSpec<P> = {\n  payload: P;\n  sender: Sender;\n};\n\n/**\n * The WebSocketHandoff is a function that either returns the payload and\n * receiver, or invokes the specified `callback` with the payload and\n * receiver. It must not do both.\n *\n * Similarly, an error can be handled by throwing synchronously from the\n * function, or invoking the `onerror` callback.\n */\nexport type WebSocketHandoff<P> = (\n  message: IncomingMessageSubset,\n  callback: (h: HandoffSpec<P>) => void,\n  onerror: (reason: unknown) => void,\n) => HandoffSpec<P> | void;\n\nexport type WebSocketReceiver<P> = (\n  ws: WebSocket,\n  payload: P,\n  msg: IncomingMessageSubset,\n) => void;\n\nexport type WebSocketHandoffHandler = (\n  message: IncomingMessageSubset,\n  socket: Socket,\n  head: ArrayBuffer,\n) => void;\n\n/**\n * Installs websocket handoff logic from either an http.Server\n * receiving requests, or a parent Worker process\n * that is handing off requests to this process.\n */\nexport function createWebSocketHandoffHandler<P>(\n  lc: LogContext,\n  handoff: WebSocketHandoff<P>,\n  serverOptions?: ServerOptions,\n): WebSocketHandoffHandler {\n  const wss = new WebSocketServer(\n    serverOptions ?? {\n      noServer: true,\n    },\n  );\n  return (\n    message: IncomingMessageSubset,\n    socket: Socket,\n    head: ArrayBuffer,\n  ) => {\n    let sent = false;\n\n    function send({payload, sender}: HandoffSpec<P>) {\n      assert(!sent, 'Handoff callback already invoked');\n      sent = true;\n\n      const data = [\n        'handoff',\n        {\n          message: serializableSubset(message),\n          head,\n          payload,\n        },\n      ] satisfies Handoff<P>;\n\n      // \"This event is guaranteed to be passed an instance of the <net.Socket> class\"\n      // https://nodejs.org/api/http.html#event-upgrade\n      sender.send(data, socket);\n    }\n\n    function onError(error: unknown) {\n      // Returning an error on the HTTP handshake looks like a hanging connection\n      // (at least from Chrome) and doesn't report any meaningful error in the browser.\n      // Instead, finish the upgrade to a websocket and then close it with an error.\n      wss.handleUpgrade(\n        message as IncomingMessage,\n        socket,\n        Buffer.from(head),\n        ws => closeWithError(lc, ws, error, PROTOCOL_ERROR),\n      );\n    }\n\n    try {\n      const spec = handoff(message, send, onError);\n      if (spec) {\n        send(spec);\n      }\n    } catch (error) {\n      onError(error);\n    }\n  };\n}\n\n/**\n * Installs websocket handoff logic from either an http.Server\n * receiving requests, or a parent Worker process\n * that is handing off requests to this process.\n */\nexport function installWebSocketHandoff<P>(\n  lc: LogContext,\n  handoff: WebSocketHandoff<P>,\n  source: Server | Worker,\n  serverOptions?: ServerOptions,\n) {\n  const handle = createWebSocketHandoffHandler(lc, handoff, serverOptions);\n\n  if (source instanceof Server) {\n    // handoff messages from an HTTP server\n    source.on('upgrade', handle);\n  } else {\n    // handoff messages from this worker's parent.\n    source.onMessageType<Handoff<P>>('handoff', (msg, socket) => {\n      const {message, head} = msg;\n      handle(message, socket as Socket, head);\n    });\n  }\n}\n\nexport function installWebSocketReceiver<P>(\n  lc: LogContext,\n  server: WebSocketServer,\n  receive: WebSocketReceiver<P>,\n  receiver: Receiver,\n  onAbort?: ((payload: P) => void) | undefined,\n  onReceived?: ((payload: P) => void) | undefined,\n) {\n  receiver.onMessageType<Handoff<P>>('handoff', (msg, socket) => {\n    const {message, head, payload} = msg;\n    onReceived?.(payload);\n    // Per https://nodejs.org/api/child_process.html#subprocesssendmessage-sendhandle-options-callback\n    //\n    // > Any 'message' handlers in the subprocess should verify that socket\n    // > exists, as the connection may have been closed during the time it\n    // > takes to send the connection to the child.\n    if (!socket) {\n      lc.warn?.('websocket closed during handoff');\n      onAbort?.(payload);\n      return;\n    }\n    try {\n      server.handleUpgrade(\n        message as IncomingMessage,\n        socket as Socket,\n        Buffer.from(head),\n        ws => {\n          // Guard against WebSocket being closed during handoff.\n          // This can happen due to network issues or client disconnection\n          // between the time the socket was sent and when handleUpgrade completes.\n          if (ws.readyState === ws.CLOSED || ws.readyState === ws.CLOSING) {\n            lc.warn?.('websocket closed during upgrade, skipping receive');\n            onAbort?.(payload);\n            return;\n          }\n          receive(ws, payload, message);\n        },\n      );\n    } catch (e) {\n      lc.warn?.('error during websocket handleUpgrade', e);\n      (socket as Socket)?.destroy?.();\n      onAbort?.(payload);\n    }\n  });\n}\n\nexport type Handoff<P> = [\n  typeof MESSAGE_TYPES.handoff,\n  {\n    message: IncomingMessageSubset;\n    head: ArrayBuffer;\n    payload: P;\n  },\n];\n"],"mappings":";;;;;;;;;;;;AA+CA,SAAgB,8BACd,IACA,SACA,eACyB;CACzB,MAAM,MAAM,IAAI,gBACd,iBAAiB,EACf,UAAU,KACZ,CACF;CACA,QACE,SACA,QACA,SACG;EACH,IAAI,OAAO;EAEX,SAAS,KAAK,EAAC,SAAS,UAAyB;GAC/C,OAAO,CAAC,MAAM,kCAAkC;GAChD,OAAO;GAEP,MAAM,OAAO,CACX,WACA;IACE,SAAS,mBAAmB,OAAO;IACnC;IACA;GACF,CACF;GAIA,OAAO,KAAK,MAAM,MAAM;EAC1B;EAEA,SAAS,QAAQ,OAAgB;GAI/B,IAAI,cACF,SACA,QACA,OAAO,KAAK,IAAI,IAChB,OAAM,eAAe,IAAI,IAAI,OAAO,cAAc,CACpD;EACF;EAEA,IAAI;GACF,MAAM,OAAO,QAAQ,SAAS,MAAM,OAAO;GAC3C,IAAI,MACF,KAAK,IAAI;EAEb,SAAS,OAAO;GACd,QAAQ,KAAK;EACf;CACF;AACF;;;;;;AAOA,SAAgB,wBACd,IACA,SACA,QACA,eACA;CACA,MAAM,SAAS,8BAA8B,IAAI,SAAS,aAAa;CAEvE,IAAI,kBAAkB,QAEpB,OAAO,GAAG,WAAW,MAAM;MAG3B,OAAO,cAA0B,YAAY,KAAK,WAAW;EAC3D,MAAM,EAAC,SAAS,SAAQ;EACxB,OAAO,SAAS,QAAkB,IAAI;CACxC,CAAC;AAEL;AAEA,SAAgB,yBACd,IACA,QACA,SACA,UACA,SACA,YACA;CACA,SAAS,cAA0B,YAAY,KAAK,WAAW;EAC7D,MAAM,EAAC,SAAS,MAAM,YAAW;EACjC,aAAa,OAAO;EAMpB,IAAI,CAAC,QAAQ;GACX,GAAG,OAAO,iCAAiC;GAC3C,UAAU,OAAO;GACjB;EACF;EACA,IAAI;GACF,OAAO,cACL,SACA,QACA,OAAO,KAAK,IAAI,IAChB,OAAM;IAIJ,IAAI,GAAG,eAAe,GAAG,UAAU,GAAG,eAAe,GAAG,SAAS;KAC/D,GAAG,OAAO,mDAAmD;KAC7D,UAAU,OAAO;KACjB;IACF;IACA,QAAQ,IAAI,SAAS,OAAO;GAC9B,CACF;EACF,SAAS,GAAG;GACV,GAAG,OAAO,wCAAwC,CAAC;GACnD,QAAoB,UAAU;GAC9B,UAAU,OAAO;EACnB;CACF,CAAC;AACH"}

package/out/zero-cache/src/types/ws.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ws.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/types/ws.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,IAAI,CAAC;AAIlC,eAAO,MAAM,cAAc,OAAO,CAAC;AACnC,eAAO,MAAM,cAAc,OAAO,CAAC;AAEnC,MAAM,MAAM,SAAS,GAAG,OAAO,cAAc,GAAG,OAAO,cAAc,CAAC;AAEtE,wBAAgB,cAAc,CAC5B,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,SAAS,EACb,GAAG,EAAE,OAAO,EACZ,IAAI,GAAE,SAA0B,QASjC;AAED,wBAAgB,oBAAoB,CAClC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,SAAS,EACb,UAAU,EAAE,MAAM,QAkCnB;AAED,wBAAgB,sBAAsB,CACpC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,SAAS,EACb,UAAU,EAAE,MAAM,EAClB,eAAe,SAAQ,QAwBxB"}
+{"version":3,"file":"ws.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/types/ws.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,IAAI,CAAC;AAIlC,eAAO,MAAM,cAAc,OAAO,CAAC;AACnC,eAAO,MAAM,cAAc,OAAO,CAAC;AAEnC,MAAM,MAAM,SAAS,GAAG,OAAO,cAAc,GAAG,OAAO,cAAc,CAAC;AAEtE,wBAAgB,cAAc,CAC5B,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,SAAS,EACb,GAAG,EAAE,OAAO,EACZ,IAAI,GAAE,SAA0B,QASjC;AAED,wBAAgB,oBAAoB,CAClC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,SAAS,EACb,UAAU,EAAE,MAAM,QAkCnB;AAED,wBAAgB,sBAAsB,CACpC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,SAAS,EACb,UAAU,EAAE,MAAM,EAClB,eAAe,SAAQ,QA+CxB"}

package/out/zero-cache/src/types/ws.js

@@ -30,19 +30,34 @@ function sendPingsForLiveness(lc, ws, intervalMs) {
 	ws.once("close", () => clearInterval(livenessTimer));
 }
 function expectPingsForLiveness(lc, ws, intervalMs, timeoutBufferMs = 3e3) {
+	const timeoutMs = intervalMs + timeoutBufferMs;
 	let gotLivenessSignal = false;
-	const livenessTimer = setInterval(() => {
-		if (!gotLivenessSignal) {
-			lc.warn?.(`socket@${ws.url} did not send heartbeat or messages. Terminating...`);
+	let livenessTimer;
+	function startLivenessChecks() {
+		livenessTimer = setInterval(() => {
+			if (!gotLivenessSignal) {
+				lc.warn?.(`socket@${ws.url} did not send heartbeat or messages. Terminating...`);
+				ws.terminate();
+				return;
+			}
+			gotLivenessSignal = false;
+		}, timeoutMs);
+	}
+	if (ws.readyState === ws.CONNECTING) {
+		const connectTimer = setTimeout(() => {
+			lc.warn?.(`socket@${ws.url} did not connect within ${timeoutMs} ms. Terminating...`);
 			ws.terminate();
-			return;
-		}
-		gotLivenessSignal = false;
-	}, intervalMs + timeoutBufferMs);
+		}, timeoutMs);
+		ws.once("open", () => {
+			clearTimeout(connectTimer);
+			startLivenessChecks();
+		});
+		ws.once("close", () => clearTimeout(connectTimer));
+	} else if (ws.readyState === ws.OPEN) startLivenessChecks();
 	const signalAlive = () => gotLivenessSignal = true;
 	ws.on("ping", signalAlive);
 	ws.on("message", signalAlive);
-	ws.once("close", () => clearTimeout(livenessTimer));
+	ws.once("close", () => clearInterval(livenessTimer));
 }
 //#endregion
 export { PROTOCOL_ERROR, closeWithError, expectPingsForLiveness, sendPingsForLiveness };

package/out/zero-cache/src/types/ws.js.map

@@ -1 +1 @@
-{"version":3,"file":"ws.js","names":[],"sources":["../../../../../zero-cache/src/types/ws.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {WebSocket} from 'ws';\nimport {elide} from './strings.ts';\n\n// https://github.com/Luka967/websocket-close-codes\nexport const PROTOCOL_ERROR = 1002;\nexport const INTERNAL_ERROR = 1011;\n\nexport type ErrorCode = typeof PROTOCOL_ERROR | typeof INTERNAL_ERROR;\n\nexport function closeWithError(\n  lc: LogContext,\n  ws: WebSocket,\n  err: unknown,\n  code: ErrorCode = INTERNAL_ERROR,\n) {\n  const endpoint = ws.url ?? 'client';\n  const errMsg = String(err);\n  lc.warn?.(`closing connection to ${endpoint} with error`, errMsg);\n\n  // close messages must be less than or equal to 123 bytes:\n  // https://developer.mozilla.org/en-US/docs/Web/API/WebSocket/close#reason\n  ws.close(code, elide(errMsg, 123));\n}\n\nexport function sendPingsForLiveness(\n  lc: LogContext,\n  ws: WebSocket,\n  intervalMs: number,\n) {\n  let gotLivenessSignal = true;\n\n  let livenessTimer: NodeJS.Timeout | undefined;\n  function startHeartBeats() {\n    livenessTimer = setInterval(() => {\n      if (!gotLivenessSignal) {\n        lc.warn?.(\n          `socket@${ws.url} did not respond to heartbeat. Terminating...`,\n        );\n        ws.terminate();\n        return;\n      }\n      // Reset gotLivenessSignal and expect another pong or message to arrive\n      // before the next interval elapses.\n      gotLivenessSignal = false;\n      ws.ping();\n    }, intervalMs);\n  }\n\n  if (ws.readyState === ws.CONNECTING) {\n    ws.once('open', () => startHeartBeats());\n  } else if (ws.readyState === ws.OPEN) {\n    startHeartBeats();\n  }\n\n  // Both pongs and messages are accepted as signs of liveness.\n  // Checking for pongs only risks false positives as pongs may be backed\n  // up behind a large stream of messages.\n  const signalAlive = () => (gotLivenessSignal = true);\n  ws.on('pong', signalAlive);\n  ws.on('message', signalAlive);\n  ws.once('close', () => clearInterval(livenessTimer));\n}\n\nexport function expectPingsForLiveness(\n  lc: LogContext,\n  ws: WebSocket,\n  intervalMs: number,\n  timeoutBufferMs = 3_000,\n) {\n  let gotLivenessSignal = false;\n\n  const livenessTimer = setInterval(() => {\n    if (!gotLivenessSignal) {\n      lc.warn?.(\n        `socket@${ws.url} did not send heartbeat or messages. Terminating...`,\n      );\n      ws.terminate();\n      return;\n    }\n    // Reset gotLivenessSignal and expect another ping or message to arrive\n    // before the next interval elapses.\n    gotLivenessSignal = false;\n  }, intervalMs + timeoutBufferMs);\n\n  // Both pings and messages are accepted as signs of liveness.\n  // Checking for pings only risks false positives as pings may be backed\n  // up behind a large stream of messages.\n  const signalAlive = () => (gotLivenessSignal = true);\n  ws.on('ping', signalAlive);\n  ws.on('message', signalAlive);\n  ws.once('close', () => clearTimeout(livenessTimer));\n}\n"],"mappings":";;AAKA,IAAa,iBAAiB;AAC9B,IAAa,iBAAiB;AAI9B,SAAgB,eACd,IACA,IACA,KACA,OAAkB,gBAClB;CACA,MAAM,WAAW,GAAG,OAAO;CAC3B,MAAM,SAAS,OAAO,GAAG;CACzB,GAAG,OAAO,yBAAyB,SAAS,cAAc,MAAM;CAIhE,GAAG,MAAM,MAAM,MAAM,QAAQ,GAAG,CAAC;AACnC;AAEA,SAAgB,qBACd,IACA,IACA,YACA;CACA,IAAI,oBAAoB;CAExB,IAAI;CACJ,SAAS,kBAAkB;EACzB,gBAAgB,kBAAkB;GAChC,IAAI,CAAC,mBAAmB;IACtB,GAAG,OACD,UAAU,GAAG,IAAI,8CACnB;IACA,GAAG,UAAU;IACb;GACF;GAGA,oBAAoB;GACpB,GAAG,KAAK;EACV,GAAG,UAAU;CACf;CAEA,IAAI,GAAG,eAAe,GAAG,YACvB,GAAG,KAAK,cAAc,gBAAgB,CAAC;MAClC,IAAI,GAAG,eAAe,GAAG,MAC9B,gBAAgB;CAMlB,MAAM,oBAAqB,oBAAoB;CAC/C,GAAG,GAAG,QAAQ,WAAW;CACzB,GAAG,GAAG,WAAW,WAAW;CAC5B,GAAG,KAAK,eAAe,cAAc,aAAa,CAAC;AACrD;AAEA,SAAgB,uBACd,IACA,IACA,YACA,kBAAkB,KAClB;CACA,IAAI,oBAAoB;CAExB,MAAM,gBAAgB,kBAAkB;EACtC,IAAI,CAAC,mBAAmB;GACtB,GAAG,OACD,UAAU,GAAG,IAAI,oDACnB;GACA,GAAG,UAAU;GACb;EACF;EAGA,oBAAoB;CACtB,GAAG,aAAa,eAAe;CAK/B,MAAM,oBAAqB,oBAAoB;CAC/C,GAAG,GAAG,QAAQ,WAAW;CACzB,GAAG,GAAG,WAAW,WAAW;CAC5B,GAAG,KAAK,eAAe,aAAa,aAAa,CAAC;AACpD"}
+{"version":3,"file":"ws.js","names":[],"sources":["../../../../../zero-cache/src/types/ws.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {WebSocket} from 'ws';\nimport {elide} from './strings.ts';\n\n// https://github.com/Luka967/websocket-close-codes\nexport const PROTOCOL_ERROR = 1002;\nexport const INTERNAL_ERROR = 1011;\n\nexport type ErrorCode = typeof PROTOCOL_ERROR | typeof INTERNAL_ERROR;\n\nexport function closeWithError(\n  lc: LogContext,\n  ws: WebSocket,\n  err: unknown,\n  code: ErrorCode = INTERNAL_ERROR,\n) {\n  const endpoint = ws.url ?? 'client';\n  const errMsg = String(err);\n  lc.warn?.(`closing connection to ${endpoint} with error`, errMsg);\n\n  // close messages must be less than or equal to 123 bytes:\n  // https://developer.mozilla.org/en-US/docs/Web/API/WebSocket/close#reason\n  ws.close(code, elide(errMsg, 123));\n}\n\nexport function sendPingsForLiveness(\n  lc: LogContext,\n  ws: WebSocket,\n  intervalMs: number,\n) {\n  let gotLivenessSignal = true;\n\n  let livenessTimer: NodeJS.Timeout | undefined;\n  function startHeartBeats() {\n    livenessTimer = setInterval(() => {\n      if (!gotLivenessSignal) {\n        lc.warn?.(\n          `socket@${ws.url} did not respond to heartbeat. Terminating...`,\n        );\n        ws.terminate();\n        return;\n      }\n      // Reset gotLivenessSignal and expect another pong or message to arrive\n      // before the next interval elapses.\n      gotLivenessSignal = false;\n      ws.ping();\n    }, intervalMs);\n  }\n\n  if (ws.readyState === ws.CONNECTING) {\n    ws.once('open', () => startHeartBeats());\n  } else if (ws.readyState === ws.OPEN) {\n    startHeartBeats();\n  }\n\n  // Both pongs and messages are accepted as signs of liveness.\n  // Checking for pongs only risks false positives as pongs may be backed\n  // up behind a large stream of messages.\n  const signalAlive = () => (gotLivenessSignal = true);\n  ws.on('pong', signalAlive);\n  ws.on('message', signalAlive);\n  ws.once('close', () => clearInterval(livenessTimer));\n}\n\nexport function expectPingsForLiveness(\n  lc: LogContext,\n  ws: WebSocket,\n  intervalMs: number,\n  timeoutBufferMs = 3_000,\n) {\n  const timeoutMs = intervalMs + timeoutBufferMs;\n  let gotLivenessSignal = false;\n\n  let livenessTimer: NodeJS.Timeout | undefined;\n  function startLivenessChecks() {\n    livenessTimer = setInterval(() => {\n      if (!gotLivenessSignal) {\n        lc.warn?.(\n          `socket@${ws.url} did not send heartbeat or messages. Terminating...`,\n        );\n        ws.terminate();\n        return;\n      }\n      // Reset gotLivenessSignal and expect another ping or message to arrive\n      // before the next interval elapses.\n      gotLivenessSignal = false;\n    }, timeoutMs);\n  }\n\n  if (ws.readyState === ws.CONNECTING) {\n    // Heartbeats only start once the connection is open, so time out the\n    // handshake separately. Otherwise a slow handshake eats into the first\n    // heartbeat interval, and a stuck one is reported as a missed heartbeat.\n    const connectTimer = setTimeout(() => {\n      lc.warn?.(\n        `socket@${ws.url} did not connect within ${timeoutMs} ms. Terminating...`,\n      );\n      ws.terminate();\n    }, timeoutMs);\n    ws.once('open', () => {\n      clearTimeout(connectTimer);\n      startLivenessChecks();\n    });\n    ws.once('close', () => clearTimeout(connectTimer));\n  } else if (ws.readyState === ws.OPEN) {\n    startLivenessChecks();\n  }\n\n  // Both pings and messages are accepted as signs of liveness.\n  // Checking for pings only risks false positives as pings may be backed\n  // up behind a large stream of messages.\n  const signalAlive = () => (gotLivenessSignal = true);\n  ws.on('ping', signalAlive);\n  ws.on('message', signalAlive);\n  ws.once('close', () => clearInterval(livenessTimer));\n}\n"],"mappings":";;AAKA,IAAa,iBAAiB;AAC9B,IAAa,iBAAiB;AAI9B,SAAgB,eACd,IACA,IACA,KACA,OAAkB,gBAClB;CACA,MAAM,WAAW,GAAG,OAAO;CAC3B,MAAM,SAAS,OAAO,GAAG;CACzB,GAAG,OAAO,yBAAyB,SAAS,cAAc,MAAM;CAIhE,GAAG,MAAM,MAAM,MAAM,QAAQ,GAAG,CAAC;AACnC;AAEA,SAAgB,qBACd,IACA,IACA,YACA;CACA,IAAI,oBAAoB;CAExB,IAAI;CACJ,SAAS,kBAAkB;EACzB,gBAAgB,kBAAkB;GAChC,IAAI,CAAC,mBAAmB;IACtB,GAAG,OACD,UAAU,GAAG,IAAI,8CACnB;IACA,GAAG,UAAU;IACb;GACF;GAGA,oBAAoB;GACpB,GAAG,KAAK;EACV,GAAG,UAAU;CACf;CAEA,IAAI,GAAG,eAAe,GAAG,YACvB,GAAG,KAAK,cAAc,gBAAgB,CAAC;MAClC,IAAI,GAAG,eAAe,GAAG,MAC9B,gBAAgB;CAMlB,MAAM,oBAAqB,oBAAoB;CAC/C,GAAG,GAAG,QAAQ,WAAW;CACzB,GAAG,GAAG,WAAW,WAAW;CAC5B,GAAG,KAAK,eAAe,cAAc,aAAa,CAAC;AACrD;AAEA,SAAgB,uBACd,IACA,IACA,YACA,kBAAkB,KAClB;CACA,MAAM,YAAY,aAAa;CAC/B,IAAI,oBAAoB;CAExB,IAAI;CACJ,SAAS,sBAAsB;EAC7B,gBAAgB,kBAAkB;GAChC,IAAI,CAAC,mBAAmB;IACtB,GAAG,OACD,UAAU,GAAG,IAAI,oDACnB;IACA,GAAG,UAAU;IACb;GACF;GAGA,oBAAoB;EACtB,GAAG,SAAS;CACd;CAEA,IAAI,GAAG,eAAe,GAAG,YAAY;EAInC,MAAM,eAAe,iBAAiB;GACpC,GAAG,OACD,UAAU,GAAG,IAAI,0BAA0B,UAAU,oBACvD;GACA,GAAG,UAAU;EACf,GAAG,SAAS;EACZ,GAAG,KAAK,cAAc;GACpB,aAAa,YAAY;GACzB,oBAAoB;EACtB,CAAC;EACD,GAAG,KAAK,eAAe,aAAa,YAAY,CAAC;CACnD,OAAO,IAAI,GAAG,eAAe,GAAG,MAC9B,oBAAoB;CAMtB,MAAM,oBAAqB,oBAAoB;CAC/C,GAAG,GAAG,QAAQ,WAAW;CACzB,GAAG,GAAG,WAAW,WAAW;CAC5B,GAAG,KAAK,eAAe,cAAc,aAAa,CAAC;AACrD"}

package/out/zero-cache/src/workers/connect-params.d.ts

@@ -16,6 +16,7 @@ export type ConnectParams = {
     readonly httpCookie: string | undefined;
     readonly origin: string | undefined;
     readonly requestHeaders?: Readonly<Record<string, string>> | undefined;
+    readonly generation?: number | undefined;
 };
 export declare function getConnectParams(protocolVersion: number, url: URL, headers: IncomingHttpHeaders): {
     params: ConnectParams;

package/out/zero-cache/src/workers/connect-params.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"connect-params.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/workers/connect-params.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,WAAW,CAAC;AAEnD,OAAO,EAEL,KAAK,qBAAqB,EAC3B,MAAM,uCAAuC,CAAC;AAG/C,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;IAC5B,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;IAClC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,QAAQ,CAAC,iBAAiB,EAAE,qBAAqB,GAAG,SAAS,CAAC;IAC9D,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IACxC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,QAAQ,CAAC,cAAc,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,GAAG,SAAS,CAAC;CACxE,CAAC;AAoBF,wBAAgB,gBAAgB,CAC9B,eAAe,EAAE,MAAM,EACvB,GAAG,EAAE,GAAG,EACR,OAAO,EAAE,mBAAmB,GAE1B;IACE,MAAM,EAAE,aAAa,CAAC;IACtB,KAAK,EAAE,IAAI,CAAC;CACb,GACD;IACE,MAAM,EAAE,IAAI,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;CACf,CA2CJ"}
+{"version":3,"file":"connect-params.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/workers/connect-params.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,WAAW,CAAC;AAEnD,OAAO,EAEL,KAAK,qBAAqB,EAC3B,MAAM,uCAAuC,CAAC;AAG/C,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;IAC5B,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;IAClC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,QAAQ,CAAC,iBAAiB,EAAE,qBAAqB,GAAG,SAAS,CAAC;IAC9D,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IACxC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,QAAQ,CAAC,cAAc,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,GAAG,SAAS,CAAC;IACvE,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC1C,CAAC;AAoBF,wBAAgB,gBAAgB,CAC9B,eAAe,EAAE,MAAM,EACvB,GAAG,EAAE,GAAG,EACR,OAAO,EAAE,mBAAmB,GAE1B;IACE,MAAM,EAAE,aAAa,CAAC;IACtB,KAAK,EAAE,IAAI,CAAC;CACb,GACD;IACE,MAAM,EAAE,IAAI,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;CACf,CA2CJ"}

package/out/zero-cache/src/workers/connect-params.js.map

@@ -1 +1 @@
-{"version":3,"file":"connect-params.js","names":[],"sources":["../../../../../zero-cache/src/workers/connect-params.ts"],"sourcesContent":["import type {IncomingHttpHeaders} from 'node:http';\nimport {must} from '../../../shared/src/must.ts';\nimport {\n  decodeSecProtocols,\n  type InitConnectionMessage,\n} from '../../../zero-protocol/src/connect.ts';\nimport {URLParams} from '../types/url-params.ts';\n\nexport type ConnectParams = {\n  readonly protocolVersion: number;\n  readonly clientID: string;\n  readonly clientGroupID: string;\n  readonly profileID: string | null;\n  readonly baseCookie: string | null;\n  readonly timestamp: number;\n  readonly lmID: number;\n  readonly wsID: string;\n  readonly debugPerf: boolean;\n  readonly auth: string | undefined;\n  readonly userID: string | undefined;\n  readonly initConnectionMsg: InitConnectionMessage | undefined;\n  readonly httpCookie: string | undefined;\n  readonly origin: string | undefined;\n  readonly requestHeaders?: Readonly<Record<string, string>> | undefined;\n};\n\n/**\n * Normalizes Node's {@link IncomingHttpHeaders} (whose values are\n * `string | string[] | undefined`) into a plain `Record<string, string>`,\n * joining array values with `, ` and dropping `undefined` values.\n */\nfunction normalizeHeaders(\n  headers: IncomingHttpHeaders,\n): Record<string, string> {\n  const normalized: Record<string, string> = {};\n  for (const [key, value] of Object.entries(headers)) {\n    if (value === undefined) {\n      continue;\n    }\n    normalized[key] = Array.isArray(value) ? value.join(', ') : value;\n  }\n  return normalized;\n}\n\nexport function getConnectParams(\n  protocolVersion: number,\n  url: URL,\n  headers: IncomingHttpHeaders,\n):\n  | {\n      params: ConnectParams;\n      error: null;\n    }\n  | {\n      params: null;\n      error: string;\n    } {\n  const params = new URLParams(url);\n\n  try {\n    const clientID = params.get('clientID', true);\n    const clientGroupID = params.get('clientGroupID', true);\n    const profileID = params.get('profileID', false);\n    const baseCookie = params.get('baseCookie', false);\n    const timestamp = params.getInteger('ts', true);\n    const lmID = params.getInteger('lmid', true);\n    const wsID = params.get('wsid', false) ?? '';\n    const userID = params.get('userID', false) ?? undefined;\n    const debugPerf = params.getBoolean('debugPerf');\n    const {initConnectionMessage, authToken} = decodeSecProtocols(\n      must(headers['sec-websocket-protocol']),\n    );\n\n    return {\n      params: {\n        protocolVersion,\n        clientID,\n        clientGroupID,\n        profileID,\n        baseCookie,\n        timestamp,\n        lmID,\n        wsID,\n        debugPerf,\n        initConnectionMsg: initConnectionMessage,\n        auth: authToken,\n        userID,\n        httpCookie: headers.cookie,\n        origin: headers.origin,\n        requestHeaders: normalizeHeaders(headers),\n      },\n      error: null,\n    };\n  } catch (e) {\n    return {\n      params: null,\n      error: e instanceof Error ? e.message : String(e),\n    };\n  }\n}\n"],"mappings":";;;;;;;;;AA+BA,SAAS,iBACP,SACwB;CACxB,MAAM,aAAqC,CAAC;CAC5C,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,OAAO,GAAG;EAClD,IAAI,UAAU,KAAA,GACZ;EAEF,WAAW,OAAO,MAAM,QAAQ,KAAK,IAAI,MAAM,KAAK,IAAI,IAAI;CAC9D;CACA,OAAO;AACT;AAEA,SAAgB,iBACd,iBACA,KACA,SASI;CACJ,MAAM,SAAS,IAAI,UAAU,GAAG;CAEhC,IAAI;EACF,MAAM,WAAW,OAAO,IAAI,YAAY,IAAI;EAC5C,MAAM,gBAAgB,OAAO,IAAI,iBAAiB,IAAI;EACtD,MAAM,YAAY,OAAO,IAAI,aAAa,KAAK;EAC/C,MAAM,aAAa,OAAO,IAAI,cAAc,KAAK;EACjD,MAAM,YAAY,OAAO,WAAW,MAAM,IAAI;EAC9C,MAAM,OAAO,OAAO,WAAW,QAAQ,IAAI;EAC3C,MAAM,OAAO,OAAO,IAAI,QAAQ,KAAK,KAAK;EAC1C,MAAM,SAAS,OAAO,IAAI,UAAU,KAAK,KAAK,KAAA;EAC9C,MAAM,YAAY,OAAO,WAAW,WAAW;EAC/C,MAAM,EAAC,uBAAuB,cAAa,mBACzC,KAAK,QAAQ,yBAAyB,CACxC;EAEA,OAAO;GACL,QAAQ;IACN;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA,mBAAmB;IACnB,MAAM;IACN;IACA,YAAY,QAAQ;IACpB,QAAQ,QAAQ;IAChB,gBAAgB,iBAAiB,OAAO;GAC1C;GACA,OAAO;EACT;CACF,SAAS,GAAG;EACV,OAAO;GACL,QAAQ;GACR,OAAO,aAAa,QAAQ,EAAE,UAAU,OAAO,CAAC;EAClD;CACF;AACF"}
+{"version":3,"file":"connect-params.js","names":[],"sources":["../../../../../zero-cache/src/workers/connect-params.ts"],"sourcesContent":["import type {IncomingHttpHeaders} from 'node:http';\nimport {must} from '../../../shared/src/must.ts';\nimport {\n  decodeSecProtocols,\n  type InitConnectionMessage,\n} from '../../../zero-protocol/src/connect.ts';\nimport {URLParams} from '../types/url-params.ts';\n\nexport type ConnectParams = {\n  readonly protocolVersion: number;\n  readonly clientID: string;\n  readonly clientGroupID: string;\n  readonly profileID: string | null;\n  readonly baseCookie: string | null;\n  readonly timestamp: number;\n  readonly lmID: number;\n  readonly wsID: string;\n  readonly debugPerf: boolean;\n  readonly auth: string | undefined;\n  readonly userID: string | undefined;\n  readonly initConnectionMsg: InitConnectionMessage | undefined;\n  readonly httpCookie: string | undefined;\n  readonly origin: string | undefined;\n  readonly requestHeaders?: Readonly<Record<string, string>> | undefined;\n  readonly generation?: number | undefined;\n};\n\n/**\n * Normalizes Node's {@link IncomingHttpHeaders} (whose values are\n * `string | string[] | undefined`) into a plain `Record<string, string>`,\n * joining array values with `, ` and dropping `undefined` values.\n */\nfunction normalizeHeaders(\n  headers: IncomingHttpHeaders,\n): Record<string, string> {\n  const normalized: Record<string, string> = {};\n  for (const [key, value] of Object.entries(headers)) {\n    if (value === undefined) {\n      continue;\n    }\n    normalized[key] = Array.isArray(value) ? value.join(', ') : value;\n  }\n  return normalized;\n}\n\nexport function getConnectParams(\n  protocolVersion: number,\n  url: URL,\n  headers: IncomingHttpHeaders,\n):\n  | {\n      params: ConnectParams;\n      error: null;\n    }\n  | {\n      params: null;\n      error: string;\n    } {\n  const params = new URLParams(url);\n\n  try {\n    const clientID = params.get('clientID', true);\n    const clientGroupID = params.get('clientGroupID', true);\n    const profileID = params.get('profileID', false);\n    const baseCookie = params.get('baseCookie', false);\n    const timestamp = params.getInteger('ts', true);\n    const lmID = params.getInteger('lmid', true);\n    const wsID = params.get('wsid', false) ?? '';\n    const userID = params.get('userID', false) ?? undefined;\n    const debugPerf = params.getBoolean('debugPerf');\n    const {initConnectionMessage, authToken} = decodeSecProtocols(\n      must(headers['sec-websocket-protocol']),\n    );\n\n    return {\n      params: {\n        protocolVersion,\n        clientID,\n        clientGroupID,\n        profileID,\n        baseCookie,\n        timestamp,\n        lmID,\n        wsID,\n        debugPerf,\n        initConnectionMsg: initConnectionMessage,\n        auth: authToken,\n        userID,\n        httpCookie: headers.cookie,\n        origin: headers.origin,\n        requestHeaders: normalizeHeaders(headers),\n      },\n      error: null,\n    };\n  } catch (e) {\n    return {\n      params: null,\n      error: e instanceof Error ? e.message : String(e),\n    };\n  }\n}\n"],"mappings":";;;;;;;;;AAgCA,SAAS,iBACP,SACwB;CACxB,MAAM,aAAqC,CAAC;CAC5C,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,OAAO,GAAG;EAClD,IAAI,UAAU,KAAA,GACZ;EAEF,WAAW,OAAO,MAAM,QAAQ,KAAK,IAAI,MAAM,KAAK,IAAI,IAAI;CAC9D;CACA,OAAO;AACT;AAEA,SAAgB,iBACd,iBACA,KACA,SASI;CACJ,MAAM,SAAS,IAAI,UAAU,GAAG;CAEhC,IAAI;EACF,MAAM,WAAW,OAAO,IAAI,YAAY,IAAI;EAC5C,MAAM,gBAAgB,OAAO,IAAI,iBAAiB,IAAI;EACtD,MAAM,YAAY,OAAO,IAAI,aAAa,KAAK;EAC/C,MAAM,aAAa,OAAO,IAAI,cAAc,KAAK;EACjD,MAAM,YAAY,OAAO,WAAW,MAAM,IAAI;EAC9C,MAAM,OAAO,OAAO,WAAW,QAAQ,IAAI;EAC3C,MAAM,OAAO,OAAO,IAAI,QAAQ,KAAK,KAAK;EAC1C,MAAM,SAAS,OAAO,IAAI,UAAU,KAAK,KAAK,KAAA;EAC9C,MAAM,YAAY,OAAO,WAAW,WAAW;EAC/C,MAAM,EAAC,uBAAuB,cAAa,mBACzC,KAAK,QAAQ,yBAAyB,CACxC;EAEA,OAAO;GACL,QAAQ;IACN;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA,mBAAmB;IACnB,MAAM;IACN;IACA,YAAY,QAAQ;IACpB,QAAQ,QAAQ;IAChB,gBAAgB,iBAAiB,OAAO;GAC1C;GACA,OAAO;EACT;CACF,SAAS,GAAG;EACV,OAAO;GACL,QAAQ;GACR,OAAO,aAAa,QAAQ,EAAE,UAAU,OAAO,CAAC;EAClD;CACF;AACF"}

package/out/zero-cache/src/workers/syncer-ws-message-handler.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"syncer-ws-message-handler.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/workers/syncer-ws-message-handler.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAOjD,OAAO,KAAK,EAAC,gCAAgC,EAAC,MAAM,kCAAkC,CAAC;AACvF,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,gCAAgC,CAAC;AAC5D,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,+BAA+B,CAAC;AAC1D,OAAO,EACL,KAAK,wBAAwB,EAE9B,MAAM,uDAAuD,CAAC;AAC/D,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,wCAAwC,CAAC;AACvE,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,qBAAqB,CAAC;AACvD,OAAO,KAAK,EAAC,aAAa,EAAE,cAAc,EAAC,MAAM,iBAAiB,CAAC;AAiBnE,qBAAa,sBAAuB,YAAW,cAAc;;IAU3D,YACE,EAAE,EAAE,UAAU,EACd,aAAa,EAAE,aAAa,EAC5B,kBAAkB,EAAE,wBAAwB,EAC5C,UAAU,EAAE,UAAU,EACtB,OAAO,EAAE,OAAO,GAAG,SAAS,EAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,EAkB3B;IAEK,aAAa,CACjB,GAAG,EAAE,gCAAgC,GACpC,OAAO,CAAC,aAAa,EAAE,CAAC,CAkN1B;CACF"}
+{"version":3,"file":"syncer-ws-message-handler.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/workers/syncer-ws-message-handler.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAOjD,OAAO,KAAK,EAAC,gCAAgC,EAAC,MAAM,kCAAkC,CAAC;AACvF,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,gCAAgC,CAAC;AAC5D,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,+BAA+B,CAAC;AAC1D,OAAO,EACL,KAAK,wBAAwB,EAE9B,MAAM,uDAAuD,CAAC;AAC/D,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,wCAAwC,CAAC;AACvE,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,qBAAqB,CAAC;AACvD,OAAO,KAAK,EAAC,aAAa,EAAE,cAAc,EAAC,MAAM,iBAAiB,CAAC;AAiBnE,qBAAa,sBAAuB,YAAW,cAAc;;IAU3D,YACE,EAAE,EAAE,UAAU,EACd,aAAa,EAAE,aAAa,EAC5B,kBAAkB,EAAE,wBAAwB,EAC5C,UAAU,EAAE,UAAU,EACtB,OAAO,EAAE,OAAO,GAAG,SAAS,EAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,EAkB3B;IAEK,aAAa,CACjB,GAAG,EAAE,gCAAgC,GACpC,OAAO,CAAC,aAAa,EAAE,CAAC,CA6N1B;CACF"}

package/out/zero-cache/src/workers/syncer-ws-message-handler.js

@@ -86,22 +86,27 @@ var SyncerWsMessageHandler = class {
 				}];
 				const auth = this.#connContextManager.mustGetConnectionContext(this.#connectionSelector).auth;
 				assert(auth?.type !== "opaque", "Only JWT auth is supported for CRUD mutations");
-				return [await this.#mutationLock.withLock(async () => {
-					const errors = [];
-					for (const mutation of mutations) {
-						const maybeError = await mutagen.processMutation(mutation, auth?.decoded, this.#pusher !== void 0);
-						if (maybeError !== void 0) errors.push({
-							kind: maybeError[0],
-							message: maybeError[1],
-							origin: ZeroCache
-						});
-					}
-					if (errors.length > 0) return {
-						type: "transient",
-						errors
-					};
-					return { type: "ok" };
-				})];
+				mutagen.ref();
+				try {
+					return [await this.#mutationLock.withLock(async () => {
+						const errors = [];
+						for (const mutation of mutations) {
+							const maybeError = await mutagen.processMutation(mutation, auth?.decoded, this.#pusher !== void 0);
+							if (maybeError !== void 0) errors.push({
+								kind: maybeError[0],
+								message: maybeError[1],
+								origin: ZeroCache
+							});
+						}
+						if (errors.length > 0) return {
+							type: "transient",
+							errors
+						};
+						return { type: "ok" };
+					})];
+				} finally {
+					mutagen.unref();
+				}
 			}));
 			case "changeDesiredQueries":
 				await withTraceparent(msg[1].traceparent, () => startAsyncSpan(tracer, "connection.changeDesiredQueries", () => viewSyncer.changeDesiredQueries(this.#connectionSelector, msg)));

package/out/zero-cache/src/workers/syncer-ws-message-handler.js.map

@@ -1 +1 @@
-{"version":3,"file":"syncer-ws-message-handler.js","names":["#viewSyncer","#mutagen","#mutationLock","#lc","#clientGroupID","#connectionSelector","#connContextManager","#pusher"],"sources":["../../../../../zero-cache/src/workers/syncer-ws-message-handler.ts"],"sourcesContent":["import {ROOT_CONTEXT, context, propagation, trace} from '@opentelemetry/api';\nimport {Lock} from '@rocicorp/lock';\nimport type {LogContext} from '@rocicorp/logger';\nimport {startAsyncSpan, startSpan} from '../../../otel/src/span.ts';\nimport {version} from '../../../otel/src/version.ts';\nimport {assert, unreachable} from '../../../shared/src/asserts.ts';\nimport {ErrorKind} from '../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../zero-protocol/src/error-origin.ts';\nimport type {ErrorBody} from '../../../zero-protocol/src/error.ts';\nimport type {UpstreamWithUnparsedAnalyzeQuery} from '../../../zero-protocol/src/up.ts';\nimport type {Mutagen} from '../services/mutagen/mutagen.ts';\nimport type {Pusher} from '../services/mutagen/pusher.ts';\nimport {\n  type ConnectionContextManager,\n  type ConnectionSelector,\n} from '../services/view-syncer/connection-context-manager.ts';\nimport {type ViewSyncer} from '../services/view-syncer/view-syncer.ts';\nimport type {ConnectParams} from './connect-params.ts';\nimport type {HandlerResult, MessageHandler} from './connection.ts';\n\nconst tracer = trace.getTracer('syncer-ws-server', version);\n\n/**\n * Wraps a function in an OTEL context extracted from a W3C traceparent header.\n * This enables distributed tracing from the client through zero-cache to\n * the user's API server.\n */\nfunction withTraceparent<T>(traceparent: string | undefined, fn: () => T): T {\n  if (!traceparent) {\n    return fn();\n  }\n  const extracted = propagation.extract(ROOT_CONTEXT, {traceparent});\n  return context.with(extracted, fn);\n}\n\nexport class SyncerWsMessageHandler implements MessageHandler {\n  readonly #viewSyncer: ViewSyncer;\n  readonly #mutagen: Mutagen | undefined;\n  readonly #mutationLock: Lock;\n  readonly #lc: LogContext;\n  readonly #clientGroupID: string;\n  readonly #connectionSelector: ConnectionSelector;\n  readonly #connContextManager: ConnectionContextManager;\n  readonly #pusher: Pusher | undefined;\n\n  constructor(\n    lc: LogContext,\n    connectParams: ConnectParams,\n    connContextManager: ConnectionContextManager,\n    viewSyncer: ViewSyncer,\n    mutagen: Mutagen | undefined,\n    pusher: Pusher | undefined,\n  ) {\n    const {clientGroupID, clientID, wsID} = connectParams;\n    this.#viewSyncer = viewSyncer;\n    this.#mutagen = mutagen;\n    this.#connContextManager = connContextManager;\n    this.#mutationLock = new Lock();\n    this.#lc = lc\n      .withContext('connection')\n      .withContext('clientID', clientID)\n      .withContext('clientGroupID', clientGroupID)\n      .withContext('wsID', wsID);\n    this.#clientGroupID = clientGroupID;\n    this.#pusher = pusher;\n    this.#connectionSelector = {\n      clientID,\n      wsID,\n    };\n  }\n\n  async handleMessage(\n    msg: UpstreamWithUnparsedAnalyzeQuery,\n  ): Promise<HandlerResult[]> {\n    const lc = this.#lc;\n    const msgType = msg[0];\n    const viewSyncer = this.#viewSyncer;\n    switch (msgType) {\n      case 'ping':\n        lc.error?.('Ping is not supported at this layer by Zero');\n        break;\n      case 'pull':\n        lc.error?.('Pull is not supported by Zero');\n        break;\n      case 'push': {\n        return withTraceparent(msg[1].traceparent, () =>\n          startAsyncSpan<HandlerResult[]>(\n            tracer,\n            'connection.push',\n            async () => {\n              const {clientGroupID, mutations} = msg[1];\n              if (clientGroupID !== this.#clientGroupID) {\n                return [\n                  {\n                    type: 'fatal',\n                    error: {\n                      kind: ErrorKind.InvalidPush,\n                      message:\n                        `clientGroupID in mutation \"${clientGroupID}\" does not match ` +\n                        `clientGroupID of connection \"${this.#clientGroupID}`,\n                      origin: ErrorOrigin.ZeroCache,\n                    },\n                  } satisfies HandlerResult,\n                ];\n              }\n\n              if (mutations.length === 0) {\n                return [\n                  {\n                    type: 'ok',\n                  },\n                ];\n              }\n\n              // The client only ever sends 1 mutation per push.\n              // #pusher will throw if it sees a CRUD mutation.\n              // #mutagen will throw if it see a custom mutation.\n              if (mutations[0].type === 'custom') {\n                if (!this.#pusher) {\n                  return [\n                    {\n                      type: 'fatal',\n                      error: {\n                        kind: ErrorKind.InvalidPush,\n                        message:\n                          'A ZERO_MUTATE_URL must be set in order to process custom mutations.',\n                        origin: ErrorOrigin.ZeroCache,\n                      },\n                    } satisfies HandlerResult,\n                  ];\n                }\n                return [\n                  this.#pusher.enqueuePush(this.#connectionSelector, msg[1]),\n                ];\n              }\n\n              const mutagen = this.#mutagen;\n              if (!mutagen) {\n                return [\n                  {\n                    type: 'fatal',\n                    error: {\n                      kind: ErrorKind.InvalidPush,\n                      message: `Support for legacy CRUD mutations is disabled`,\n                      origin: ErrorOrigin.ZeroCache,\n                    },\n                  } satisfies HandlerResult,\n                ];\n              }\n\n              const auth = this.#connContextManager.mustGetConnectionContext(\n                this.#connectionSelector,\n              ).auth;\n              assert(\n                auth?.type !== 'opaque',\n                'Only JWT auth is supported for CRUD mutations',\n              );\n\n              // Hold a connection-level lock while processing mutations so that:\n              // 1. Mutations are processed in the order in which they are received and\n              // 2. A single view syncer connection cannot hog multiple upstream connections.\n              const ret = await this.#mutationLock.withLock(async () => {\n                const errors: ErrorBody[] = [];\n                for (const mutation of mutations) {\n                  const maybeError = await mutagen.processMutation(\n                    mutation,\n                    auth?.decoded,\n                    this.#pusher !== undefined,\n                  );\n                  if (maybeError !== undefined) {\n                    errors.push({\n                      kind: maybeError[0],\n                      message: maybeError[1],\n                      origin: ErrorOrigin.ZeroCache,\n                    });\n                  }\n                }\n                if (errors.length > 0) {\n                  return {type: 'transient', errors} satisfies HandlerResult;\n                }\n                return {type: 'ok'} satisfies HandlerResult;\n              });\n              return [ret];\n            },\n          ),\n        );\n      }\n      case 'changeDesiredQueries':\n        await withTraceparent(msg[1].traceparent, () =>\n          startAsyncSpan(tracer, 'connection.changeDesiredQueries', () =>\n            viewSyncer.changeDesiredQueries(this.#connectionSelector, msg),\n          ),\n        );\n        break;\n      case 'updateAuth':\n        await startAsyncSpan(tracer, 'connection.updateAuth', async () => {\n          const initialConnCtx =\n            this.#connContextManager.mustGetConnectionContext(\n              this.#connectionSelector,\n            );\n          const updatedConnCtx = await this.#connContextManager.updateAuth(\n            this.#connectionSelector,\n            msg[1],\n          );\n          const authRevisionChanged =\n            updatedConnCtx.revision !== initialConnCtx.revision;\n\n          await viewSyncer.updateAuth(\n            this.#connectionSelector,\n            msg,\n            authRevisionChanged,\n          );\n        });\n        break;\n      case 'deleteClients': {\n        const deletedClientIDs = await startAsyncSpan(\n          tracer,\n          'connection.deleteClients',\n          () => viewSyncer.deleteClients(this.#connectionSelector, msg),\n        );\n        if (this.#pusher && deletedClientIDs.length > 0) {\n          await this.#pusher.deleteClientMutations(\n            this.#connectionSelector,\n            deletedClientIDs,\n          );\n        }\n        break;\n      }\n      case 'initConnection': {\n        this.#connContextManager.initConnection(\n          this.#connectionSelector,\n          msg[1],\n        );\n        return withTraceparent(msg[1].traceparent, () => {\n          const ret: HandlerResult[] = [\n            {\n              type: 'stream',\n              source: 'viewSyncer',\n              stream: startSpan(tracer, 'connection.initConnection', () =>\n                viewSyncer.initConnection(this.#connectionSelector, msg),\n              ),\n            },\n          ];\n\n          // Given we support both CRUD and Custom mutators,\n          // we do not initialize the `pusher` unless the user has opted\n          // into custom mutations. We detect that by checking\n          // if the pushURL has been set.\n          if (this.#pusher) {\n            ret.push({\n              type: 'stream',\n              source: 'pusher',\n              stream: this.#pusher.initConnection(this.#connectionSelector),\n            });\n          }\n\n          return ret;\n        });\n      }\n      case 'closeConnection':\n        // This message is deprecated and no longer used.\n        break;\n\n      case 'inspect':\n        await startAsyncSpan(tracer, 'connection.inspect', () =>\n          viewSyncer.inspect(this.#connectionSelector, msg),\n        );\n        break;\n\n      case 'ackMutationResponses':\n        if (this.#pusher) {\n          await this.#pusher.ackMutationResponses(\n            this.#connectionSelector,\n            msg[1],\n          );\n        }\n        break;\n\n      default:\n        unreachable(msgType);\n    }\n\n    return [{type: 'ok'}];\n  }\n}\n"],"mappings":";;;;;;;;;;AAoBA,IAAM,SAAS,MAAM,UAAU,oBAAoB,OAAO;;;;;;AAO1D,SAAS,gBAAmB,aAAiC,IAAgB;CAC3E,IAAI,CAAC,aACH,OAAO,GAAG;CAEZ,MAAM,YAAY,YAAY,QAAQ,cAAc,EAAC,YAAW,CAAC;CACjE,OAAO,QAAQ,KAAK,WAAW,EAAE;AACnC;AAEA,IAAa,yBAAb,MAA8D;CAC5D;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YACE,IACA,eACA,oBACA,YACA,SACA,QACA;EACA,MAAM,EAAC,eAAe,UAAU,SAAQ;EACxC,KAAKA,cAAc;EACnB,KAAKC,WAAW;EAChB,KAAKK,sBAAsB;EAC3B,KAAKJ,gBAAgB,IAAI,KAAK;EAC9B,KAAKC,MAAM,GACR,YAAY,YAAY,EACxB,YAAY,YAAY,QAAQ,EAChC,YAAY,iBAAiB,aAAa,EAC1C,YAAY,QAAQ,IAAI;EAC3B,KAAKC,iBAAiB;EACtB,KAAKG,UAAU;EACf,KAAKF,sBAAsB;GACzB;GACA;EACF;CACF;CAEA,MAAM,cACJ,KAC0B;EAC1B,MAAM,KAAK,KAAKF;EAChB,MAAM,UAAU,IAAI;EACpB,MAAM,aAAa,KAAKH;EACxB,QAAQ,SAAR;GACE,KAAK;IACH,GAAG,QAAQ,6CAA6C;IACxD;GACF,KAAK;IACH,GAAG,QAAQ,+BAA+B;IAC1C;GACF,KAAK,QACH,OAAO,gBAAgB,IAAI,GAAG,mBAC5B,eACE,QACA,mBACA,YAAY;IACV,MAAM,EAAC,eAAe,cAAa,IAAI;IACvC,IAAI,kBAAkB,KAAKI,gBACzB,OAAO,CACL;KACE,MAAM;KACN,OAAO;MACL,MAAM;MACN,SACE,8BAA8B,cAAc,gDACZ,KAAKA;MACvC,QAAQ;KACV;IACF,CACF;IAGF,IAAI,UAAU,WAAW,GACvB,OAAO,CACL,EACE,MAAM,KACR,CACF;IAMF,IAAI,UAAU,GAAG,SAAS,UAAU;KAClC,IAAI,CAAC,KAAKG,SACR,OAAO,CACL;MACE,MAAM;MACN,OAAO;OACL,MAAM;OACN,SACE;OACF,QAAQ;MACV;KACF,CACF;KAEF,OAAO,CACL,KAAKA,QAAQ,YAAY,KAAKF,qBAAqB,IAAI,EAAE,CAC3D;IACF;IAEA,MAAM,UAAU,KAAKJ;IACrB,IAAI,CAAC,SACH,OAAO,CACL;KACE,MAAM;KACN,OAAO;MACL,MAAM;MACN,SAAS;MACT,QAAQ;KACV;IACF,CACF;IAGF,MAAM,OAAO,KAAKK,oBAAoB,yBACpC,KAAKD,mBACP,EAAE;IACF,OACE,MAAM,SAAS,UACf,+CACF;IA0BA,OAAO,CAAC,MArBU,KAAKH,cAAc,SAAS,YAAY;KACxD,MAAM,SAAsB,CAAC;KAC7B,KAAK,MAAM,YAAY,WAAW;MAChC,MAAM,aAAa,MAAM,QAAQ,gBAC/B,UACA,MAAM,SACN,KAAKK,YAAY,KAAA,CACnB;MACA,IAAI,eAAe,KAAA,GACjB,OAAO,KAAK;OACV,MAAM,WAAW;OACjB,SAAS,WAAW;OACpB,QAAQ;MACV,CAAC;KAEL;KACA,IAAI,OAAO,SAAS,GAClB,OAAO;MAAC,MAAM;MAAa;KAAM;KAEnC,OAAO,EAAC,MAAM,KAAI;IACpB,CAAC,CACU;GACb,CACF,CACF;GAEF,KAAK;IACH,MAAM,gBAAgB,IAAI,GAAG,mBAC3B,eAAe,QAAQ,yCACrB,WAAW,qBAAqB,KAAKF,qBAAqB,GAAG,CAC/D,CACF;IACA;GACF,KAAK;IACH,MAAM,eAAe,QAAQ,yBAAyB,YAAY;KAChE,MAAM,iBACJ,KAAKC,oBAAoB,yBACvB,KAAKD,mBACP;KAKF,MAAM,uBACJ,MAL2B,KAAKC,oBAAoB,WACpD,KAAKD,qBACL,IAAI,EACN,GAEiB,aAAa,eAAe;KAE7C,MAAM,WAAW,WACf,KAAKA,qBACL,KACA,mBACF;IACF,CAAC;IACD;GACF,KAAK,iBAAiB;IACpB,MAAM,mBAAmB,MAAM,eAC7B,QACA,kCACM,WAAW,cAAc,KAAKA,qBAAqB,GAAG,CAC9D;IACA,IAAI,KAAKE,WAAW,iBAAiB,SAAS,GAC5C,MAAM,KAAKA,QAAQ,sBACjB,KAAKF,qBACL,gBACF;IAEF;GACF;GACA,KAAK;IACH,KAAKC,oBAAoB,eACvB,KAAKD,qBACL,IAAI,EACN;IACA,OAAO,gBAAgB,IAAI,GAAG,mBAAmB;KAC/C,MAAM,MAAuB,CAC3B;MACE,MAAM;MACN,QAAQ;MACR,QAAQ,UAAU,QAAQ,mCACxB,WAAW,eAAe,KAAKA,qBAAqB,GAAG,CACzD;KACF,CACF;KAMA,IAAI,KAAKE,SACP,IAAI,KAAK;MACP,MAAM;MACN,QAAQ;MACR,QAAQ,KAAKA,QAAQ,eAAe,KAAKF,mBAAmB;KAC9D,CAAC;KAGH,OAAO;IACT,CAAC;GAEH,KAAK,mBAEH;GAEF,KAAK;IACH,MAAM,eAAe,QAAQ,4BAC3B,WAAW,QAAQ,KAAKA,qBAAqB,GAAG,CAClD;IACA;GAEF,KAAK;IACH,IAAI,KAAKE,SACP,MAAM,KAAKA,QAAQ,qBACjB,KAAKF,qBACL,IAAI,EACN;IAEF;GAEF,SACE,YAAY,OAAO;EACvB;EAEA,OAAO,CAAC,EAAC,MAAM,KAAI,CAAC;CACtB;AACF"}
+{"version":3,"file":"syncer-ws-message-handler.js","names":["#viewSyncer","#mutagen","#mutationLock","#lc","#clientGroupID","#connectionSelector","#connContextManager","#pusher"],"sources":["../../../../../zero-cache/src/workers/syncer-ws-message-handler.ts"],"sourcesContent":["import {ROOT_CONTEXT, context, propagation, trace} from '@opentelemetry/api';\nimport {Lock} from '@rocicorp/lock';\nimport type {LogContext} from '@rocicorp/logger';\nimport {startAsyncSpan, startSpan} from '../../../otel/src/span.ts';\nimport {version} from '../../../otel/src/version.ts';\nimport {assert, unreachable} from '../../../shared/src/asserts.ts';\nimport {ErrorKind} from '../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../zero-protocol/src/error-origin.ts';\nimport type {ErrorBody} from '../../../zero-protocol/src/error.ts';\nimport type {UpstreamWithUnparsedAnalyzeQuery} from '../../../zero-protocol/src/up.ts';\nimport type {Mutagen} from '../services/mutagen/mutagen.ts';\nimport type {Pusher} from '../services/mutagen/pusher.ts';\nimport {\n  type ConnectionContextManager,\n  type ConnectionSelector,\n} from '../services/view-syncer/connection-context-manager.ts';\nimport {type ViewSyncer} from '../services/view-syncer/view-syncer.ts';\nimport type {ConnectParams} from './connect-params.ts';\nimport type {HandlerResult, MessageHandler} from './connection.ts';\n\nconst tracer = trace.getTracer('syncer-ws-server', version);\n\n/**\n * Wraps a function in an OTEL context extracted from a W3C traceparent header.\n * This enables distributed tracing from the client through zero-cache to\n * the user's API server.\n */\nfunction withTraceparent<T>(traceparent: string | undefined, fn: () => T): T {\n  if (!traceparent) {\n    return fn();\n  }\n  const extracted = propagation.extract(ROOT_CONTEXT, {traceparent});\n  return context.with(extracted, fn);\n}\n\nexport class SyncerWsMessageHandler implements MessageHandler {\n  readonly #viewSyncer: ViewSyncer;\n  readonly #mutagen: Mutagen | undefined;\n  readonly #mutationLock: Lock;\n  readonly #lc: LogContext;\n  readonly #clientGroupID: string;\n  readonly #connectionSelector: ConnectionSelector;\n  readonly #connContextManager: ConnectionContextManager;\n  readonly #pusher: Pusher | undefined;\n\n  constructor(\n    lc: LogContext,\n    connectParams: ConnectParams,\n    connContextManager: ConnectionContextManager,\n    viewSyncer: ViewSyncer,\n    mutagen: Mutagen | undefined,\n    pusher: Pusher | undefined,\n  ) {\n    const {clientGroupID, clientID, wsID} = connectParams;\n    this.#viewSyncer = viewSyncer;\n    this.#mutagen = mutagen;\n    this.#connContextManager = connContextManager;\n    this.#mutationLock = new Lock();\n    this.#lc = lc\n      .withContext('connection')\n      .withContext('clientID', clientID)\n      .withContext('clientGroupID', clientGroupID)\n      .withContext('wsID', wsID);\n    this.#clientGroupID = clientGroupID;\n    this.#pusher = pusher;\n    this.#connectionSelector = {\n      clientID,\n      wsID,\n    };\n  }\n\n  async handleMessage(\n    msg: UpstreamWithUnparsedAnalyzeQuery,\n  ): Promise<HandlerResult[]> {\n    const lc = this.#lc;\n    const msgType = msg[0];\n    const viewSyncer = this.#viewSyncer;\n    switch (msgType) {\n      case 'ping':\n        lc.error?.('Ping is not supported at this layer by Zero');\n        break;\n      case 'pull':\n        lc.error?.('Pull is not supported by Zero');\n        break;\n      case 'push': {\n        return withTraceparent(msg[1].traceparent, () =>\n          startAsyncSpan<HandlerResult[]>(\n            tracer,\n            'connection.push',\n            async () => {\n              const {clientGroupID, mutations} = msg[1];\n              if (clientGroupID !== this.#clientGroupID) {\n                return [\n                  {\n                    type: 'fatal',\n                    error: {\n                      kind: ErrorKind.InvalidPush,\n                      message:\n                        `clientGroupID in mutation \"${clientGroupID}\" does not match ` +\n                        `clientGroupID of connection \"${this.#clientGroupID}`,\n                      origin: ErrorOrigin.ZeroCache,\n                    },\n                  } satisfies HandlerResult,\n                ];\n              }\n\n              if (mutations.length === 0) {\n                return [\n                  {\n                    type: 'ok',\n                  },\n                ];\n              }\n\n              // The client only ever sends 1 mutation per push.\n              // #pusher will throw if it sees a CRUD mutation.\n              // #mutagen will throw if it see a custom mutation.\n              if (mutations[0].type === 'custom') {\n                if (!this.#pusher) {\n                  return [\n                    {\n                      type: 'fatal',\n                      error: {\n                        kind: ErrorKind.InvalidPush,\n                        message:\n                          'A ZERO_MUTATE_URL must be set in order to process custom mutations.',\n                        origin: ErrorOrigin.ZeroCache,\n                      },\n                    } satisfies HandlerResult,\n                  ];\n                }\n                return [\n                  this.#pusher.enqueuePush(this.#connectionSelector, msg[1]),\n                ];\n              }\n\n              const mutagen = this.#mutagen;\n              if (!mutagen) {\n                return [\n                  {\n                    type: 'fatal',\n                    error: {\n                      kind: ErrorKind.InvalidPush,\n                      message: `Support for legacy CRUD mutations is disabled`,\n                      origin: ErrorOrigin.ZeroCache,\n                    },\n                  } satisfies HandlerResult,\n                ];\n              }\n\n              const auth = this.#connContextManager.mustGetConnectionContext(\n                this.#connectionSelector,\n              ).auth;\n              assert(\n                auth?.type !== 'opaque',\n                'Only JWT auth is supported for CRUD mutations',\n              );\n\n              // Hold a ref on the mutagen for the duration of the push so that\n              // the connection closing mid-push does not stop the service\n              // (closing its replica handle) between two of its mutations.\n              // (The connection holds a ref while open, and messages are not\n              // dispatched after it has closed, so the service is running\n              // here.)\n              mutagen.ref();\n              try {\n                // Hold a connection-level lock while processing mutations so that:\n                // 1. Mutations are processed in the order in which they are received and\n                // 2. A single view syncer connection cannot hog multiple upstream connections.\n                const ret = await this.#mutationLock.withLock(async () => {\n                  const errors: ErrorBody[] = [];\n                  for (const mutation of mutations) {\n                    const maybeError = await mutagen.processMutation(\n                      mutation,\n                      auth?.decoded,\n                      this.#pusher !== undefined,\n                    );\n                    if (maybeError !== undefined) {\n                      errors.push({\n                        kind: maybeError[0],\n                        message: maybeError[1],\n                        origin: ErrorOrigin.ZeroCache,\n                      });\n                    }\n                  }\n                  if (errors.length > 0) {\n                    return {type: 'transient', errors} satisfies HandlerResult;\n                  }\n                  return {type: 'ok'} satisfies HandlerResult;\n                });\n                return [ret];\n              } finally {\n                mutagen.unref();\n              }\n            },\n          ),\n        );\n      }\n      case 'changeDesiredQueries':\n        await withTraceparent(msg[1].traceparent, () =>\n          startAsyncSpan(tracer, 'connection.changeDesiredQueries', () =>\n            viewSyncer.changeDesiredQueries(this.#connectionSelector, msg),\n          ),\n        );\n        break;\n      case 'updateAuth':\n        await startAsyncSpan(tracer, 'connection.updateAuth', async () => {\n          const initialConnCtx =\n            this.#connContextManager.mustGetConnectionContext(\n              this.#connectionSelector,\n            );\n          const updatedConnCtx = await this.#connContextManager.updateAuth(\n            this.#connectionSelector,\n            msg[1],\n          );\n          const authRevisionChanged =\n            updatedConnCtx.revision !== initialConnCtx.revision;\n\n          await viewSyncer.updateAuth(\n            this.#connectionSelector,\n            msg,\n            authRevisionChanged,\n          );\n        });\n        break;\n      case 'deleteClients': {\n        const deletedClientIDs = await startAsyncSpan(\n          tracer,\n          'connection.deleteClients',\n          () => viewSyncer.deleteClients(this.#connectionSelector, msg),\n        );\n        if (this.#pusher && deletedClientIDs.length > 0) {\n          await this.#pusher.deleteClientMutations(\n            this.#connectionSelector,\n            deletedClientIDs,\n          );\n        }\n        break;\n      }\n      case 'initConnection': {\n        this.#connContextManager.initConnection(\n          this.#connectionSelector,\n          msg[1],\n        );\n        return withTraceparent(msg[1].traceparent, () => {\n          const ret: HandlerResult[] = [\n            {\n              type: 'stream',\n              source: 'viewSyncer',\n              stream: startSpan(tracer, 'connection.initConnection', () =>\n                viewSyncer.initConnection(this.#connectionSelector, msg),\n              ),\n            },\n          ];\n\n          // Given we support both CRUD and Custom mutators,\n          // we do not initialize the `pusher` unless the user has opted\n          // into custom mutations. We detect that by checking\n          // if the pushURL has been set.\n          if (this.#pusher) {\n            ret.push({\n              type: 'stream',\n              source: 'pusher',\n              stream: this.#pusher.initConnection(this.#connectionSelector),\n            });\n          }\n\n          return ret;\n        });\n      }\n      case 'closeConnection':\n        // This message is deprecated and no longer used.\n        break;\n\n      case 'inspect':\n        await startAsyncSpan(tracer, 'connection.inspect', () =>\n          viewSyncer.inspect(this.#connectionSelector, msg),\n        );\n        break;\n\n      case 'ackMutationResponses':\n        if (this.#pusher) {\n          await this.#pusher.ackMutationResponses(\n            this.#connectionSelector,\n            msg[1],\n          );\n        }\n        break;\n\n      default:\n        unreachable(msgType);\n    }\n\n    return [{type: 'ok'}];\n  }\n}\n"],"mappings":";;;;;;;;;;AAoBA,IAAM,SAAS,MAAM,UAAU,oBAAoB,OAAO;;;;;;AAO1D,SAAS,gBAAmB,aAAiC,IAAgB;CAC3E,IAAI,CAAC,aACH,OAAO,GAAG;CAEZ,MAAM,YAAY,YAAY,QAAQ,cAAc,EAAC,YAAW,CAAC;CACjE,OAAO,QAAQ,KAAK,WAAW,EAAE;AACnC;AAEA,IAAa,yBAAb,MAA8D;CAC5D;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YACE,IACA,eACA,oBACA,YACA,SACA,QACA;EACA,MAAM,EAAC,eAAe,UAAU,SAAQ;EACxC,KAAKA,cAAc;EACnB,KAAKC,WAAW;EAChB,KAAKK,sBAAsB;EAC3B,KAAKJ,gBAAgB,IAAI,KAAK;EAC9B,KAAKC,MAAM,GACR,YAAY,YAAY,EACxB,YAAY,YAAY,QAAQ,EAChC,YAAY,iBAAiB,aAAa,EAC1C,YAAY,QAAQ,IAAI;EAC3B,KAAKC,iBAAiB;EACtB,KAAKG,UAAU;EACf,KAAKF,sBAAsB;GACzB;GACA;EACF;CACF;CAEA,MAAM,cACJ,KAC0B;EAC1B,MAAM,KAAK,KAAKF;EAChB,MAAM,UAAU,IAAI;EACpB,MAAM,aAAa,KAAKH;EACxB,QAAQ,SAAR;GACE,KAAK;IACH,GAAG,QAAQ,6CAA6C;IACxD;GACF,KAAK;IACH,GAAG,QAAQ,+BAA+B;IAC1C;GACF,KAAK,QACH,OAAO,gBAAgB,IAAI,GAAG,mBAC5B,eACE,QACA,mBACA,YAAY;IACV,MAAM,EAAC,eAAe,cAAa,IAAI;IACvC,IAAI,kBAAkB,KAAKI,gBACzB,OAAO,CACL;KACE,MAAM;KACN,OAAO;MACL,MAAM;MACN,SACE,8BAA8B,cAAc,gDACZ,KAAKA;MACvC,QAAQ;KACV;IACF,CACF;IAGF,IAAI,UAAU,WAAW,GACvB,OAAO,CACL,EACE,MAAM,KACR,CACF;IAMF,IAAI,UAAU,GAAG,SAAS,UAAU;KAClC,IAAI,CAAC,KAAKG,SACR,OAAO,CACL;MACE,MAAM;MACN,OAAO;OACL,MAAM;OACN,SACE;OACF,QAAQ;MACV;KACF,CACF;KAEF,OAAO,CACL,KAAKA,QAAQ,YAAY,KAAKF,qBAAqB,IAAI,EAAE,CAC3D;IACF;IAEA,MAAM,UAAU,KAAKJ;IACrB,IAAI,CAAC,SACH,OAAO,CACL;KACE,MAAM;KACN,OAAO;MACL,MAAM;MACN,SAAS;MACT,QAAQ;KACV;IACF,CACF;IAGF,MAAM,OAAO,KAAKK,oBAAoB,yBACpC,KAAKD,mBACP,EAAE;IACF,OACE,MAAM,SAAS,UACf,+CACF;IAQA,QAAQ,IAAI;IACZ,IAAI;KAyBF,OAAO,CAAC,MArBU,KAAKH,cAAc,SAAS,YAAY;MACxD,MAAM,SAAsB,CAAC;MAC7B,KAAK,MAAM,YAAY,WAAW;OAChC,MAAM,aAAa,MAAM,QAAQ,gBAC/B,UACA,MAAM,SACN,KAAKK,YAAY,KAAA,CACnB;OACA,IAAI,eAAe,KAAA,GACjB,OAAO,KAAK;QACV,MAAM,WAAW;QACjB,SAAS,WAAW;QACpB,QAAQ;OACV,CAAC;MAEL;MACA,IAAI,OAAO,SAAS,GAClB,OAAO;OAAC,MAAM;OAAa;MAAM;MAEnC,OAAO,EAAC,MAAM,KAAI;KACpB,CAAC,CACU;IACb,UAAU;KACR,QAAQ,MAAM;IAChB;GACF,CACF,CACF;GAEF,KAAK;IACH,MAAM,gBAAgB,IAAI,GAAG,mBAC3B,eAAe,QAAQ,yCACrB,WAAW,qBAAqB,KAAKF,qBAAqB,GAAG,CAC/D,CACF;IACA;GACF,KAAK;IACH,MAAM,eAAe,QAAQ,yBAAyB,YAAY;KAChE,MAAM,iBACJ,KAAKC,oBAAoB,yBACvB,KAAKD,mBACP;KAKF,MAAM,uBACJ,MAL2B,KAAKC,oBAAoB,WACpD,KAAKD,qBACL,IAAI,EACN,GAEiB,aAAa,eAAe;KAE7C,MAAM,WAAW,WACf,KAAKA,qBACL,KACA,mBACF;IACF,CAAC;IACD;GACF,KAAK,iBAAiB;IACpB,MAAM,mBAAmB,MAAM,eAC7B,QACA,kCACM,WAAW,cAAc,KAAKA,qBAAqB,GAAG,CAC9D;IACA,IAAI,KAAKE,WAAW,iBAAiB,SAAS,GAC5C,MAAM,KAAKA,QAAQ,sBACjB,KAAKF,qBACL,gBACF;IAEF;GACF;GACA,KAAK;IACH,KAAKC,oBAAoB,eACvB,KAAKD,qBACL,IAAI,EACN;IACA,OAAO,gBAAgB,IAAI,GAAG,mBAAmB;KAC/C,MAAM,MAAuB,CAC3B;MACE,MAAM;MACN,QAAQ;MACR,QAAQ,UAAU,QAAQ,mCACxB,WAAW,eAAe,KAAKA,qBAAqB,GAAG,CACzD;KACF,CACF;KAMA,IAAI,KAAKE,SACP,IAAI,KAAK;MACP,MAAM;MACN,QAAQ;MACR,QAAQ,KAAKA,QAAQ,eAAe,KAAKF,mBAAmB;KAC9D,CAAC;KAGH,OAAO;IACT,CAAC;GAEH,KAAK,mBAEH;GAEF,KAAK;IACH,MAAM,eAAe,QAAQ,4BAC3B,WAAW,QAAQ,KAAKA,qBAAqB,GAAG,CAClD;IACA;GAEF,KAAK;IACH,IAAI,KAAKE,SACP,MAAM,KAAKA,QAAQ,qBACjB,KAAKF,qBACL,IAAI,EACN;IAEF;GAEF,SACE,YAAY,OAAO;EACvB;EAEA,OAAO,CAAC,EAAC,MAAM,KAAI,CAAC;CACtB;AACF"}

package/out/zero-cache/src/workers/syncer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"syncer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/workers/syncer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,qBAAqB,CAAC;AACrD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAUjD,OAAO,EAAyB,KAAK,iBAAiB,EAAC,MAAM,iBAAiB,CAAC;AAE/E,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,0BAA0B,CAAC;AAYzD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,gCAAgC,CAAC;AAC5D,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,+BAA+B,CAAC;AAC1D,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,sCAAsC,CAAC;AAEvE,OAAO,KAAK,EACV,oBAAoB,EACpB,OAAO,EACP,gBAAgB,EACjB,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAC,wBAAwB,EAAC,MAAM,uDAAuD,CAAC;AACpG,OAAO,EAAC,gBAAgB,EAAC,MAAM,8CAA8C,CAAC;AAC9E,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,wCAAwC,CAAC;AACvE,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,uBAAuB,CAAC;AAClD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAO3D,MAAM,MAAM,gBAAgB,GAAG;IAC7B,cAAc,EAAE,WAAW,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACrC,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG,IAAI,CACrC,UAAU,EACV,aAAa,GAAG,eAAe,GAAG,oBAAoB,CACvD,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB,CAAC;AAMF,MAAM,MAAM,uBAAuB,GAAG,IAAI,CACxC,UAAU,EACV,gBAAgB,GAAG,iBAAiB,CACrC,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAG/B,QAAQ,CAAC,YAAY,EAAE,WAAW,CAChC,MAAM,EACN;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,GAAG,SAAS,CAAA;KAAC,CAC/C,CAAC;IACF,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;CAChC,CAAC;AAEF,wBAAgB,yBAAyB,CACvC,WAAW,EAAE,QAAQ,CAAC,uBAAuB,CAAC,GAC7C,kBAAkB,CAkDpB;AAED,eAAO,MAAM,wBAAwB,QAAS,CAAC;AAuJ/C,wBAAgB,wBAAwB,CACtC,GAAG,EAAE,MAAM,EACX,kBAAkB,EAAE,iBAAiB,EAAE,EACvC,WAAW,EAAE,QAAQ,CAAC,oBAAoB,CAAC,GAC1C,eAAe,CAejB;AAED,wBAAgB,sBAAsB,CACpC,GAAG,EAAE,MAAM,EACX,kBAAkB,EAAE,iBAAiB,EAAE,EACvC,WAAW,EAAE,QAAQ,CAAC,oBAAoB,CAAC,GAC1C,MAAM,CAER;AA4BD;;;;;;GAMG;AACH,qBAAa,MAAO,YAAW,gBAAgB;;IAC7C,QAAQ,CAAC,EAAE,SAAmB;IAiD9B,YACE,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,EAClB,iBAAiB,EAAE,CACjB,EAAE,EAAE,MAAM,EACV,GAAG,EAAE,YAAY,CAAC,YAAY,CAAC,EAC/B,gBAAgB,EAAE,gBAAgB,KAC/B,UAAU,GAAG,oBAAoB,EACtC,cAAc,EAAE,CAAC,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,GAAG,OAAO,CAAC,GAAG,SAAS,EAC/D,aAAa,EACT,CAAC,CACC,EAAE,EAAE,MAAM,EACV,kBAAkB,EAAE,wBAAwB,KACzC,MAAM,GAAG,OAAO,CAAC,GACtB,SAAS,EACb,MAAM,EAAE,MAAM,EACd,iBAAiB,EAAE,iBAAiB,GAAG,SAAS,EAoJjD;IAyRD,GAAG,kBAEF;IAED;;;;;OAKG;IACG,KAAK,kBAmBV;IAED,IAAI,kBAMH;CACF"}
+{"version":3,"file":"syncer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/workers/syncer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,qBAAqB,CAAC;AACrD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAUjD,OAAO,EAAyB,KAAK,iBAAiB,EAAC,MAAM,iBAAiB,CAAC;AAE/E,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,0BAA0B,CAAC;AAYzD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,gCAAgC,CAAC;AAC5D,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,+BAA+B,CAAC;AAC1D,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,sCAAsC,CAAC;AAEvE,OAAO,KAAK,EACV,oBAAoB,EACpB,OAAO,EACP,gBAAgB,EACjB,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAC,wBAAwB,EAAC,MAAM,uDAAuD,CAAC;AACpG,OAAO,EAAC,gBAAgB,EAAC,MAAM,8CAA8C,CAAC;AAC9E,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,wCAAwC,CAAC;AACvE,OAAO,KAAK,EAA2B,MAAM,EAAC,MAAM,uBAAuB,CAAC;AAC5E,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAO3D,MAAM,MAAM,gBAAgB,GAAG;IAC7B,cAAc,EAAE,WAAW,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACrC,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG,IAAI,CACrC,UAAU,EACV,aAAa,GAAG,eAAe,GAAG,oBAAoB,CACvD,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB,CAAC;AAMF,MAAM,MAAM,uBAAuB,GAAG,IAAI,CACxC,UAAU,EACV,gBAAgB,GAAG,iBAAiB,CACrC,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAG/B,QAAQ,CAAC,YAAY,EAAE,WAAW,CAChC,MAAM,EACN;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,GAAG,SAAS,CAAA;KAAC,CAC/C,CAAC;IACF,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;CAChC,CAAC;AAEF,wBAAgB,yBAAyB,CACvC,WAAW,EAAE,QAAQ,CAAC,uBAAuB,CAAC,GAC7C,kBAAkB,CAkDpB;AAED,eAAO,MAAM,wBAAwB,QAAS,CAAC;AAuJ/C,wBAAgB,wBAAwB,CACtC,GAAG,EAAE,MAAM,EACX,kBAAkB,EAAE,iBAAiB,EAAE,EACvC,WAAW,EAAE,QAAQ,CAAC,oBAAoB,CAAC,GAC1C,eAAe,CAejB;AAED,wBAAgB,sBAAsB,CACpC,GAAG,EAAE,MAAM,EACX,kBAAkB,EAAE,iBAAiB,EAAE,EACvC,WAAW,EAAE,QAAQ,CAAC,oBAAoB,CAAC,GAC1C,MAAM,CAER;AA4BD;;;;;;GAMG;AACH,qBAAa,MAAO,YAAW,gBAAgB;;IAC7C,QAAQ,CAAC,EAAE,SAAmB;IAmD9B,YACE,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,EAClB,iBAAiB,EAAE,CACjB,EAAE,EAAE,MAAM,EACV,GAAG,EAAE,YAAY,CAAC,YAAY,CAAC,EAC/B,gBAAgB,EAAE,gBAAgB,KAC/B,UAAU,GAAG,oBAAoB,EACtC,cAAc,EAAE,CAAC,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,GAAG,OAAO,CAAC,GAAG,SAAS,EAC/D,aAAa,EACT,CAAC,CACC,EAAE,EAAE,MAAM,EACV,kBAAkB,EAAE,wBAAwB,KACzC,MAAM,GAAG,OAAO,CAAC,GACtB,SAAS,EACb,MAAM,EAAE,MAAM,EACd,iBAAiB,EAAE,iBAAiB,GAAG,SAAS,EAsLjD;IAyUD,GAAG,kBAEF;IAED;;;;;OAKG;IACG,KAAK,kBAmBV;IAED,IAAI,kBAMH;CACF"}

package/out/zero-cache/src/workers/syncer.js

@@ -14,7 +14,7 @@ import { DrainCoordinator } from "../services/view-syncer/drain-coordinator.js";
 import { Connection, sendError } from "./connection.js";
 import { SyncerWsMessageHandler } from "./syncer-ws-message-handler.js";
 import { resolver } from "@rocicorp/resolver";
-import { WebSocketServer } from "ws";
+import { WebSocket, WebSocketServer } from "ws";
 import { pid } from "node:process";
 //#region ../zero-cache/src/workers/syncer.ts
 function computePipelineDedupStats(viewSyncers) {
@@ -153,6 +153,8 @@ var Syncer = class {
 	#mutagens;
 	#pushers;
 	#connections = /* @__PURE__ */ new Map();
+	#pendingConnections = /* @__PURE__ */ new Map();
+	#clientGroupGenerations = /* @__PURE__ */ new Map();
 	#drainCoordinator = new DrainCoordinator();
 	#parent;
 	#wss;
@@ -178,12 +180,30 @@ var Syncer = class {
 		const notifier = createNotifierFrom(lc, parent, (state) => this.#recordReplicaReadyState(state));
 		subscribeTo(lc, parent);
 		this.#lc = lc;
-		this.#viewSyncers = new ServiceRunner(lc, (id) => viewSyncerFactory(id, notifier.subscribe(), this.#drainCoordinator), (v) => v.keepalive());
+		this.#parent = parent;
+		this.#viewSyncers = new ServiceRunner(lc, (id) => viewSyncerFactory(id, notifier.subscribe(), this.#drainCoordinator), (v) => v.keepalive(), (id) => {
+			if ((this.#pendingConnections.get(id) ?? 0) === 0) {
+				const generation = this.#clientGroupGenerations.get(id);
+				this.#parent.send(["clientGroupStatus", {
+					clientGroupID: id,
+					active: false,
+					...generation !== void 0 ? { generation } : {}
+				}]);
+				this.#clientGroupGenerations.delete(id);
+			}
+		});
 		if (mutagenFactory) this.#mutagens = new ServiceRunner(lc, mutagenFactory, (m) => m.hasRefs());
 		if (pusherFactory) this.#pushers = new ServiceRunner(lc, (id) => pusherFactory(id, this.#viewSyncers.getService(id).connContextManager), (p) => p.hasRefs());
-		this.#parent = parent;
 		this.#wss = new WebSocketServer(getWebSocketServerOptions(config));
-		installWebSocketReceiver(lc, this.#wss, this.#createConnection, this.#parent);
+		installWebSocketReceiver(lc, this.#wss, this.#receiveConnection, this.#parent, (params) => {
+			this.#releasePending(params.clientGroupID, params.generation);
+		}, (params) => {
+			this.#pendingConnections.set(params.clientGroupID, (this.#pendingConnections.get(params.clientGroupID) ?? 0) + 1);
+			if (params.generation !== void 0) {
+				const current = this.#clientGroupGenerations.get(params.clientGroupID);
+				if (current === void 0 || params.generation > current) this.#clientGroupGenerations.set(params.clientGroupID, params.generation);
+			}
+		});
 		setActiveClientGroupsGetter(() => this.#viewSyncers.size);
 		getOrCreateGauge("sync", "active-client-groups", "Number of active client groups").addCallback((result) => result.observe(this.#viewSyncers.size));
 		getOrCreateGauge("sync", "queries", "Active queries (pipelines) across all client groups").addCallback((result) => {
@@ -341,6 +361,15 @@ var Syncer = class {
 			recordConnectionFailure("internal");
 			throw e;
 		}
+		if (ws.readyState !== WebSocket.OPEN) {
+			this.#lc.debug?.("websocket closed while resolving auth", {
+				clientGroupID,
+				clientID,
+				readyState: ws.readyState
+			});
+			recordConnectionFailure("closed_during_auth");
+			return;
+		}
 		const viewSyncer = this.#viewSyncers.getService(clientGroupID);
 		const connContextManager = viewSyncer.connContextManager;
 		const group = connContextManager.getGroupState();
@@ -399,6 +428,28 @@ var Syncer = class {
 			await connection.handleInitConnection(JSON.stringify(params.initConnectionMsg));
 		}
 	};
+	#receiveConnection = async (ws, params) => {
+		try {
+			await this.#createConnection(ws, params);
+		} finally {
+			this.#releasePending(params.clientGroupID, params.generation);
+		}
+	};
+	#releasePending(clientGroupID, generation) {
+		const remaining = (this.#pendingConnections.get(clientGroupID) ?? 1) - 1;
+		if (remaining <= 0) {
+			this.#pendingConnections.delete(clientGroupID);
+			if (!this.#viewSyncers.hasService(clientGroupID)) {
+				const gen = this.#clientGroupGenerations.get(clientGroupID) ?? generation;
+				this.#parent.send(["clientGroupStatus", {
+					clientGroupID,
+					active: false,
+					...gen !== void 0 ? { generation: gen } : {}
+				}]);
+				this.#clientGroupGenerations.delete(clientGroupID);
+			}
+		} else this.#pendingConnections.set(clientGroupID, remaining);
+	}
 	run() {
 		return this.#stopped.promise;
 	}

package/out/zero-cache/src/workers/syncer.js.map

@@ -1 +1 @@
-{"version":3,"file":"syncer.js","names":["#lc","#viewSyncers","#mutagens","#pushers","#connections","#drainCoordinator","#parent","#wss","#stopped","#config","#validateLegacyJWT","#replicaReadyStates","#webSocketOpenConnections","#webSocketConnectionAttempts","#webSocketConnectionSuccesses","#webSocketConnectionFailures","#viewSyncerLag","#recordReplicaReadyState","#createConnection","#computeServingLagDistribution","#computePipelineDedupStats","#viewSyncerLagSampleInterval","#recordViewSyncerLagSamples","#eligibilityLogInterval","#logSharedAdvanceEligibility","#servingLagDistributionCache","#servingLagDistributionCacheClearQueued"],"sources":["../../../../../zero-cache/src/workers/syncer.ts"],"sourcesContent":["import {pid} from 'node:process';\nimport type {MessagePort} from 'node:worker_threads';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {WebSocketServer, type ServerOptions, type WebSocket} from 'ws';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {ErrorKind} from '../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../zero-protocol/src/error-origin.ts';\nimport {\n  isProtocolError,\n  ProtocolError,\n} from '../../../zero-protocol/src/error.ts';\nimport {resolveAuth, type Auth, type ValidateLegacyJWT} from '../auth/auth.ts';\nimport {tokenConfigOptions} from '../auth/jwt.ts';\nimport {type ZeroConfig} from '../config/zero-config.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateGauge,\n  getOrCreateNativeHistogram,\n  getOrCreateUpDownCounter,\n} from '../observability/metrics.ts';\nimport {\n  recordConnectionAttempted,\n  recordConnectionSuccess,\n  setActiveClientGroupsGetter,\n} from '../server/anonymous-otel-start.ts';\nimport type {Mutagen} from '../services/mutagen/mutagen.ts';\nimport type {Pusher} from '../services/mutagen/pusher.ts';\nimport type {ReplicaState} from '../services/replicator/replicator.ts';\nimport {ServiceRunner} from '../services/runner.ts';\nimport type {\n  ActivityBasedService,\n  Service,\n  SingletonService,\n} from '../services/service.ts';\nimport type {ConnectionContextManager} from '../services/view-syncer/connection-context-manager.ts';\nimport {DrainCoordinator} from '../services/view-syncer/drain-coordinator.ts';\nimport type {ViewSyncer} from '../services/view-syncer/view-syncer.ts';\nimport type {Worker} from '../types/processes.ts';\nimport type {Subscription} from '../types/subscription.ts';\nimport {installWebSocketReceiver} from '../types/websocket-handoff.ts';\nimport type {ConnectParams} from './connect-params.ts';\nimport {Connection, sendError} from './connection.ts';\nimport {createNotifierFrom, subscribeTo} from './replicator.ts';\nimport {SyncerWsMessageHandler} from './syncer-ws-message-handler.ts';\n\nexport type SyncerWorkerData = {\n  replicatorPort: MessagePort;\n};\n\nexport type ReplicaReadyState = {\n  readonly watermark: string;\n  readonly replicaReadyTimeMs: number;\n};\n\nexport type ServingLagViewSyncer = Pick<\n  ViewSyncer,\n  'createdAtMs' | 'servedVersion' | 'servingLagEligible'\n>;\n\nexport type ServingLagStats = {\n  readonly activeClientGroups: number;\n  readonly laggingClientGroups: number;\n  readonly minMs: number;\n  readonly p50Ms: number;\n  readonly p75Ms: number;\n  readonly p99Ms: number;\n  readonly maxMs: number;\n};\n\ntype ServingLagDistribution = ServingLagStats & {\n  readonly lagsMs: readonly number[];\n};\n\nexport type PipelineDedupViewSyncer = Pick<\n  ViewSyncer,\n  'pipelineHashes' | 'clientSchemaKey'\n>;\n\nexport type PipelineDedupStats = {\n  readonly clientTotal: number;\n  readonly internalTotal: number;\n  // (clientSchemaKey, transformationHash) -> occurrence count across client\n  // groups, for client/custom queries.\n  readonly clientHashes: ReadonlyMap<\n    string,\n    {count: number; queryName: string | undefined}\n  >;\n  readonly internalUnique: number;\n  readonly clientSchemas: number;\n};\n\nexport function computePipelineDedupStats(\n  viewSyncers: Iterable<PipelineDedupViewSyncer>,\n): PipelineDedupStats {\n  let clientTotal = 0;\n  let internalTotal = 0;\n  const clientHashes = new Map<\n    string,\n    {count: number; queryName: string | undefined}\n  >();\n  const internalHashes = new Set<string>();\n  const schemaKeys = new Set<string>();\n  let vsIndex = 0;\n  for (const vs of viewSyncers) {\n    const schemaKey = vs.clientSchemaKey;\n    if (schemaKey !== undefined) {\n      schemaKeys.add(schemaKey);\n    }\n    // Without a client schema the pipeline's dedup identity is unknown, so key\n    // it uniquely per view-syncer; otherwise schema-less pipelines from\n    // unrelated client groups would collapse under a shared `undefined/<hash>`\n    // key and overstate deduplication.\n    const dedupSchemaKey = schemaKey ?? `#vs${vsIndex}`;\n    vsIndex++;\n    for (const {\n      transformationHash,\n      internal,\n      queryName,\n    } of vs.pipelineHashes()) {\n      // Pipelines only share when both the client schema (which determines\n      // row keys) and the transformed AST are identical.\n      const key = `${dedupSchemaKey}/${transformationHash}`;\n      if (internal) {\n        internalTotal++;\n        internalHashes.add(key);\n      } else {\n        clientTotal++;\n        const entry = clientHashes.get(key);\n        if (entry) {\n          entry.count++;\n        } else {\n          clientHashes.set(key, {count: 1, queryName});\n        }\n      }\n    }\n  }\n  return {\n    clientTotal,\n    internalTotal,\n    clientHashes,\n    internalUnique: internalHashes.size,\n    clientSchemas: schemaKeys.size,\n  };\n}\n\nexport const MAX_REPLICA_READY_STATES = 10_000;\nconst VIEW_SYNCER_LAG_SAMPLE_INTERVAL_MS = 60_000;\nconst SHARED_ADVANCE_ELIGIBILITY_LOG_INTERVAL_MS = 5 * 60_000;\n\nconst PROTOCOL_VERSION_ATTRIBUTE = 'protocol.version';\nconst FAILURE_REASON_ATTRIBUTE = 'reason';\n\nfunction boundReplicaReadyStates(\n  replicaReadyStates: ReplicaReadyState[],\n): void {\n  if (replicaReadyStates.length > MAX_REPLICA_READY_STATES) {\n    replicaReadyStates.splice(\n      0,\n      replicaReadyStates.length - MAX_REPLICA_READY_STATES,\n    );\n  }\n}\n\nfunction pruneReplicaReadyStates(\n  replicaReadyStates: ReplicaReadyState[],\n  firstNeededIndex: number,\n): void {\n  if (firstNeededIndex > 0) {\n    replicaReadyStates.splice(0, firstNeededIndex);\n  }\n\n  boundReplicaReadyStates(replicaReadyStates);\n}\n\nfunction lowerBoundReplicaReadyTimeMs(\n  replicaReadyStates: readonly ReplicaReadyState[],\n  replicaReadyTimeMs: number,\n): number {\n  let low = 0;\n  let high = replicaReadyStates.length;\n  while (low < high) {\n    const mid = low + Math.floor((high - low) / 2);\n    if (replicaReadyStates[mid].replicaReadyTimeMs < replicaReadyTimeMs) {\n      low = mid + 1;\n    } else {\n      high = mid;\n    }\n  }\n  return low;\n}\n\nfunction upperBoundWatermark(\n  replicaReadyStates: readonly ReplicaReadyState[],\n  watermark: string,\n): number {\n  let low = 0;\n  let high = replicaReadyStates.length;\n  while (low < high) {\n    const mid = low + Math.floor((high - low) / 2);\n    if (replicaReadyStates[mid].watermark <= watermark) {\n      low = mid + 1;\n    } else {\n      high = mid;\n    }\n  }\n  return low;\n}\n\nfunction findFirstUnservedIndex(\n  replicaReadyStates: readonly ReplicaReadyState[],\n  viewSyncer: ServingLagViewSyncer,\n): number {\n  const firstReadyAfterCreation = lowerBoundReplicaReadyTimeMs(\n    replicaReadyStates,\n    viewSyncer.createdAtMs,\n  );\n  const firstAfterServedVersion =\n    viewSyncer.servedVersion === null\n      ? 0\n      : upperBoundWatermark(replicaReadyStates, viewSyncer.servedVersion);\n\n  const firstUnservedIndex = Math.max(\n    firstReadyAfterCreation,\n    firstAfterServedVersion,\n  );\n  return firstUnservedIndex < replicaReadyStates.length\n    ? firstUnservedIndex\n    : -1;\n}\n\nfunction percentileNearestRank(\n  sortedValues: readonly number[],\n  percentile: number,\n): number {\n  if (sortedValues.length === 0) {\n    return 0;\n  }\n  const index = Math.min(\n    sortedValues.length - 1,\n    Math.max(0, Math.ceil((percentile / 100) * sortedValues.length) - 1),\n  );\n  return sortedValues[index];\n}\n\nfunction computeServingLagDistributionMs(\n  now: number,\n  replicaReadyStates: ReplicaReadyState[],\n  viewSyncers: Iterable<ServingLagViewSyncer>,\n): ServingLagDistribution {\n  const lags: number[] = [];\n  let laggingClientGroups = 0;\n  let firstNeededIndex = replicaReadyStates.length;\n\n  for (const viewSyncer of viewSyncers) {\n    if (!viewSyncer.servingLagEligible) {\n      continue;\n    }\n\n    const firstUnservedIndex = findFirstUnservedIndex(\n      replicaReadyStates,\n      viewSyncer,\n    );\n    if (firstUnservedIndex === -1) {\n      lags.push(0);\n      continue;\n    }\n\n    firstNeededIndex = Math.min(firstNeededIndex, firstUnservedIndex);\n    const lagMs = Math.max(\n      0,\n      now - replicaReadyStates[firstUnservedIndex].replicaReadyTimeMs,\n    );\n    lags.push(lagMs);\n    if (lagMs > 0) {\n      laggingClientGroups++;\n    }\n  }\n\n  pruneReplicaReadyStates(replicaReadyStates, firstNeededIndex);\n\n  lags.sort((a, b) => a - b);\n  return {\n    activeClientGroups: lags.length,\n    laggingClientGroups,\n    // The percentile fields are for the legacy serving_lag_stats gauge.\n    // The native view_syncer_lag histogram records lagsMs and computes\n    // percentiles in Prometheus/AMP.\n    minMs: lags[0] ?? 0,\n    p50Ms: percentileNearestRank(lags, 50),\n    p75Ms: percentileNearestRank(lags, 75),\n    p99Ms: percentileNearestRank(lags, 99),\n    maxMs: lags.at(-1) ?? 0,\n    lagsMs: lags,\n  };\n}\n\nexport function computeServingLagStatsMs(\n  now: number,\n  replicaReadyStates: ReplicaReadyState[],\n  viewSyncers: Iterable<ServingLagViewSyncer>,\n): ServingLagStats {\n  const distribution = computeServingLagDistributionMs(\n    now,\n    replicaReadyStates,\n    viewSyncers,\n  );\n  return {\n    activeClientGroups: distribution.activeClientGroups,\n    laggingClientGroups: distribution.laggingClientGroups,\n    minMs: distribution.minMs,\n    p50Ms: distribution.p50Ms,\n    p75Ms: distribution.p75Ms,\n    p99Ms: distribution.p99Ms,\n    maxMs: distribution.maxMs,\n  };\n}\n\nexport function computeMaxServingLagMs(\n  now: number,\n  replicaReadyStates: ReplicaReadyState[],\n  viewSyncers: Iterable<ServingLagViewSyncer>,\n): number {\n  return computeServingLagStatsMs(now, replicaReadyStates, viewSyncers).maxMs;\n}\n\nfunction getWebSocketServerOptions(config: ZeroConfig): ServerOptions {\n  const options: ServerOptions = {\n    noServer: true,\n    maxPayload: config.websocketMaxPayloadBytes,\n  };\n\n  if (config.websocketCompression) {\n    options.perMessageDeflate = true;\n\n    if (config.websocketCompressionOptions) {\n      try {\n        const compressionOptions = JSON.parse(\n          config.websocketCompressionOptions,\n        );\n        options.perMessageDeflate = compressionOptions;\n      } catch (e) {\n        throw new Error(\n          `Failed to parse ZERO_WEBSOCKET_COMPRESSION_OPTIONS: ${String(e)}. Expected valid JSON.`,\n        );\n      }\n    }\n  }\n\n  return options;\n}\n\n/**\n * The Syncer worker receives websocket handoffs for \"/sync\" connections\n * from the Dispatcher in the main thread, and creates websocket\n * {@link Connection}s with a corresponding {@link ViewSyncer}, {@link Mutagen},\n * and {@link Subscription} to version notifications from the Replicator\n * worker.\n */\nexport class Syncer implements SingletonService {\n  readonly id = `syncer-${pid}`;\n  readonly #lc: LogContext;\n  readonly #viewSyncers: ServiceRunner<ViewSyncer & ActivityBasedService>;\n  readonly #mutagens: ServiceRunner<Mutagen & Service> | undefined;\n  readonly #pushers: ServiceRunner<Pusher & Service> | undefined;\n  readonly #connections = new Map<string, Connection>();\n  readonly #drainCoordinator = new DrainCoordinator();\n  readonly #parent: Worker;\n  readonly #wss: WebSocketServer;\n  readonly #stopped = resolver();\n  readonly #config: ZeroConfig;\n  readonly #validateLegacyJWT: ValidateLegacyJWT | undefined;\n  readonly #replicaReadyStates: ReplicaReadyState[] = [];\n  readonly #webSocketOpenConnections = getOrCreateUpDownCounter(\n    'sync',\n    'websocket.open_connections',\n    'Open client WebSocket connections.',\n  );\n  readonly #webSocketConnectionAttempts = getOrCreateCounter(\n    'sync',\n    'websocket.connection_attempts',\n    'Client WebSocket connection attempts.',\n  );\n  readonly #webSocketConnectionSuccesses = getOrCreateCounter(\n    'sync',\n    'websocket.connection_successes',\n    'Client WebSocket connections successfully initialized.',\n  );\n  readonly #webSocketConnectionFailures = getOrCreateCounter(\n    'sync',\n    'websocket.connection_failures',\n    'Client WebSocket connection attempts that failed before initialization.',\n  );\n  readonly #viewSyncerLag = getOrCreateNativeHistogram(\n    'sync',\n    'view_syncer_lag',\n    {\n      description:\n        'Lag from replica-ready change to ViewSyncer output for active ' +\n        'client groups. A change is output after IVM advancement, CVR flush, ' +\n        'and pokeEnd.',\n      unit: 's',\n    },\n  );\n  #servingLagDistributionCache: ServingLagDistribution | undefined;\n  #servingLagDistributionCacheClearQueued = false;\n  #viewSyncerLagSampleInterval: ReturnType<typeof setInterval> | undefined;\n  #eligibilityLogInterval: ReturnType<typeof setInterval> | undefined;\n\n  constructor(\n    lc: LogContext,\n    config: ZeroConfig,\n    viewSyncerFactory: (\n      id: string,\n      sub: Subscription<ReplicaState>,\n      drainCoordinator: DrainCoordinator,\n    ) => ViewSyncer & ActivityBasedService,\n    mutagenFactory: ((id: string) => Mutagen & Service) | undefined,\n    pusherFactory:\n      | ((\n          id: string,\n          connContextManager: ConnectionContextManager,\n        ) => Pusher & Service)\n      | undefined,\n    parent: Worker,\n    validateLegacyJWT: ValidateLegacyJWT | undefined,\n  ) {\n    this.#config = config;\n    this.#validateLegacyJWT = validateLegacyJWT;\n    // Relays notifications from the parent thread subscription\n    // to ViewSyncers within this thread.\n    const notifier = createNotifierFrom(lc, parent, state =>\n      this.#recordReplicaReadyState(state),\n    );\n    subscribeTo(lc, parent);\n\n    this.#lc = lc;\n    this.#viewSyncers = new ServiceRunner(\n      lc,\n      id => viewSyncerFactory(id, notifier.subscribe(), this.#drainCoordinator),\n      v => v.keepalive(),\n    );\n    if (mutagenFactory) {\n      this.#mutagens = new ServiceRunner(lc, mutagenFactory, m => m.hasRefs());\n    }\n    if (pusherFactory) {\n      this.#pushers = new ServiceRunner(\n        lc,\n        id =>\n          pusherFactory(\n            id,\n            this.#viewSyncers.getService(id).connContextManager,\n          ),\n        p => p.hasRefs(),\n      );\n    }\n    this.#parent = parent;\n    this.#wss = new WebSocketServer(getWebSocketServerOptions(config));\n\n    installWebSocketReceiver(\n      lc,\n      this.#wss,\n      this.#createConnection,\n      this.#parent,\n    );\n\n    setActiveClientGroupsGetter(() => this.#viewSyncers.size);\n\n    getOrCreateGauge(\n      'sync',\n      'active-client-groups',\n      'Number of active client groups',\n    ).addCallback(result => result.observe(this.#viewSyncers.size));\n\n    getOrCreateGauge(\n      'sync',\n      'queries',\n      'Active queries (pipelines) across all client groups',\n    ).addCallback(result => {\n      let total = 0;\n      for (const vs of this.#viewSyncers.getServices()) {\n        total += vs.queryCount;\n      }\n      result.observe(total);\n    });\n\n    getOrCreateGauge(\n      'sync',\n      'rows',\n      'Tracked rows across all client groups',\n    ).addCallback(result => {\n      let total = 0;\n      for (const vs of this.#viewSyncers.getServices()) {\n        total += vs.rowCount;\n      }\n      result.observe(total);\n    });\n\n    getOrCreateGauge('sync', 'serving_lag', {\n      description:\n        'Maximum time active ViewSyncer client groups have had unserved ' +\n        'replica changes. A change is served after IVM advancement, CVR flush, ' +\n        'and pokeEnd.',\n      unit: 'millisecond',\n    }).addCallback(result => {\n      result.observe(this.#computeServingLagDistribution().maxMs);\n    });\n\n    getOrCreateGauge('sync', 'serving_lag_stats', {\n      description:\n        'Distribution of time active ViewSyncer client groups have had ' +\n        'unserved replica changes. A change is served after IVM advancement, ' +\n        'CVR flush, and pokeEnd.',\n      unit: 'millisecond',\n    }).addCallback(result => {\n      const stats = this.#computeServingLagDistribution();\n      result.observe(stats.minMs, {stat: 'min'});\n      result.observe(stats.p50Ms, {stat: 'p50'});\n      result.observe(stats.p75Ms, {stat: 'p75'});\n      result.observe(stats.p99Ms, {stat: 'p99'});\n      result.observe(stats.maxMs, {stat: 'max'});\n    });\n\n    getOrCreateGauge(\n      'sync',\n      'serving_lagging_client_groups',\n      'Number of active client groups with unserved replica changes',\n    ).addCallback(result => {\n      result.observe(this.#computeServingLagDistribution().laggingClientGroups);\n    });\n\n    getOrCreateGauge(\n      'sync',\n      'pipelines_total',\n      'Query pipelines across all client groups on this worker, by query ' +\n        'type. The ratio to sync.pipelines_unique is the dedup factor ' +\n        'available to shared pipeline advancement.',\n    ).addCallback(result => {\n      const stats = this.#computePipelineDedupStats();\n      result.observe(stats.clientTotal, {type: 'client'});\n      result.observe(stats.internalTotal, {type: 'internal'});\n    });\n\n    getOrCreateGauge(\n      'sync',\n      'pipelines_unique',\n      'Distinct (clientSchema, transformationHash) pipelines across all ' +\n        'client groups on this worker, by query type',\n    ).addCallback(result => {\n      const stats = this.#computePipelineDedupStats();\n      result.observe(stats.clientHashes.size, {type: 'client'});\n      result.observe(stats.internalUnique, {type: 'internal'});\n    });\n\n    getOrCreateGauge(\n      'sync',\n      'client_schemas',\n      'Distinct client schemas across all client groups on this worker',\n    ).addCallback(result => {\n      result.observe(this.#computePipelineDedupStats().clientSchemas);\n    });\n\n    this.#viewSyncerLagSampleInterval = setInterval(\n      () => this.#recordViewSyncerLagSamples(),\n      VIEW_SYNCER_LAG_SAMPLE_INTERVAL_MS,\n    );\n    this.#viewSyncerLagSampleInterval.unref?.();\n\n    this.#eligibilityLogInterval = setInterval(\n      () => this.#logSharedAdvanceEligibility(),\n      SHARED_ADVANCE_ELIGIBILITY_LOG_INTERVAL_MS,\n    );\n    this.#eligibilityLogInterval.unref?.();\n  }\n\n  #computeServingLagDistribution(): ServingLagDistribution {\n    if (this.#servingLagDistributionCache) {\n      return this.#servingLagDistributionCache;\n    }\n\n    const distribution = computeServingLagDistributionMs(\n      Date.now(),\n      this.#replicaReadyStates,\n      this.#viewSyncers.getServices(),\n    );\n    this.#servingLagDistributionCache = distribution;\n    if (!this.#servingLagDistributionCacheClearQueued) {\n      this.#servingLagDistributionCacheClearQueued = true;\n      queueMicrotask(() => {\n        this.#servingLagDistributionCache = undefined;\n        this.#servingLagDistributionCacheClearQueued = false;\n      });\n    }\n    return distribution;\n  }\n\n  #recordViewSyncerLagSamples(): void {\n    const {lagsMs} = this.#computeServingLagDistribution();\n    for (const lagMs of lagsMs) {\n      this.#viewSyncerLag.recordMs(lagMs);\n    }\n  }\n\n  #computePipelineDedupStats(): PipelineDedupStats {\n    return computePipelineDedupStats(this.#viewSyncers.getServices());\n  }\n\n  #logSharedAdvanceEligibility(): void {\n    const stats = this.#computePipelineDedupStats();\n    if (stats.clientTotal === 0) {\n      return;\n    }\n    const unique = stats.clientHashes.size;\n    const topDuplicated = [...stats.clientHashes.entries()]\n      .filter(([, {count}]) => count > 1)\n      .sort((a, b) => b[1].count - a[1].count)\n      .slice(0, 10)\n      .map(([key, {count, queryName}]) => ({key, count, queryName}));\n    this.#lc.info?.('shared-advance-eligibility', {\n      clientPipelines: stats.clientTotal,\n      uniqueClientPipelines: unique,\n      dedupFactor: Number((stats.clientTotal / Math.max(unique, 1)).toFixed(2)),\n      internalPipelines: stats.internalTotal,\n      clientSchemas: stats.clientSchemas,\n      topDuplicated,\n    });\n  }\n\n  #recordReplicaReadyState(state: ReplicaState): void {\n    if (\n      state.watermark === undefined ||\n      state.replicaReadyTimeMs === undefined\n    ) {\n      return;\n    }\n    const last = this.#replicaReadyStates.at(-1);\n    if (last && last.watermark >= state.watermark) {\n      return;\n    }\n    this.#replicaReadyStates.push({\n      watermark: state.watermark,\n      replicaReadyTimeMs: state.replicaReadyTimeMs,\n    });\n    if (this.#viewSyncers.size === 0) {\n      this.#replicaReadyStates.length = 0;\n      return;\n    }\n    boundReplicaReadyStates(this.#replicaReadyStates);\n  }\n\n  readonly #createConnection = async (ws: WebSocket, params: ConnectParams) => {\n    const metricAttributes = {\n      [PROTOCOL_VERSION_ATTRIBUTE]: params.protocolVersion,\n    };\n    let connectionResultRecorded = false;\n    const recordConnectionFailure = (reason: string) => {\n      if (connectionResultRecorded) {\n        return;\n      }\n      connectionResultRecorded = true;\n      this.#webSocketConnectionFailures.add(1, {\n        ...metricAttributes,\n        [FAILURE_REASON_ATTRIBUTE]: reason,\n      });\n    };\n    const recordConnectionSuccessMetric = () => {\n      if (connectionResultRecorded) {\n        return;\n      }\n      connectionResultRecorded = true;\n      this.#webSocketConnectionSuccesses.add(1, metricAttributes);\n    };\n\n    this.#webSocketConnectionAttempts.add(1, metricAttributes);\n    this.#webSocketOpenConnections.add(1, metricAttributes);\n    ws.once('close', () => {\n      this.#webSocketOpenConnections.add(-1, metricAttributes);\n    });\n\n    this.#lc.debug?.(\n      'creating connection',\n      params.clientGroupID,\n      params.clientID,\n    );\n    recordConnectionAttempted();\n    const {clientID, clientGroupID, auth, userID} = params;\n    const hasProvidedAuth = auth !== undefined && auth !== '';\n    const incomingUserID = userID ?? null;\n\n    if (hasProvidedAuth) {\n      const tokenOptions = tokenConfigOptions(this.#config.auth ?? {});\n\n      const hasPushOrMutate =\n        this.#config?.push?.url !== undefined ||\n        this.#config?.mutate?.url !== undefined;\n      const hasQueries =\n        this.#config?.query?.url !== undefined ||\n        this.#config?.getQueries?.url !== undefined;\n\n      // must either have one of the token options set or have custom mutations & queries enabled\n      const hasExactlyOneTokenOption = tokenOptions.length === 1;\n      const hasCustomEndpoints = hasPushOrMutate && hasQueries;\n      if (!hasExactlyOneTokenOption && !hasCustomEndpoints) {\n        recordConnectionFailure('configuration');\n        throw new Error(\n          'Exactly one of jwk, secret, or jwksUrl must be set in order to verify tokens but actually the following were set: ' +\n            JSON.stringify(tokenOptions) +\n            '. You may also set both ZERO_MUTATE_URL and ZERO_QUERY_URL to enable custom mutations and queries without passing token verification options.',\n        );\n      }\n    }\n\n    let initialAuth: Auth | undefined;\n\n    // Verify JWT BEFORE touching existing connections - prevents unauthenticated\n    // attackers from force-disconnecting legitimate users via DoS.\n    try {\n      initialAuth = await resolveAuth(\n        this.#lc\n          .withContext('clientGroupID', clientGroupID)\n          .withContext('clientID', clientID),\n        // no previous auth, since this is a new connection, and resolveAuth is\n        // connection scoped, not client group scoped\n        undefined,\n        incomingUserID,\n        auth,\n        this.#validateLegacyJWT,\n      );\n    } catch (e) {\n      if (isProtocolError(e)) {\n        this.#lc.warn?.(\n          'Rejecting sync connection during initial auth resolution',\n          {\n            clientGroupID,\n            clientID,\n            incomingUserID,\n            hasProvidedAuth,\n            errorKind: e.message,\n          },\n        );\n        recordConnectionFailure('auth');\n        sendError(this.#lc, ws, e.errorBody);\n        ws.close(3000, e.errorBody.message);\n        return;\n      }\n      recordConnectionFailure('internal');\n      throw e;\n    }\n\n    const viewSyncer = this.#viewSyncers.getService(clientGroupID);\n    const connContextManager = viewSyncer.connContextManager;\n    const group = connContextManager.getGroupState();\n\n    // TODO(0xcadams): we only check for user ID mismatch here if the group is\n    // already validated. This prevents wrong-user reconnects from evicting a\n    // healthy connection, but it does not protect against same-user reconnects\n    // with an invalid opaque token. The long-term fix is to keep the replacement\n    // connection pending until its auth is fully validated, and only then replace\n    // the existing socket.\n    if (\n      group.pinnedUser !== undefined &&\n      group.pinnedUser.id !== incomingUserID\n    ) {\n      const error = new ProtocolError({\n        kind: ErrorKind.Unauthorized,\n        message:\n          'Client groups are pinned to a single userID. Connection userID does not match existing client group userID.',\n        origin: ErrorOrigin.ZeroCache,\n      });\n      recordConnectionFailure('user_mismatch');\n      sendError(this.#lc, ws, error.errorBody);\n      ws.close(3000, error.message);\n      return;\n    }\n\n    // Check for and close existing connections AFTER auth is validated\n    const existing = this.#connections.get(clientID);\n    if (existing) {\n      this.#lc.debug?.(\n        `client ${clientID} already connected, closing existing connection`,\n      );\n      existing.close(`replaced by ${params.wsID}`);\n    }\n\n    connContextManager.registerConnection(\n      {clientID, wsID: params.wsID},\n      params,\n      initialAuth,\n    );\n\n    const mutagen = this.#mutagens?.getService(clientGroupID);\n    const pusher = this.#pushers?.getService(clientGroupID);\n    // a new connection is using the mutagen and pusher. Bump their ref counts.\n    mutagen?.ref();\n    pusher?.ref();\n\n    let connection: Connection;\n    try {\n      connection = new Connection(\n        this.#lc,\n        params,\n        ws,\n        this.#config.allowLegacyQueries,\n        new SyncerWsMessageHandler(\n          this.#lc,\n          params,\n          connContextManager,\n          viewSyncer,\n          mutagen,\n          pusher,\n        ),\n        () => {\n          connContextManager.closeConnection({\n            clientID,\n            wsID: params.wsID,\n          });\n          if (this.#connections.get(clientID) === connection) {\n            this.#connections.delete(clientID);\n          }\n          // Connection is closed. We can unref the mutagen and pusher.\n          // If their ref counts are zero, they will stop themselves and set themselves invalid.\n          mutagen?.unref();\n          pusher?.unref();\n        },\n      );\n    } catch (e) {\n      recordConnectionFailure('internal');\n      connContextManager.closeConnection({clientID, wsID: params.wsID});\n      mutagen?.unref();\n      pusher?.unref();\n      throw e;\n    }\n\n    this.#connections.set(clientID, connection);\n\n    if (connection.init()) {\n      recordConnectionSuccessMetric();\n      recordConnectionSuccess();\n    } else {\n      recordConnectionFailure('protocol_version');\n    }\n\n    if (params.initConnectionMsg) {\n      this.#lc.debug?.(\n        'handling init connection message from sec header',\n        params.clientGroupID,\n        params.clientID,\n      );\n      await connection.handleInitConnection(\n        JSON.stringify(params.initConnectionMsg),\n      );\n    }\n  };\n\n  run() {\n    return this.#stopped.promise;\n  }\n\n  /**\n   * Graceful shutdown involves shutting down view syncers one at a time, pausing\n   * for the duration of view syncer's hydration between each one. This paces the\n   * disconnects to avoid creating a backlog of hydrations in the receiving server\n   * when the clients reconnect.\n   */\n  async drain() {\n    const start = Date.now();\n    this.#lc.info?.(`draining ${this.#viewSyncers.size} view-syncers`);\n\n    this.#drainCoordinator.drainNextIn(0);\n\n    while (this.#viewSyncers.size) {\n      await this.#drainCoordinator.forceDrainTimeout;\n\n      // Pick an arbitrary view syncer to force drain.\n      for (const vs of this.#viewSyncers.getServices()) {\n        this.#lc.debug?.(`draining view-syncer ${vs.id} (forced)`);\n        // When this drain or an elective drain completes, the forceDrainTimeout will\n        // resolve after the next drain interval.\n        void vs.stop();\n        break;\n      }\n    }\n    this.#lc.info?.(`finished draining (${Date.now() - start} ms)`);\n  }\n\n  stop() {\n    clearInterval(this.#viewSyncerLagSampleInterval);\n    clearInterval(this.#eligibilityLogInterval);\n    this.#wss.close();\n    this.#stopped.resolve();\n    return promiseVoid;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;AA4FA,SAAgB,0BACd,aACoB;CACpB,IAAI,cAAc;CAClB,IAAI,gBAAgB;CACpB,MAAM,+BAAe,IAAI,IAGvB;CACF,MAAM,iCAAiB,IAAI,IAAY;CACvC,MAAM,6BAAa,IAAI,IAAY;CACnC,IAAI,UAAU;CACd,KAAK,MAAM,MAAM,aAAa;EAC5B,MAAM,YAAY,GAAG;EACrB,IAAI,cAAc,KAAA,GAChB,WAAW,IAAI,SAAS;EAM1B,MAAM,iBAAiB,aAAa,MAAM;EAC1C;EACA,KAAK,MAAM,EACT,oBACA,UACA,eACG,GAAG,eAAe,GAAG;GAGxB,MAAM,MAAM,GAAG,eAAe,GAAG;GACjC,IAAI,UAAU;IACZ;IACA,eAAe,IAAI,GAAG;GACxB,OAAO;IACL;IACA,MAAM,QAAQ,aAAa,IAAI,GAAG;IAClC,IAAI,OACF,MAAM;SAEN,aAAa,IAAI,KAAK;KAAC,OAAO;KAAG;IAAS,CAAC;GAE/C;EACF;CACF;CACA,OAAO;EACL;EACA;EACA;EACA,gBAAgB,eAAe;EAC/B,eAAe,WAAW;CAC5B;AACF;AAEA,IAAa,2BAA2B;AACxC,IAAM,qCAAqC;AAC3C,IAAM,6CAA6C,IAAI;AAEvD,IAAM,6BAA6B;AACnC,IAAM,2BAA2B;AAEjC,SAAS,wBACP,oBACM;CACN,IAAI,mBAAmB,SAAA,KACrB,mBAAmB,OACjB,GACA,mBAAmB,SAAS,wBAC9B;AAEJ;AAEA,SAAS,wBACP,oBACA,kBACM;CACN,IAAI,mBAAmB,GACrB,mBAAmB,OAAO,GAAG,gBAAgB;CAG/C,wBAAwB,kBAAkB;AAC5C;AAEA,SAAS,6BACP,oBACA,oBACQ;CACR,IAAI,MAAM;CACV,IAAI,OAAO,mBAAmB;CAC9B,OAAO,MAAM,MAAM;EACjB,MAAM,MAAM,MAAM,KAAK,OAAO,OAAO,OAAO,CAAC;EAC7C,IAAI,mBAAmB,KAAK,qBAAqB,oBAC/C,MAAM,MAAM;OAEZ,OAAO;CAEX;CACA,OAAO;AACT;AAEA,SAAS,oBACP,oBACA,WACQ;CACR,IAAI,MAAM;CACV,IAAI,OAAO,mBAAmB;CAC9B,OAAO,MAAM,MAAM;EACjB,MAAM,MAAM,MAAM,KAAK,OAAO,OAAO,OAAO,CAAC;EAC7C,IAAI,mBAAmB,KAAK,aAAa,WACvC,MAAM,MAAM;OAEZ,OAAO;CAEX;CACA,OAAO;AACT;AAEA,SAAS,uBACP,oBACA,YACQ;CACR,MAAM,0BAA0B,6BAC9B,oBACA,WAAW,WACb;CACA,MAAM,0BACJ,WAAW,kBAAkB,OACzB,IACA,oBAAoB,oBAAoB,WAAW,aAAa;CAEtE,MAAM,qBAAqB,KAAK,IAC9B,yBACA,uBACF;CACA,OAAO,qBAAqB,mBAAmB,SAC3C,qBACA;AACN;AAEA,SAAS,sBACP,cACA,YACQ;CACR,IAAI,aAAa,WAAW,GAC1B,OAAO;CAMT,OAAO,aAJO,KAAK,IACjB,aAAa,SAAS,GACtB,KAAK,IAAI,GAAG,KAAK,KAAM,aAAa,MAAO,aAAa,MAAM,IAAI,CAAC,CAEjD;AACtB;AAEA,SAAS,gCACP,KACA,oBACA,aACwB;CACxB,MAAM,OAAiB,CAAC;CACxB,IAAI,sBAAsB;CAC1B,IAAI,mBAAmB,mBAAmB;CAE1C,KAAK,MAAM,cAAc,aAAa;EACpC,IAAI,CAAC,WAAW,oBACd;EAGF,MAAM,qBAAqB,uBACzB,oBACA,UACF;EACA,IAAI,uBAAuB,IAAI;GAC7B,KAAK,KAAK,CAAC;GACX;EACF;EAEA,mBAAmB,KAAK,IAAI,kBAAkB,kBAAkB;EAChE,MAAM,QAAQ,KAAK,IACjB,GACA,MAAM,mBAAmB,oBAAoB,kBAC/C;EACA,KAAK,KAAK,KAAK;EACf,IAAI,QAAQ,GACV;CAEJ;CAEA,wBAAwB,oBAAoB,gBAAgB;CAE5D,KAAK,MAAM,GAAG,MAAM,IAAI,CAAC;CACzB,OAAO;EACL,oBAAoB,KAAK;EACzB;EAIA,OAAO,KAAK,MAAM;EAClB,OAAO,sBAAsB,MAAM,EAAE;EACrC,OAAO,sBAAsB,MAAM,EAAE;EACrC,OAAO,sBAAsB,MAAM,EAAE;EACrC,OAAO,KAAK,GAAG,EAAE,KAAK;EACtB,QAAQ;CACV;AACF;AA+BA,SAAS,0BAA0B,QAAmC;CACpE,MAAM,UAAyB;EAC7B,UAAU;EACV,YAAY,OAAO;CACrB;CAEA,IAAI,OAAO,sBAAsB;EAC/B,QAAQ,oBAAoB;EAE5B,IAAI,OAAO,6BACT,IAAI;GAIF,QAAQ,oBAHmB,KAAK,MAC9B,OAAO,2BAEmB;EAC9B,SAAS,GAAG;GACV,MAAM,IAAI,MACR,uDAAuD,OAAO,CAAC,EAAE,uBACnE;EACF;CAEJ;CAEA,OAAO;AACT;;;;;;;;AASA,IAAa,SAAb,MAAgD;CAC9C,KAAc,UAAU;CACxB;CACA;CACA;CACA;CACA,+BAAwB,IAAI,IAAwB;CACpD,oBAA6B,IAAI,iBAAiB;CAClD;CACA;CACA,WAAoB,SAAS;CAC7B;CACA;CACA,sBAAoD,CAAC;CACrD,4BAAqC,yBACnC,QACA,8BACA,oCACF;CACA,+BAAwC,mBACtC,QACA,iCACA,uCACF;CACA,gCAAyC,mBACvC,QACA,kCACA,wDACF;CACA,+BAAwC,mBACtC,QACA,iCACA,yEACF;CACA,iBAA0B,2BACxB,QACA,mBACA;EACE,aACE;EAGF,MAAM;CACR,CACF;CACA;CACA,0CAA0C;CAC1C;CACA;CAEA,YACE,IACA,QACA,mBAKA,gBACA,eAMA,QACA,mBACA;EACA,KAAKS,UAAU;EACf,KAAKC,qBAAqB;EAG1B,MAAM,WAAW,mBAAmB,IAAI,SAAQ,UAC9C,KAAKO,yBAAyB,KAAK,CACrC;EACA,YAAY,IAAI,MAAM;EAEtB,KAAKjB,MAAM;EACX,KAAKC,eAAe,IAAI,cACtB,KACA,OAAM,kBAAkB,IAAI,SAAS,UAAU,GAAG,KAAKI,iBAAiB,IACxE,MAAK,EAAE,UAAU,CACnB;EACA,IAAI,gBACF,KAAKH,YAAY,IAAI,cAAc,IAAI,iBAAgB,MAAK,EAAE,QAAQ,CAAC;EAEzE,IAAI,eACF,KAAKC,WAAW,IAAI,cAClB,KACA,OACE,cACE,IACA,KAAKF,aAAa,WAAW,EAAE,EAAE,kBACnC,IACF,MAAK,EAAE,QAAQ,CACjB;EAEF,KAAKK,UAAU;EACf,KAAKC,OAAO,IAAI,gBAAgB,0BAA0B,MAAM,CAAC;EAEjE,yBACE,IACA,KAAKA,MACL,KAAKW,mBACL,KAAKZ,OACP;EAEA,kCAAkC,KAAKL,aAAa,IAAI;EAExD,iBACE,QACA,wBACA,gCACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKA,aAAa,IAAI,CAAC;EAE9D,iBACE,QACA,WACA,qDACF,EAAE,aAAY,WAAU;GACtB,IAAI,QAAQ;GACZ,KAAK,MAAM,MAAM,KAAKA,aAAa,YAAY,GAC7C,SAAS,GAAG;GAEd,OAAO,QAAQ,KAAK;EACtB,CAAC;EAED,iBACE,QACA,QACA,uCACF,EAAE,aAAY,WAAU;GACtB,IAAI,QAAQ;GACZ,KAAK,MAAM,MAAM,KAAKA,aAAa,YAAY,GAC7C,SAAS,GAAG;GAEd,OAAO,QAAQ,KAAK;EACtB,CAAC;EAED,iBAAiB,QAAQ,eAAe;GACtC,aACE;GAGF,MAAM;EACR,CAAC,EAAE,aAAY,WAAU;GACvB,OAAO,QAAQ,KAAKkB,+BAA+B,EAAE,KAAK;EAC5D,CAAC;EAED,iBAAiB,QAAQ,qBAAqB;GAC5C,aACE;GAGF,MAAM;EACR,CAAC,EAAE,aAAY,WAAU;GACvB,MAAM,QAAQ,KAAKA,+BAA+B;GAClD,OAAO,QAAQ,MAAM,OAAO,EAAC,MAAM,MAAK,CAAC;GACzC,OAAO,QAAQ,MAAM,OAAO,EAAC,MAAM,MAAK,CAAC;GACzC,OAAO,QAAQ,MAAM,OAAO,EAAC,MAAM,MAAK,CAAC;GACzC,OAAO,QAAQ,MAAM,OAAO,EAAC,MAAM,MAAK,CAAC;GACzC,OAAO,QAAQ,MAAM,OAAO,EAAC,MAAM,MAAK,CAAC;EAC3C,CAAC;EAED,iBACE,QACA,iCACA,8DACF,EAAE,aAAY,WAAU;GACtB,OAAO,QAAQ,KAAKA,+BAA+B,EAAE,mBAAmB;EAC1E,CAAC;EAED,iBACE,QACA,mBACA,0KAGF,EAAE,aAAY,WAAU;GACtB,MAAM,QAAQ,KAAKC,2BAA2B;GAC9C,OAAO,QAAQ,MAAM,aAAa,EAAC,MAAM,SAAQ,CAAC;GAClD,OAAO,QAAQ,MAAM,eAAe,EAAC,MAAM,WAAU,CAAC;EACxD,CAAC;EAED,iBACE,QACA,oBACA,8GAEF,EAAE,aAAY,WAAU;GACtB,MAAM,QAAQ,KAAKA,2BAA2B;GAC9C,OAAO,QAAQ,MAAM,aAAa,MAAM,EAAC,MAAM,SAAQ,CAAC;GACxD,OAAO,QAAQ,MAAM,gBAAgB,EAAC,MAAM,WAAU,CAAC;EACzD,CAAC;EAED,iBACE,QACA,kBACA,iEACF,EAAE,aAAY,WAAU;GACtB,OAAO,QAAQ,KAAKA,2BAA2B,EAAE,aAAa;EAChE,CAAC;EAED,KAAKC,+BAA+B,kBAC5B,KAAKC,4BAA4B,GACvC,kCACF;EACA,KAAKD,6BAA6B,QAAQ;EAE1C,KAAKE,0BAA0B,kBACvB,KAAKC,6BAA6B,GACxC,0CACF;EACA,KAAKD,wBAAwB,QAAQ;CACvC;CAEA,iCAAyD;EACvD,IAAI,KAAKE,8BACP,OAAO,KAAKA;EAGd,MAAM,eAAe,gCACnB,KAAK,IAAI,GACT,KAAKd,qBACL,KAAKV,aAAa,YAAY,CAChC;EACA,KAAKwB,+BAA+B;EACpC,IAAI,CAAC,KAAKC,yCAAyC;GACjD,KAAKA,0CAA0C;GAC/C,qBAAqB;IACnB,KAAKD,+BAA+B,KAAA;IACpC,KAAKC,0CAA0C;GACjD,CAAC;EACH;EACA,OAAO;CACT;CAEA,8BAAoC;EAClC,MAAM,EAAC,WAAU,KAAKP,+BAA+B;EACrD,KAAK,MAAM,SAAS,QAClB,KAAKH,eAAe,SAAS,KAAK;CAEtC;CAEA,6BAAiD;EAC/C,OAAO,0BAA0B,KAAKf,aAAa,YAAY,CAAC;CAClE;CAEA,+BAAqC;EACnC,MAAM,QAAQ,KAAKmB,2BAA2B;EAC9C,IAAI,MAAM,gBAAgB,GACxB;EAEF,MAAM,SAAS,MAAM,aAAa;EAClC,MAAM,gBAAgB,CAAC,GAAG,MAAM,aAAa,QAAQ,CAAC,EACnD,QAAQ,GAAG,EAAC,aAAY,QAAQ,CAAC,EACjC,MAAM,GAAG,MAAM,EAAE,GAAG,QAAQ,EAAE,GAAG,KAAK,EACtC,MAAM,GAAG,EAAE,EACX,KAAK,CAAC,KAAK,EAAC,OAAO,kBAAiB;GAAC;GAAK;GAAO;EAAS,EAAE;EAC/D,KAAKpB,IAAI,OAAO,8BAA8B;GAC5C,iBAAiB,MAAM;GACvB,uBAAuB;GACvB,aAAa,QAAQ,MAAM,cAAc,KAAK,IAAI,QAAQ,CAAC,GAAG,QAAQ,CAAC,CAAC;GACxE,mBAAmB,MAAM;GACzB,eAAe,MAAM;GACrB;EACF,CAAC;CACH;CAEA,yBAAyB,OAA2B;EAClD,IACE,MAAM,cAAc,KAAA,KACpB,MAAM,uBAAuB,KAAA,GAE7B;EAEF,MAAM,OAAO,KAAKW,oBAAoB,GAAG,EAAE;EAC3C,IAAI,QAAQ,KAAK,aAAa,MAAM,WAClC;EAEF,KAAKA,oBAAoB,KAAK;GAC5B,WAAW,MAAM;GACjB,oBAAoB,MAAM;EAC5B,CAAC;EACD,IAAI,KAAKV,aAAa,SAAS,GAAG;GAChC,KAAKU,oBAAoB,SAAS;GAClC;EACF;EACA,wBAAwB,KAAKA,mBAAmB;CAClD;CAEA,oBAA6B,OAAO,IAAe,WAA0B;EAC3E,MAAM,mBAAmB,GACtB,6BAA6B,OAAO,gBACvC;EACA,IAAI,2BAA2B;EAC/B,MAAM,2BAA2B,WAAmB;GAClD,IAAI,0BACF;GAEF,2BAA2B;GAC3B,KAAKI,6BAA6B,IAAI,GAAG;IACvC,GAAG;KACF,2BAA2B;GAC9B,CAAC;EACH;EACA,MAAM,sCAAsC;GAC1C,IAAI,0BACF;GAEF,2BAA2B;GAC3B,KAAKD,8BAA8B,IAAI,GAAG,gBAAgB;EAC5D;EAEA,KAAKD,6BAA6B,IAAI,GAAG,gBAAgB;EACzD,KAAKD,0BAA0B,IAAI,GAAG,gBAAgB;EACtD,GAAG,KAAK,eAAe;GACrB,KAAKA,0BAA0B,IAAI,IAAI,gBAAgB;EACzD,CAAC;EAED,KAAKZ,IAAI,QACP,uBACA,OAAO,eACP,OAAO,QACT;EACA,0BAA0B;EAC1B,MAAM,EAAC,UAAU,eAAe,MAAM,WAAU;EAChD,MAAM,kBAAkB,SAAS,KAAA,KAAa,SAAS;EACvD,MAAM,iBAAiB,UAAU;EAEjC,IAAI,iBAAiB;GACnB,MAAM,eAAe,mBAAmB,KAAKS,QAAQ,QAAQ,CAAC,CAAC;GAE/D,MAAM,kBACJ,KAAKA,SAAS,MAAM,QAAQ,KAAA,KAC5B,KAAKA,SAAS,QAAQ,QAAQ,KAAA;GAChC,MAAM,aACJ,KAAKA,SAAS,OAAO,QAAQ,KAAA,KAC7B,KAAKA,SAAS,YAAY,QAAQ,KAAA;GAKpC,IAAI,EAF6B,aAAa,WAAW,MAExB,EADN,mBAAmB,aACQ;IACpD,wBAAwB,eAAe;IACvC,MAAM,IAAI,MACR,uHACE,KAAK,UAAU,YAAY,IAC3B,+IACJ;GACF;EACF;EAEA,IAAI;EAIJ,IAAI;GACF,cAAc,MAAM,YAClB,KAAKT,IACF,YAAY,iBAAiB,aAAa,EAC1C,YAAY,YAAY,QAAQ,GAGnC,KAAA,GACA,gBACA,MACA,KAAKU,kBACP;EACF,SAAS,GAAG;GACV,IAAI,gBAAgB,CAAC,GAAG;IACtB,KAAKV,IAAI,OACP,4DACA;KACE;KACA;KACA;KACA;KACA,WAAW,EAAE;IACf,CACF;IACA,wBAAwB,MAAM;IAC9B,UAAU,KAAKA,KAAK,IAAI,EAAE,SAAS;IACnC,GAAG,MAAM,KAAM,EAAE,UAAU,OAAO;IAClC;GACF;GACA,wBAAwB,UAAU;GAClC,MAAM;EACR;EAEA,MAAM,aAAa,KAAKC,aAAa,WAAW,aAAa;EAC7D,MAAM,qBAAqB,WAAW;EACtC,MAAM,QAAQ,mBAAmB,cAAc;EAQ/C,IACE,MAAM,eAAe,KAAA,KACrB,MAAM,WAAW,OAAO,gBACxB;GACA,MAAM,QAAQ,IAAI,cAAc;IAC9B,MAAM;IACN,SACE;IACF,QAAQ;GACV,CAAC;GACD,wBAAwB,eAAe;GACvC,UAAU,KAAKD,KAAK,IAAI,MAAM,SAAS;GACvC,GAAG,MAAM,KAAM,MAAM,OAAO;GAC5B;EACF;EAGA,MAAM,WAAW,KAAKI,aAAa,IAAI,QAAQ;EAC/C,IAAI,UAAU;GACZ,KAAKJ,IAAI,QACP,UAAU,SAAS,gDACrB;GACA,SAAS,MAAM,eAAe,OAAO,MAAM;EAC7C;EAEA,mBAAmB,mBACjB;GAAC;GAAU,MAAM,OAAO;EAAI,GAC5B,QACA,WACF;EAEA,MAAM,UAAU,KAAKE,WAAW,WAAW,aAAa;EACxD,MAAM,SAAS,KAAKC,UAAU,WAAW,aAAa;EAEtD,SAAS,IAAI;EACb,QAAQ,IAAI;EAEZ,IAAI;EACJ,IAAI;GACF,aAAa,IAAI,WACf,KAAKH,KACL,QACA,IACA,KAAKS,QAAQ,oBACb,IAAI,uBACF,KAAKT,KACL,QACA,oBACA,YACA,SACA,MACF,SACM;IACJ,mBAAmB,gBAAgB;KACjC;KACA,MAAM,OAAO;IACf,CAAC;IACD,IAAI,KAAKI,aAAa,IAAI,QAAQ,MAAM,YACtC,KAAKA,aAAa,OAAO,QAAQ;IAInC,SAAS,MAAM;IACf,QAAQ,MAAM;GAChB,CACF;EACF,SAAS,GAAG;GACV,wBAAwB,UAAU;GAClC,mBAAmB,gBAAgB;IAAC;IAAU,MAAM,OAAO;GAAI,CAAC;GAChE,SAAS,MAAM;GACf,QAAQ,MAAM;GACd,MAAM;EACR;EAEA,KAAKA,aAAa,IAAI,UAAU,UAAU;EAE1C,IAAI,WAAW,KAAK,GAAG;GACrB,8BAA8B;GAC9B,wBAAwB;EAC1B,OACE,wBAAwB,kBAAkB;EAG5C,IAAI,OAAO,mBAAmB;GAC5B,KAAKJ,IAAI,QACP,oDACA,OAAO,eACP,OAAO,QACT;GACA,MAAM,WAAW,qBACf,KAAK,UAAU,OAAO,iBAAiB,CACzC;EACF;CACF;CAEA,MAAM;EACJ,OAAO,KAAKQ,SAAS;CACvB;;;;;;;CAQA,MAAM,QAAQ;EACZ,MAAM,QAAQ,KAAK,IAAI;EACvB,KAAKR,IAAI,OAAO,YAAY,KAAKC,aAAa,KAAK,cAAc;EAEjE,KAAKI,kBAAkB,YAAY,CAAC;EAEpC,OAAO,KAAKJ,aAAa,MAAM;GAC7B,MAAM,KAAKI,kBAAkB;GAG7B,KAAK,MAAM,MAAM,KAAKJ,aAAa,YAAY,GAAG;IAChD,KAAKD,IAAI,QAAQ,wBAAwB,GAAG,GAAG,UAAU;IAGzD,GAAQ,KAAK;IACb;GACF;EACF;EACA,KAAKA,IAAI,OAAO,sBAAsB,KAAK,IAAI,IAAI,MAAM,KAAK;CAChE;CAEA,OAAO;EACL,cAAc,KAAKqB,4BAA4B;EAC/C,cAAc,KAAKE,uBAAuB;EAC1C,KAAKhB,KAAK,MAAM;EAChB,KAAKC,SAAS,QAAQ;EACtB,OAAO;CACT;AACF"}
+{"version":3,"file":"syncer.js","names":["#lc","#viewSyncers","#mutagens","#pushers","#connections","#pendingConnections","#clientGroupGenerations","#drainCoordinator","#parent","#wss","#stopped","#config","#validateLegacyJWT","#replicaReadyStates","#webSocketOpenConnections","#webSocketConnectionAttempts","#webSocketConnectionSuccesses","#webSocketConnectionFailures","#viewSyncerLag","#recordReplicaReadyState","#receiveConnection","#releasePending","#computeServingLagDistribution","#computePipelineDedupStats","#viewSyncerLagSampleInterval","#recordViewSyncerLagSamples","#eligibilityLogInterval","#logSharedAdvanceEligibility","#servingLagDistributionCache","#servingLagDistributionCacheClearQueued","#createConnection"],"sources":["../../../../../zero-cache/src/workers/syncer.ts"],"sourcesContent":["import {pid} from 'node:process';\nimport type {MessagePort} from 'node:worker_threads';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {WebSocket, WebSocketServer, type ServerOptions} from 'ws';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {ErrorKind} from '../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../zero-protocol/src/error-origin.ts';\nimport {\n  isProtocolError,\n  ProtocolError,\n} from '../../../zero-protocol/src/error.ts';\nimport {resolveAuth, type Auth, type ValidateLegacyJWT} from '../auth/auth.ts';\nimport {tokenConfigOptions} from '../auth/jwt.ts';\nimport {type ZeroConfig} from '../config/zero-config.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateGauge,\n  getOrCreateNativeHistogram,\n  getOrCreateUpDownCounter,\n} from '../observability/metrics.ts';\nimport {\n  recordConnectionAttempted,\n  recordConnectionSuccess,\n  setActiveClientGroupsGetter,\n} from '../server/anonymous-otel-start.ts';\nimport type {Mutagen} from '../services/mutagen/mutagen.ts';\nimport type {Pusher} from '../services/mutagen/pusher.ts';\nimport type {ReplicaState} from '../services/replicator/replicator.ts';\nimport {ServiceRunner} from '../services/runner.ts';\nimport type {\n  ActivityBasedService,\n  Service,\n  SingletonService,\n} from '../services/service.ts';\nimport type {ConnectionContextManager} from '../services/view-syncer/connection-context-manager.ts';\nimport {DrainCoordinator} from '../services/view-syncer/drain-coordinator.ts';\nimport type {ViewSyncer} from '../services/view-syncer/view-syncer.ts';\nimport type {ClientGroupStatusMessage, Worker} from '../types/processes.ts';\nimport type {Subscription} from '../types/subscription.ts';\nimport {installWebSocketReceiver} from '../types/websocket-handoff.ts';\nimport type {ConnectParams} from './connect-params.ts';\nimport {Connection, sendError} from './connection.ts';\nimport {createNotifierFrom, subscribeTo} from './replicator.ts';\nimport {SyncerWsMessageHandler} from './syncer-ws-message-handler.ts';\n\nexport type SyncerWorkerData = {\n  replicatorPort: MessagePort;\n};\n\nexport type ReplicaReadyState = {\n  readonly watermark: string;\n  readonly replicaReadyTimeMs: number;\n};\n\nexport type ServingLagViewSyncer = Pick<\n  ViewSyncer,\n  'createdAtMs' | 'servedVersion' | 'servingLagEligible'\n>;\n\nexport type ServingLagStats = {\n  readonly activeClientGroups: number;\n  readonly laggingClientGroups: number;\n  readonly minMs: number;\n  readonly p50Ms: number;\n  readonly p75Ms: number;\n  readonly p99Ms: number;\n  readonly maxMs: number;\n};\n\ntype ServingLagDistribution = ServingLagStats & {\n  readonly lagsMs: readonly number[];\n};\n\nexport type PipelineDedupViewSyncer = Pick<\n  ViewSyncer,\n  'pipelineHashes' | 'clientSchemaKey'\n>;\n\nexport type PipelineDedupStats = {\n  readonly clientTotal: number;\n  readonly internalTotal: number;\n  // (clientSchemaKey, transformationHash) -> occurrence count across client\n  // groups, for client/custom queries.\n  readonly clientHashes: ReadonlyMap<\n    string,\n    {count: number; queryName: string | undefined}\n  >;\n  readonly internalUnique: number;\n  readonly clientSchemas: number;\n};\n\nexport function computePipelineDedupStats(\n  viewSyncers: Iterable<PipelineDedupViewSyncer>,\n): PipelineDedupStats {\n  let clientTotal = 0;\n  let internalTotal = 0;\n  const clientHashes = new Map<\n    string,\n    {count: number; queryName: string | undefined}\n  >();\n  const internalHashes = new Set<string>();\n  const schemaKeys = new Set<string>();\n  let vsIndex = 0;\n  for (const vs of viewSyncers) {\n    const schemaKey = vs.clientSchemaKey;\n    if (schemaKey !== undefined) {\n      schemaKeys.add(schemaKey);\n    }\n    // Without a client schema the pipeline's dedup identity is unknown, so key\n    // it uniquely per view-syncer; otherwise schema-less pipelines from\n    // unrelated client groups would collapse under a shared `undefined/<hash>`\n    // key and overstate deduplication.\n    const dedupSchemaKey = schemaKey ?? `#vs${vsIndex}`;\n    vsIndex++;\n    for (const {\n      transformationHash,\n      internal,\n      queryName,\n    } of vs.pipelineHashes()) {\n      // Pipelines only share when both the client schema (which determines\n      // row keys) and the transformed AST are identical.\n      const key = `${dedupSchemaKey}/${transformationHash}`;\n      if (internal) {\n        internalTotal++;\n        internalHashes.add(key);\n      } else {\n        clientTotal++;\n        const entry = clientHashes.get(key);\n        if (entry) {\n          entry.count++;\n        } else {\n          clientHashes.set(key, {count: 1, queryName});\n        }\n      }\n    }\n  }\n  return {\n    clientTotal,\n    internalTotal,\n    clientHashes,\n    internalUnique: internalHashes.size,\n    clientSchemas: schemaKeys.size,\n  };\n}\n\nexport const MAX_REPLICA_READY_STATES = 10_000;\nconst VIEW_SYNCER_LAG_SAMPLE_INTERVAL_MS = 60_000;\nconst SHARED_ADVANCE_ELIGIBILITY_LOG_INTERVAL_MS = 5 * 60_000;\n\nconst PROTOCOL_VERSION_ATTRIBUTE = 'protocol.version';\nconst FAILURE_REASON_ATTRIBUTE = 'reason';\n\nfunction boundReplicaReadyStates(\n  replicaReadyStates: ReplicaReadyState[],\n): void {\n  if (replicaReadyStates.length > MAX_REPLICA_READY_STATES) {\n    replicaReadyStates.splice(\n      0,\n      replicaReadyStates.length - MAX_REPLICA_READY_STATES,\n    );\n  }\n}\n\nfunction pruneReplicaReadyStates(\n  replicaReadyStates: ReplicaReadyState[],\n  firstNeededIndex: number,\n): void {\n  if (firstNeededIndex > 0) {\n    replicaReadyStates.splice(0, firstNeededIndex);\n  }\n\n  boundReplicaReadyStates(replicaReadyStates);\n}\n\nfunction lowerBoundReplicaReadyTimeMs(\n  replicaReadyStates: readonly ReplicaReadyState[],\n  replicaReadyTimeMs: number,\n): number {\n  let low = 0;\n  let high = replicaReadyStates.length;\n  while (low < high) {\n    const mid = low + Math.floor((high - low) / 2);\n    if (replicaReadyStates[mid].replicaReadyTimeMs < replicaReadyTimeMs) {\n      low = mid + 1;\n    } else {\n      high = mid;\n    }\n  }\n  return low;\n}\n\nfunction upperBoundWatermark(\n  replicaReadyStates: readonly ReplicaReadyState[],\n  watermark: string,\n): number {\n  let low = 0;\n  let high = replicaReadyStates.length;\n  while (low < high) {\n    const mid = low + Math.floor((high - low) / 2);\n    if (replicaReadyStates[mid].watermark <= watermark) {\n      low = mid + 1;\n    } else {\n      high = mid;\n    }\n  }\n  return low;\n}\n\nfunction findFirstUnservedIndex(\n  replicaReadyStates: readonly ReplicaReadyState[],\n  viewSyncer: ServingLagViewSyncer,\n): number {\n  const firstReadyAfterCreation = lowerBoundReplicaReadyTimeMs(\n    replicaReadyStates,\n    viewSyncer.createdAtMs,\n  );\n  const firstAfterServedVersion =\n    viewSyncer.servedVersion === null\n      ? 0\n      : upperBoundWatermark(replicaReadyStates, viewSyncer.servedVersion);\n\n  const firstUnservedIndex = Math.max(\n    firstReadyAfterCreation,\n    firstAfterServedVersion,\n  );\n  return firstUnservedIndex < replicaReadyStates.length\n    ? firstUnservedIndex\n    : -1;\n}\n\nfunction percentileNearestRank(\n  sortedValues: readonly number[],\n  percentile: number,\n): number {\n  if (sortedValues.length === 0) {\n    return 0;\n  }\n  const index = Math.min(\n    sortedValues.length - 1,\n    Math.max(0, Math.ceil((percentile / 100) * sortedValues.length) - 1),\n  );\n  return sortedValues[index];\n}\n\nfunction computeServingLagDistributionMs(\n  now: number,\n  replicaReadyStates: ReplicaReadyState[],\n  viewSyncers: Iterable<ServingLagViewSyncer>,\n): ServingLagDistribution {\n  const lags: number[] = [];\n  let laggingClientGroups = 0;\n  let firstNeededIndex = replicaReadyStates.length;\n\n  for (const viewSyncer of viewSyncers) {\n    if (!viewSyncer.servingLagEligible) {\n      continue;\n    }\n\n    const firstUnservedIndex = findFirstUnservedIndex(\n      replicaReadyStates,\n      viewSyncer,\n    );\n    if (firstUnservedIndex === -1) {\n      lags.push(0);\n      continue;\n    }\n\n    firstNeededIndex = Math.min(firstNeededIndex, firstUnservedIndex);\n    const lagMs = Math.max(\n      0,\n      now - replicaReadyStates[firstUnservedIndex].replicaReadyTimeMs,\n    );\n    lags.push(lagMs);\n    if (lagMs > 0) {\n      laggingClientGroups++;\n    }\n  }\n\n  pruneReplicaReadyStates(replicaReadyStates, firstNeededIndex);\n\n  lags.sort((a, b) => a - b);\n  return {\n    activeClientGroups: lags.length,\n    laggingClientGroups,\n    // The percentile fields are for the legacy serving_lag_stats gauge.\n    // The native view_syncer_lag histogram records lagsMs and computes\n    // percentiles in Prometheus/AMP.\n    minMs: lags[0] ?? 0,\n    p50Ms: percentileNearestRank(lags, 50),\n    p75Ms: percentileNearestRank(lags, 75),\n    p99Ms: percentileNearestRank(lags, 99),\n    maxMs: lags.at(-1) ?? 0,\n    lagsMs: lags,\n  };\n}\n\nexport function computeServingLagStatsMs(\n  now: number,\n  replicaReadyStates: ReplicaReadyState[],\n  viewSyncers: Iterable<ServingLagViewSyncer>,\n): ServingLagStats {\n  const distribution = computeServingLagDistributionMs(\n    now,\n    replicaReadyStates,\n    viewSyncers,\n  );\n  return {\n    activeClientGroups: distribution.activeClientGroups,\n    laggingClientGroups: distribution.laggingClientGroups,\n    minMs: distribution.minMs,\n    p50Ms: distribution.p50Ms,\n    p75Ms: distribution.p75Ms,\n    p99Ms: distribution.p99Ms,\n    maxMs: distribution.maxMs,\n  };\n}\n\nexport function computeMaxServingLagMs(\n  now: number,\n  replicaReadyStates: ReplicaReadyState[],\n  viewSyncers: Iterable<ServingLagViewSyncer>,\n): number {\n  return computeServingLagStatsMs(now, replicaReadyStates, viewSyncers).maxMs;\n}\n\nfunction getWebSocketServerOptions(config: ZeroConfig): ServerOptions {\n  const options: ServerOptions = {\n    noServer: true,\n    maxPayload: config.websocketMaxPayloadBytes,\n  };\n\n  if (config.websocketCompression) {\n    options.perMessageDeflate = true;\n\n    if (config.websocketCompressionOptions) {\n      try {\n        const compressionOptions = JSON.parse(\n          config.websocketCompressionOptions,\n        );\n        options.perMessageDeflate = compressionOptions;\n      } catch (e) {\n        throw new Error(\n          `Failed to parse ZERO_WEBSOCKET_COMPRESSION_OPTIONS: ${String(e)}. Expected valid JSON.`,\n        );\n      }\n    }\n  }\n\n  return options;\n}\n\n/**\n * The Syncer worker receives websocket handoffs for \"/sync\" connections\n * from the Dispatcher in the main thread, and creates websocket\n * {@link Connection}s with a corresponding {@link ViewSyncer}, {@link Mutagen},\n * and {@link Subscription} to version notifications from the Replicator\n * worker.\n */\nexport class Syncer implements SingletonService {\n  readonly id = `syncer-${pid}`;\n  readonly #lc: LogContext;\n  readonly #viewSyncers: ServiceRunner<ViewSyncer & ActivityBasedService>;\n  readonly #mutagens: ServiceRunner<Mutagen & Service> | undefined;\n  readonly #pushers: ServiceRunner<Pusher & Service> | undefined;\n  readonly #connections = new Map<string, Connection>();\n  readonly #pendingConnections = new Map<string, number>();\n  readonly #clientGroupGenerations = new Map<string, number>();\n  readonly #drainCoordinator = new DrainCoordinator();\n  readonly #parent: Worker;\n  readonly #wss: WebSocketServer;\n  readonly #stopped = resolver();\n  readonly #config: ZeroConfig;\n  readonly #validateLegacyJWT: ValidateLegacyJWT | undefined;\n  readonly #replicaReadyStates: ReplicaReadyState[] = [];\n  readonly #webSocketOpenConnections = getOrCreateUpDownCounter(\n    'sync',\n    'websocket.open_connections',\n    'Open client WebSocket connections.',\n  );\n  readonly #webSocketConnectionAttempts = getOrCreateCounter(\n    'sync',\n    'websocket.connection_attempts',\n    'Client WebSocket connection attempts.',\n  );\n  readonly #webSocketConnectionSuccesses = getOrCreateCounter(\n    'sync',\n    'websocket.connection_successes',\n    'Client WebSocket connections successfully initialized.',\n  );\n  readonly #webSocketConnectionFailures = getOrCreateCounter(\n    'sync',\n    'websocket.connection_failures',\n    'Client WebSocket connection attempts that failed before initialization.',\n  );\n  readonly #viewSyncerLag = getOrCreateNativeHistogram(\n    'sync',\n    'view_syncer_lag',\n    {\n      description:\n        'Lag from replica-ready change to ViewSyncer output for active ' +\n        'client groups. A change is output after IVM advancement, CVR flush, ' +\n        'and pokeEnd.',\n      unit: 's',\n    },\n  );\n  #servingLagDistributionCache: ServingLagDistribution | undefined;\n  #servingLagDistributionCacheClearQueued = false;\n  #viewSyncerLagSampleInterval: ReturnType<typeof setInterval> | undefined;\n  #eligibilityLogInterval: ReturnType<typeof setInterval> | undefined;\n\n  constructor(\n    lc: LogContext,\n    config: ZeroConfig,\n    viewSyncerFactory: (\n      id: string,\n      sub: Subscription<ReplicaState>,\n      drainCoordinator: DrainCoordinator,\n    ) => ViewSyncer & ActivityBasedService,\n    mutagenFactory: ((id: string) => Mutagen & Service) | undefined,\n    pusherFactory:\n      | ((\n          id: string,\n          connContextManager: ConnectionContextManager,\n        ) => Pusher & Service)\n      | undefined,\n    parent: Worker,\n    validateLegacyJWT: ValidateLegacyJWT | undefined,\n  ) {\n    this.#config = config;\n    this.#validateLegacyJWT = validateLegacyJWT;\n    // Relays notifications from the parent thread subscription\n    // to ViewSyncers within this thread.\n    const notifier = createNotifierFrom(lc, parent, state =>\n      this.#recordReplicaReadyState(state),\n    );\n    subscribeTo(lc, parent);\n\n    this.#lc = lc;\n    this.#parent = parent;\n    this.#viewSyncers = new ServiceRunner(\n      lc,\n      id => viewSyncerFactory(id, notifier.subscribe(), this.#drainCoordinator),\n      v => v.keepalive(),\n      id => {\n        if ((this.#pendingConnections.get(id) ?? 0) === 0) {\n          const generation = this.#clientGroupGenerations.get(id);\n          this.#parent.send<ClientGroupStatusMessage>([\n            'clientGroupStatus',\n            {\n              clientGroupID: id,\n              active: false,\n              ...(generation !== undefined ? {generation} : {}),\n            },\n          ]);\n          this.#clientGroupGenerations.delete(id);\n        }\n      },\n    );\n    if (mutagenFactory) {\n      this.#mutagens = new ServiceRunner(lc, mutagenFactory, m => m.hasRefs());\n    }\n    if (pusherFactory) {\n      this.#pushers = new ServiceRunner(\n        lc,\n        id =>\n          pusherFactory(\n            id,\n            this.#viewSyncers.getService(id).connContextManager,\n          ),\n        p => p.hasRefs(),\n      );\n    }\n    this.#wss = new WebSocketServer(getWebSocketServerOptions(config));\n\n    installWebSocketReceiver(\n      lc,\n      this.#wss,\n      this.#receiveConnection,\n      this.#parent,\n      params => {\n        this.#releasePending(params.clientGroupID, params.generation);\n      },\n      params => {\n        this.#pendingConnections.set(\n          params.clientGroupID,\n          (this.#pendingConnections.get(params.clientGroupID) ?? 0) + 1,\n        );\n        if (params.generation !== undefined) {\n          const current = this.#clientGroupGenerations.get(\n            params.clientGroupID,\n          );\n          if (current === undefined || params.generation > current) {\n            this.#clientGroupGenerations.set(\n              params.clientGroupID,\n              params.generation,\n            );\n          }\n        }\n      },\n    );\n\n    setActiveClientGroupsGetter(() => this.#viewSyncers.size);\n\n    getOrCreateGauge(\n      'sync',\n      'active-client-groups',\n      'Number of active client groups',\n    ).addCallback(result => result.observe(this.#viewSyncers.size));\n\n    getOrCreateGauge(\n      'sync',\n      'queries',\n      'Active queries (pipelines) across all client groups',\n    ).addCallback(result => {\n      let total = 0;\n      for (const vs of this.#viewSyncers.getServices()) {\n        total += vs.queryCount;\n      }\n      result.observe(total);\n    });\n\n    getOrCreateGauge(\n      'sync',\n      'rows',\n      'Tracked rows across all client groups',\n    ).addCallback(result => {\n      let total = 0;\n      for (const vs of this.#viewSyncers.getServices()) {\n        total += vs.rowCount;\n      }\n      result.observe(total);\n    });\n\n    getOrCreateGauge('sync', 'serving_lag', {\n      description:\n        'Maximum time active ViewSyncer client groups have had unserved ' +\n        'replica changes. A change is served after IVM advancement, CVR flush, ' +\n        'and pokeEnd.',\n      unit: 'millisecond',\n    }).addCallback(result => {\n      result.observe(this.#computeServingLagDistribution().maxMs);\n    });\n\n    getOrCreateGauge('sync', 'serving_lag_stats', {\n      description:\n        'Distribution of time active ViewSyncer client groups have had ' +\n        'unserved replica changes. A change is served after IVM advancement, ' +\n        'CVR flush, and pokeEnd.',\n      unit: 'millisecond',\n    }).addCallback(result => {\n      const stats = this.#computeServingLagDistribution();\n      result.observe(stats.minMs, {stat: 'min'});\n      result.observe(stats.p50Ms, {stat: 'p50'});\n      result.observe(stats.p75Ms, {stat: 'p75'});\n      result.observe(stats.p99Ms, {stat: 'p99'});\n      result.observe(stats.maxMs, {stat: 'max'});\n    });\n\n    getOrCreateGauge(\n      'sync',\n      'serving_lagging_client_groups',\n      'Number of active client groups with unserved replica changes',\n    ).addCallback(result => {\n      result.observe(this.#computeServingLagDistribution().laggingClientGroups);\n    });\n\n    getOrCreateGauge(\n      'sync',\n      'pipelines_total',\n      'Query pipelines across all client groups on this worker, by query ' +\n        'type. The ratio to sync.pipelines_unique is the dedup factor ' +\n        'available to shared pipeline advancement.',\n    ).addCallback(result => {\n      const stats = this.#computePipelineDedupStats();\n      result.observe(stats.clientTotal, {type: 'client'});\n      result.observe(stats.internalTotal, {type: 'internal'});\n    });\n\n    getOrCreateGauge(\n      'sync',\n      'pipelines_unique',\n      'Distinct (clientSchema, transformationHash) pipelines across all ' +\n        'client groups on this worker, by query type',\n    ).addCallback(result => {\n      const stats = this.#computePipelineDedupStats();\n      result.observe(stats.clientHashes.size, {type: 'client'});\n      result.observe(stats.internalUnique, {type: 'internal'});\n    });\n\n    getOrCreateGauge(\n      'sync',\n      'client_schemas',\n      'Distinct client schemas across all client groups on this worker',\n    ).addCallback(result => {\n      result.observe(this.#computePipelineDedupStats().clientSchemas);\n    });\n\n    this.#viewSyncerLagSampleInterval = setInterval(\n      () => this.#recordViewSyncerLagSamples(),\n      VIEW_SYNCER_LAG_SAMPLE_INTERVAL_MS,\n    );\n    this.#viewSyncerLagSampleInterval.unref?.();\n\n    this.#eligibilityLogInterval = setInterval(\n      () => this.#logSharedAdvanceEligibility(),\n      SHARED_ADVANCE_ELIGIBILITY_LOG_INTERVAL_MS,\n    );\n    this.#eligibilityLogInterval.unref?.();\n  }\n\n  #computeServingLagDistribution(): ServingLagDistribution {\n    if (this.#servingLagDistributionCache) {\n      return this.#servingLagDistributionCache;\n    }\n\n    const distribution = computeServingLagDistributionMs(\n      Date.now(),\n      this.#replicaReadyStates,\n      this.#viewSyncers.getServices(),\n    );\n    this.#servingLagDistributionCache = distribution;\n    if (!this.#servingLagDistributionCacheClearQueued) {\n      this.#servingLagDistributionCacheClearQueued = true;\n      queueMicrotask(() => {\n        this.#servingLagDistributionCache = undefined;\n        this.#servingLagDistributionCacheClearQueued = false;\n      });\n    }\n    return distribution;\n  }\n\n  #recordViewSyncerLagSamples(): void {\n    const {lagsMs} = this.#computeServingLagDistribution();\n    for (const lagMs of lagsMs) {\n      this.#viewSyncerLag.recordMs(lagMs);\n    }\n  }\n\n  #computePipelineDedupStats(): PipelineDedupStats {\n    return computePipelineDedupStats(this.#viewSyncers.getServices());\n  }\n\n  #logSharedAdvanceEligibility(): void {\n    const stats = this.#computePipelineDedupStats();\n    if (stats.clientTotal === 0) {\n      return;\n    }\n    const unique = stats.clientHashes.size;\n    const topDuplicated = [...stats.clientHashes.entries()]\n      .filter(([, {count}]) => count > 1)\n      .sort((a, b) => b[1].count - a[1].count)\n      .slice(0, 10)\n      .map(([key, {count, queryName}]) => ({key, count, queryName}));\n    this.#lc.info?.('shared-advance-eligibility', {\n      clientPipelines: stats.clientTotal,\n      uniqueClientPipelines: unique,\n      dedupFactor: Number((stats.clientTotal / Math.max(unique, 1)).toFixed(2)),\n      internalPipelines: stats.internalTotal,\n      clientSchemas: stats.clientSchemas,\n      topDuplicated,\n    });\n  }\n\n  #recordReplicaReadyState(state: ReplicaState): void {\n    if (\n      state.watermark === undefined ||\n      state.replicaReadyTimeMs === undefined\n    ) {\n      return;\n    }\n    const last = this.#replicaReadyStates.at(-1);\n    if (last && last.watermark >= state.watermark) {\n      return;\n    }\n    this.#replicaReadyStates.push({\n      watermark: state.watermark,\n      replicaReadyTimeMs: state.replicaReadyTimeMs,\n    });\n    if (this.#viewSyncers.size === 0) {\n      this.#replicaReadyStates.length = 0;\n      return;\n    }\n    boundReplicaReadyStates(this.#replicaReadyStates);\n  }\n\n  readonly #createConnection = async (ws: WebSocket, params: ConnectParams) => {\n    const metricAttributes = {\n      [PROTOCOL_VERSION_ATTRIBUTE]: params.protocolVersion,\n    };\n    let connectionResultRecorded = false;\n    const recordConnectionFailure = (reason: string) => {\n      if (connectionResultRecorded) {\n        return;\n      }\n      connectionResultRecorded = true;\n      this.#webSocketConnectionFailures.add(1, {\n        ...metricAttributes,\n        [FAILURE_REASON_ATTRIBUTE]: reason,\n      });\n    };\n    const recordConnectionSuccessMetric = () => {\n      if (connectionResultRecorded) {\n        return;\n      }\n      connectionResultRecorded = true;\n      this.#webSocketConnectionSuccesses.add(1, metricAttributes);\n    };\n\n    this.#webSocketConnectionAttempts.add(1, metricAttributes);\n    this.#webSocketOpenConnections.add(1, metricAttributes);\n    ws.once('close', () => {\n      this.#webSocketOpenConnections.add(-1, metricAttributes);\n    });\n\n    this.#lc.debug?.(\n      'creating connection',\n      params.clientGroupID,\n      params.clientID,\n    );\n    recordConnectionAttempted();\n    const {clientID, clientGroupID, auth, userID} = params;\n    const hasProvidedAuth = auth !== undefined && auth !== '';\n    const incomingUserID = userID ?? null;\n\n    if (hasProvidedAuth) {\n      const tokenOptions = tokenConfigOptions(this.#config.auth ?? {});\n\n      const hasPushOrMutate =\n        this.#config?.push?.url !== undefined ||\n        this.#config?.mutate?.url !== undefined;\n      const hasQueries =\n        this.#config?.query?.url !== undefined ||\n        this.#config?.getQueries?.url !== undefined;\n\n      // must either have one of the token options set or have custom mutations & queries enabled\n      const hasExactlyOneTokenOption = tokenOptions.length === 1;\n      const hasCustomEndpoints = hasPushOrMutate && hasQueries;\n      if (!hasExactlyOneTokenOption && !hasCustomEndpoints) {\n        recordConnectionFailure('configuration');\n        throw new Error(\n          'Exactly one of jwk, secret, or jwksUrl must be set in order to verify tokens but actually the following were set: ' +\n            JSON.stringify(tokenOptions) +\n            '. You may also set both ZERO_MUTATE_URL and ZERO_QUERY_URL to enable custom mutations and queries without passing token verification options.',\n        );\n      }\n    }\n\n    let initialAuth: Auth | undefined;\n\n    // Verify JWT BEFORE touching existing connections - prevents unauthenticated\n    // attackers from force-disconnecting legitimate users via DoS.\n    try {\n      initialAuth = await resolveAuth(\n        this.#lc\n          .withContext('clientGroupID', clientGroupID)\n          .withContext('clientID', clientID),\n        // no previous auth, since this is a new connection, and resolveAuth is\n        // connection scoped, not client group scoped\n        undefined,\n        incomingUserID,\n        auth,\n        this.#validateLegacyJWT,\n      );\n    } catch (e) {\n      if (isProtocolError(e)) {\n        this.#lc.warn?.(\n          'Rejecting sync connection during initial auth resolution',\n          {\n            clientGroupID,\n            clientID,\n            incomingUserID,\n            hasProvidedAuth,\n            errorKind: e.message,\n          },\n        );\n        recordConnectionFailure('auth');\n        sendError(this.#lc, ws, e.errorBody);\n        ws.close(3000, e.errorBody.message);\n        return;\n      }\n      recordConnectionFailure('internal');\n      throw e;\n    }\n\n    // Resolving auth can take a while (e.g. a remote JWKS fetch), during\n    // which the client may have disconnected. A Connection created for a\n    // closed socket never receives the 'close' event and is thus never\n    // cleaned up, leaking its timer, its registrations in the connection\n    // context manager and its refs on the mutagen and pusher services.\n    if (ws.readyState !== WebSocket.OPEN) {\n      this.#lc.debug?.('websocket closed while resolving auth', {\n        clientGroupID,\n        clientID,\n        readyState: ws.readyState,\n      });\n      recordConnectionFailure('closed_during_auth');\n      return;\n    }\n\n    const viewSyncer = this.#viewSyncers.getService(clientGroupID);\n    const connContextManager = viewSyncer.connContextManager;\n    const group = connContextManager.getGroupState();\n\n    // TODO(0xcadams): we only check for user ID mismatch here if the group is\n    // already validated. This prevents wrong-user reconnects from evicting a\n    // healthy connection, but it does not protect against same-user reconnects\n    // with an invalid opaque token. The long-term fix is to keep the replacement\n    // connection pending until its auth is fully validated, and only then replace\n    // the existing socket.\n    if (\n      group.pinnedUser !== undefined &&\n      group.pinnedUser.id !== incomingUserID\n    ) {\n      const error = new ProtocolError({\n        kind: ErrorKind.Unauthorized,\n        message:\n          'Client groups are pinned to a single userID. Connection userID does not match existing client group userID.',\n        origin: ErrorOrigin.ZeroCache,\n      });\n      recordConnectionFailure('user_mismatch');\n      sendError(this.#lc, ws, error.errorBody);\n      ws.close(3000, error.message);\n      return;\n    }\n\n    // Check for and close existing connections AFTER auth is validated\n    const existing = this.#connections.get(clientID);\n    if (existing) {\n      this.#lc.debug?.(\n        `client ${clientID} already connected, closing existing connection`,\n      );\n      existing.close(`replaced by ${params.wsID}`);\n    }\n\n    connContextManager.registerConnection(\n      {clientID, wsID: params.wsID},\n      params,\n      initialAuth,\n    );\n\n    const mutagen = this.#mutagens?.getService(clientGroupID);\n    const pusher = this.#pushers?.getService(clientGroupID);\n    // a new connection is using the mutagen and pusher. Bump their ref counts.\n    mutagen?.ref();\n    pusher?.ref();\n\n    let connection: Connection;\n    try {\n      connection = new Connection(\n        this.#lc,\n        params,\n        ws,\n        this.#config.allowLegacyQueries,\n        new SyncerWsMessageHandler(\n          this.#lc,\n          params,\n          connContextManager,\n          viewSyncer,\n          mutagen,\n          pusher,\n        ),\n        () => {\n          connContextManager.closeConnection({\n            clientID,\n            wsID: params.wsID,\n          });\n          if (this.#connections.get(clientID) === connection) {\n            this.#connections.delete(clientID);\n          }\n          // Connection is closed. We can unref the mutagen and pusher.\n          // If their ref counts are zero, they will stop themselves and set themselves invalid.\n          mutagen?.unref();\n          pusher?.unref();\n        },\n      );\n    } catch (e) {\n      recordConnectionFailure('internal');\n      connContextManager.closeConnection({clientID, wsID: params.wsID});\n      mutagen?.unref();\n      pusher?.unref();\n      throw e;\n    }\n\n    this.#connections.set(clientID, connection);\n\n    if (connection.init()) {\n      recordConnectionSuccessMetric();\n      recordConnectionSuccess();\n    } else {\n      recordConnectionFailure('protocol_version');\n    }\n\n    if (params.initConnectionMsg) {\n      this.#lc.debug?.(\n        'handling init connection message from sec header',\n        params.clientGroupID,\n        params.clientID,\n      );\n      await connection.handleInitConnection(\n        JSON.stringify(params.initConnectionMsg),\n      );\n    }\n  };\n\n  readonly #receiveConnection = async (\n    ws: WebSocket,\n    params: ConnectParams,\n  ) => {\n    try {\n      await this.#createConnection(ws, params);\n    } finally {\n      this.#releasePending(params.clientGroupID, params.generation);\n    }\n  };\n\n  #releasePending(clientGroupID: string, generation: number | undefined): void {\n    const remaining = (this.#pendingConnections.get(clientGroupID) ?? 1) - 1;\n    if (remaining <= 0) {\n      this.#pendingConnections.delete(clientGroupID);\n      if (!this.#viewSyncers.hasService(clientGroupID)) {\n        const gen =\n          this.#clientGroupGenerations.get(clientGroupID) ?? generation;\n        this.#parent.send<ClientGroupStatusMessage>([\n          'clientGroupStatus',\n          {\n            clientGroupID,\n            active: false,\n            ...(gen !== undefined ? {generation: gen} : {}),\n          },\n        ]);\n        this.#clientGroupGenerations.delete(clientGroupID);\n      }\n    } else {\n      this.#pendingConnections.set(clientGroupID, remaining);\n    }\n  }\n\n  run() {\n    return this.#stopped.promise;\n  }\n\n  /**\n   * Graceful shutdown involves shutting down view syncers one at a time, pausing\n   * for the duration of view syncer's hydration between each one. This paces the\n   * disconnects to avoid creating a backlog of hydrations in the receiving server\n   * when the clients reconnect.\n   */\n  async drain() {\n    const start = Date.now();\n    this.#lc.info?.(`draining ${this.#viewSyncers.size} view-syncers`);\n\n    this.#drainCoordinator.drainNextIn(0);\n\n    while (this.#viewSyncers.size) {\n      await this.#drainCoordinator.forceDrainTimeout;\n\n      // Pick an arbitrary view syncer to force drain.\n      for (const vs of this.#viewSyncers.getServices()) {\n        this.#lc.debug?.(`draining view-syncer ${vs.id} (forced)`);\n        // When this drain or an elective drain completes, the forceDrainTimeout will\n        // resolve after the next drain interval.\n        void vs.stop();\n        break;\n      }\n    }\n    this.#lc.info?.(`finished draining (${Date.now() - start} ms)`);\n  }\n\n  stop() {\n    clearInterval(this.#viewSyncerLagSampleInterval);\n    clearInterval(this.#eligibilityLogInterval);\n    this.#wss.close();\n    this.#stopped.resolve();\n    return promiseVoid;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;AA4FA,SAAgB,0BACd,aACoB;CACpB,IAAI,cAAc;CAClB,IAAI,gBAAgB;CACpB,MAAM,+BAAe,IAAI,IAGvB;CACF,MAAM,iCAAiB,IAAI,IAAY;CACvC,MAAM,6BAAa,IAAI,IAAY;CACnC,IAAI,UAAU;CACd,KAAK,MAAM,MAAM,aAAa;EAC5B,MAAM,YAAY,GAAG;EACrB,IAAI,cAAc,KAAA,GAChB,WAAW,IAAI,SAAS;EAM1B,MAAM,iBAAiB,aAAa,MAAM;EAC1C;EACA,KAAK,MAAM,EACT,oBACA,UACA,eACG,GAAG,eAAe,GAAG;GAGxB,MAAM,MAAM,GAAG,eAAe,GAAG;GACjC,IAAI,UAAU;IACZ;IACA,eAAe,IAAI,GAAG;GACxB,OAAO;IACL;IACA,MAAM,QAAQ,aAAa,IAAI,GAAG;IAClC,IAAI,OACF,MAAM;SAEN,aAAa,IAAI,KAAK;KAAC,OAAO;KAAG;IAAS,CAAC;GAE/C;EACF;CACF;CACA,OAAO;EACL;EACA;EACA;EACA,gBAAgB,eAAe;EAC/B,eAAe,WAAW;CAC5B;AACF;AAEA,IAAa,2BAA2B;AACxC,IAAM,qCAAqC;AAC3C,IAAM,6CAA6C,IAAI;AAEvD,IAAM,6BAA6B;AACnC,IAAM,2BAA2B;AAEjC,SAAS,wBACP,oBACM;CACN,IAAI,mBAAmB,SAAA,KACrB,mBAAmB,OACjB,GACA,mBAAmB,SAAS,wBAC9B;AAEJ;AAEA,SAAS,wBACP,oBACA,kBACM;CACN,IAAI,mBAAmB,GACrB,mBAAmB,OAAO,GAAG,gBAAgB;CAG/C,wBAAwB,kBAAkB;AAC5C;AAEA,SAAS,6BACP,oBACA,oBACQ;CACR,IAAI,MAAM;CACV,IAAI,OAAO,mBAAmB;CAC9B,OAAO,MAAM,MAAM;EACjB,MAAM,MAAM,MAAM,KAAK,OAAO,OAAO,OAAO,CAAC;EAC7C,IAAI,mBAAmB,KAAK,qBAAqB,oBAC/C,MAAM,MAAM;OAEZ,OAAO;CAEX;CACA,OAAO;AACT;AAEA,SAAS,oBACP,oBACA,WACQ;CACR,IAAI,MAAM;CACV,IAAI,OAAO,mBAAmB;CAC9B,OAAO,MAAM,MAAM;EACjB,MAAM,MAAM,MAAM,KAAK,OAAO,OAAO,OAAO,CAAC;EAC7C,IAAI,mBAAmB,KAAK,aAAa,WACvC,MAAM,MAAM;OAEZ,OAAO;CAEX;CACA,OAAO;AACT;AAEA,SAAS,uBACP,oBACA,YACQ;CACR,MAAM,0BAA0B,6BAC9B,oBACA,WAAW,WACb;CACA,MAAM,0BACJ,WAAW,kBAAkB,OACzB,IACA,oBAAoB,oBAAoB,WAAW,aAAa;CAEtE,MAAM,qBAAqB,KAAK,IAC9B,yBACA,uBACF;CACA,OAAO,qBAAqB,mBAAmB,SAC3C,qBACA;AACN;AAEA,SAAS,sBACP,cACA,YACQ;CACR,IAAI,aAAa,WAAW,GAC1B,OAAO;CAMT,OAAO,aAJO,KAAK,IACjB,aAAa,SAAS,GACtB,KAAK,IAAI,GAAG,KAAK,KAAM,aAAa,MAAO,aAAa,MAAM,IAAI,CAAC,CAEjD;AACtB;AAEA,SAAS,gCACP,KACA,oBACA,aACwB;CACxB,MAAM,OAAiB,CAAC;CACxB,IAAI,sBAAsB;CAC1B,IAAI,mBAAmB,mBAAmB;CAE1C,KAAK,MAAM,cAAc,aAAa;EACpC,IAAI,CAAC,WAAW,oBACd;EAGF,MAAM,qBAAqB,uBACzB,oBACA,UACF;EACA,IAAI,uBAAuB,IAAI;GAC7B,KAAK,KAAK,CAAC;GACX;EACF;EAEA,mBAAmB,KAAK,IAAI,kBAAkB,kBAAkB;EAChE,MAAM,QAAQ,KAAK,IACjB,GACA,MAAM,mBAAmB,oBAAoB,kBAC/C;EACA,KAAK,KAAK,KAAK;EACf,IAAI,QAAQ,GACV;CAEJ;CAEA,wBAAwB,oBAAoB,gBAAgB;CAE5D,KAAK,MAAM,GAAG,MAAM,IAAI,CAAC;CACzB,OAAO;EACL,oBAAoB,KAAK;EACzB;EAIA,OAAO,KAAK,MAAM;EAClB,OAAO,sBAAsB,MAAM,EAAE;EACrC,OAAO,sBAAsB,MAAM,EAAE;EACrC,OAAO,sBAAsB,MAAM,EAAE;EACrC,OAAO,KAAK,GAAG,EAAE,KAAK;EACtB,QAAQ;CACV;AACF;AA+BA,SAAS,0BAA0B,QAAmC;CACpE,MAAM,UAAyB;EAC7B,UAAU;EACV,YAAY,OAAO;CACrB;CAEA,IAAI,OAAO,sBAAsB;EAC/B,QAAQ,oBAAoB;EAE5B,IAAI,OAAO,6BACT,IAAI;GAIF,QAAQ,oBAHmB,KAAK,MAC9B,OAAO,2BAEmB;EAC9B,SAAS,GAAG;GACV,MAAM,IAAI,MACR,uDAAuD,OAAO,CAAC,EAAE,uBACnE;EACF;CAEJ;CAEA,OAAO;AACT;;;;;;;;AASA,IAAa,SAAb,MAAgD;CAC9C,KAAc,UAAU;CACxB;CACA;CACA;CACA;CACA,+BAAwB,IAAI,IAAwB;CACpD,sCAA+B,IAAI,IAAoB;CACvD,0CAAmC,IAAI,IAAoB;CAC3D,oBAA6B,IAAI,iBAAiB;CAClD;CACA;CACA,WAAoB,SAAS;CAC7B;CACA;CACA,sBAAoD,CAAC;CACrD,4BAAqC,yBACnC,QACA,8BACA,oCACF;CACA,+BAAwC,mBACtC,QACA,iCACA,uCACF;CACA,gCAAyC,mBACvC,QACA,kCACA,wDACF;CACA,+BAAwC,mBACtC,QACA,iCACA,yEACF;CACA,iBAA0B,2BACxB,QACA,mBACA;EACE,aACE;EAGF,MAAM;CACR,CACF;CACA;CACA,0CAA0C;CAC1C;CACA;CAEA,YACE,IACA,QACA,mBAKA,gBACA,eAMA,QACA,mBACA;EACA,KAAKW,UAAU;EACf,KAAKC,qBAAqB;EAG1B,MAAM,WAAW,mBAAmB,IAAI,SAAQ,UAC9C,KAAKO,yBAAyB,KAAK,CACrC;EACA,YAAY,IAAI,MAAM;EAEtB,KAAKnB,MAAM;EACX,KAAKQ,UAAU;EACf,KAAKP,eAAe,IAAI,cACtB,KACA,OAAM,kBAAkB,IAAI,SAAS,UAAU,GAAG,KAAKM,iBAAiB,IACxE,MAAK,EAAE,UAAU,IACjB,OAAM;GACJ,KAAK,KAAKF,oBAAoB,IAAI,EAAE,KAAK,OAAO,GAAG;IACjD,MAAM,aAAa,KAAKC,wBAAwB,IAAI,EAAE;IACtD,KAAKE,QAAQ,KAA+B,CAC1C,qBACA;KACE,eAAe;KACf,QAAQ;KACR,GAAI,eAAe,KAAA,IAAY,EAAC,WAAU,IAAI,CAAC;IACjD,CACF,CAAC;IACD,KAAKF,wBAAwB,OAAO,EAAE;GACxC;EACF,CACF;EACA,IAAI,gBACF,KAAKJ,YAAY,IAAI,cAAc,IAAI,iBAAgB,MAAK,EAAE,QAAQ,CAAC;EAEzE,IAAI,eACF,KAAKC,WAAW,IAAI,cAClB,KACA,OACE,cACE,IACA,KAAKF,aAAa,WAAW,EAAE,EAAE,kBACnC,IACF,MAAK,EAAE,QAAQ,CACjB;EAEF,KAAKQ,OAAO,IAAI,gBAAgB,0BAA0B,MAAM,CAAC;EAEjE,yBACE,IACA,KAAKA,MACL,KAAKW,oBACL,KAAKZ,UACL,WAAU;GACR,KAAKa,gBAAgB,OAAO,eAAe,OAAO,UAAU;EAC9D,IACA,WAAU;GACR,KAAKhB,oBAAoB,IACvB,OAAO,gBACN,KAAKA,oBAAoB,IAAI,OAAO,aAAa,KAAK,KAAK,CAC9D;GACA,IAAI,OAAO,eAAe,KAAA,GAAW;IACnC,MAAM,UAAU,KAAKC,wBAAwB,IAC3C,OAAO,aACT;IACA,IAAI,YAAY,KAAA,KAAa,OAAO,aAAa,SAC/C,KAAKA,wBAAwB,IAC3B,OAAO,eACP,OAAO,UACT;GAEJ;EACF,CACF;EAEA,kCAAkC,KAAKL,aAAa,IAAI;EAExD,iBACE,QACA,wBACA,gCACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKA,aAAa,IAAI,CAAC;EAE9D,iBACE,QACA,WACA,qDACF,EAAE,aAAY,WAAU;GACtB,IAAI,QAAQ;GACZ,KAAK,MAAM,MAAM,KAAKA,aAAa,YAAY,GAC7C,SAAS,GAAG;GAEd,OAAO,QAAQ,KAAK;EACtB,CAAC;EAED,iBACE,QACA,QACA,uCACF,EAAE,aAAY,WAAU;GACtB,IAAI,QAAQ;GACZ,KAAK,MAAM,MAAM,KAAKA,aAAa,YAAY,GAC7C,SAAS,GAAG;GAEd,OAAO,QAAQ,KAAK;EACtB,CAAC;EAED,iBAAiB,QAAQ,eAAe;GACtC,aACE;GAGF,MAAM;EACR,CAAC,EAAE,aAAY,WAAU;GACvB,OAAO,QAAQ,KAAKqB,+BAA+B,EAAE,KAAK;EAC5D,CAAC;EAED,iBAAiB,QAAQ,qBAAqB;GAC5C,aACE;GAGF,MAAM;EACR,CAAC,EAAE,aAAY,WAAU;GACvB,MAAM,QAAQ,KAAKA,+BAA+B;GAClD,OAAO,QAAQ,MAAM,OAAO,EAAC,MAAM,MAAK,CAAC;GACzC,OAAO,QAAQ,MAAM,OAAO,EAAC,MAAM,MAAK,CAAC;GACzC,OAAO,QAAQ,MAAM,OAAO,EAAC,MAAM,MAAK,CAAC;GACzC,OAAO,QAAQ,MAAM,OAAO,EAAC,MAAM,MAAK,CAAC;GACzC,OAAO,QAAQ,MAAM,OAAO,EAAC,MAAM,MAAK,CAAC;EAC3C,CAAC;EAED,iBACE,QACA,iCACA,8DACF,EAAE,aAAY,WAAU;GACtB,OAAO,QAAQ,KAAKA,+BAA+B,EAAE,mBAAmB;EAC1E,CAAC;EAED,iBACE,QACA,mBACA,0KAGF,EAAE,aAAY,WAAU;GACtB,MAAM,QAAQ,KAAKC,2BAA2B;GAC9C,OAAO,QAAQ,MAAM,aAAa,EAAC,MAAM,SAAQ,CAAC;GAClD,OAAO,QAAQ,MAAM,eAAe,EAAC,MAAM,WAAU,CAAC;EACxD,CAAC;EAED,iBACE,QACA,oBACA,8GAEF,EAAE,aAAY,WAAU;GACtB,MAAM,QAAQ,KAAKA,2BAA2B;GAC9C,OAAO,QAAQ,MAAM,aAAa,MAAM,EAAC,MAAM,SAAQ,CAAC;GACxD,OAAO,QAAQ,MAAM,gBAAgB,EAAC,MAAM,WAAU,CAAC;EACzD,CAAC;EAED,iBACE,QACA,kBACA,iEACF,EAAE,aAAY,WAAU;GACtB,OAAO,QAAQ,KAAKA,2BAA2B,EAAE,aAAa;EAChE,CAAC;EAED,KAAKC,+BAA+B,kBAC5B,KAAKC,4BAA4B,GACvC,kCACF;EACA,KAAKD,6BAA6B,QAAQ;EAE1C,KAAKE,0BAA0B,kBACvB,KAAKC,6BAA6B,GACxC,0CACF;EACA,KAAKD,wBAAwB,QAAQ;CACvC;CAEA,iCAAyD;EACvD,IAAI,KAAKE,8BACP,OAAO,KAAKA;EAGd,MAAM,eAAe,gCACnB,KAAK,IAAI,GACT,KAAKf,qBACL,KAAKZ,aAAa,YAAY,CAChC;EACA,KAAK2B,+BAA+B;EACpC,IAAI,CAAC,KAAKC,yCAAyC;GACjD,KAAKA,0CAA0C;GAC/C,qBAAqB;IACnB,KAAKD,+BAA+B,KAAA;IACpC,KAAKC,0CAA0C;GACjD,CAAC;EACH;EACA,OAAO;CACT;CAEA,8BAAoC;EAClC,MAAM,EAAC,WAAU,KAAKP,+BAA+B;EACrD,KAAK,MAAM,SAAS,QAClB,KAAKJ,eAAe,SAAS,KAAK;CAEtC;CAEA,6BAAiD;EAC/C,OAAO,0BAA0B,KAAKjB,aAAa,YAAY,CAAC;CAClE;CAEA,+BAAqC;EACnC,MAAM,QAAQ,KAAKsB,2BAA2B;EAC9C,IAAI,MAAM,gBAAgB,GACxB;EAEF,MAAM,SAAS,MAAM,aAAa;EAClC,MAAM,gBAAgB,CAAC,GAAG,MAAM,aAAa,QAAQ,CAAC,EACnD,QAAQ,GAAG,EAAC,aAAY,QAAQ,CAAC,EACjC,MAAM,GAAG,MAAM,EAAE,GAAG,QAAQ,EAAE,GAAG,KAAK,EACtC,MAAM,GAAG,EAAE,EACX,KAAK,CAAC,KAAK,EAAC,OAAO,kBAAiB;GAAC;GAAK;GAAO;EAAS,EAAE;EAC/D,KAAKvB,IAAI,OAAO,8BAA8B;GAC5C,iBAAiB,MAAM;GACvB,uBAAuB;GACvB,aAAa,QAAQ,MAAM,cAAc,KAAK,IAAI,QAAQ,CAAC,GAAG,QAAQ,CAAC,CAAC;GACxE,mBAAmB,MAAM;GACzB,eAAe,MAAM;GACrB;EACF,CAAC;CACH;CAEA,yBAAyB,OAA2B;EAClD,IACE,MAAM,cAAc,KAAA,KACpB,MAAM,uBAAuB,KAAA,GAE7B;EAEF,MAAM,OAAO,KAAKa,oBAAoB,GAAG,EAAE;EAC3C,IAAI,QAAQ,KAAK,aAAa,MAAM,WAClC;EAEF,KAAKA,oBAAoB,KAAK;GAC5B,WAAW,MAAM;GACjB,oBAAoB,MAAM;EAC5B,CAAC;EACD,IAAI,KAAKZ,aAAa,SAAS,GAAG;GAChC,KAAKY,oBAAoB,SAAS;GAClC;EACF;EACA,wBAAwB,KAAKA,mBAAmB;CAClD;CAEA,oBAA6B,OAAO,IAAe,WAA0B;EAC3E,MAAM,mBAAmB,GACtB,6BAA6B,OAAO,gBACvC;EACA,IAAI,2BAA2B;EAC/B,MAAM,2BAA2B,WAAmB;GAClD,IAAI,0BACF;GAEF,2BAA2B;GAC3B,KAAKI,6BAA6B,IAAI,GAAG;IACvC,GAAG;KACF,2BAA2B;GAC9B,CAAC;EACH;EACA,MAAM,sCAAsC;GAC1C,IAAI,0BACF;GAEF,2BAA2B;GAC3B,KAAKD,8BAA8B,IAAI,GAAG,gBAAgB;EAC5D;EAEA,KAAKD,6BAA6B,IAAI,GAAG,gBAAgB;EACzD,KAAKD,0BAA0B,IAAI,GAAG,gBAAgB;EACtD,GAAG,KAAK,eAAe;GACrB,KAAKA,0BAA0B,IAAI,IAAI,gBAAgB;EACzD,CAAC;EAED,KAAKd,IAAI,QACP,uBACA,OAAO,eACP,OAAO,QACT;EACA,0BAA0B;EAC1B,MAAM,EAAC,UAAU,eAAe,MAAM,WAAU;EAChD,MAAM,kBAAkB,SAAS,KAAA,KAAa,SAAS;EACvD,MAAM,iBAAiB,UAAU;EAEjC,IAAI,iBAAiB;GACnB,MAAM,eAAe,mBAAmB,KAAKW,QAAQ,QAAQ,CAAC,CAAC;GAE/D,MAAM,kBACJ,KAAKA,SAAS,MAAM,QAAQ,KAAA,KAC5B,KAAKA,SAAS,QAAQ,QAAQ,KAAA;GAChC,MAAM,aACJ,KAAKA,SAAS,OAAO,QAAQ,KAAA,KAC7B,KAAKA,SAAS,YAAY,QAAQ,KAAA;GAKpC,IAAI,EAF6B,aAAa,WAAW,MAExB,EADN,mBAAmB,aACQ;IACpD,wBAAwB,eAAe;IACvC,MAAM,IAAI,MACR,uHACE,KAAK,UAAU,YAAY,IAC3B,+IACJ;GACF;EACF;EAEA,IAAI;EAIJ,IAAI;GACF,cAAc,MAAM,YAClB,KAAKX,IACF,YAAY,iBAAiB,aAAa,EAC1C,YAAY,YAAY,QAAQ,GAGnC,KAAA,GACA,gBACA,MACA,KAAKY,kBACP;EACF,SAAS,GAAG;GACV,IAAI,gBAAgB,CAAC,GAAG;IACtB,KAAKZ,IAAI,OACP,4DACA;KACE;KACA;KACA;KACA;KACA,WAAW,EAAE;IACf,CACF;IACA,wBAAwB,MAAM;IAC9B,UAAU,KAAKA,KAAK,IAAI,EAAE,SAAS;IACnC,GAAG,MAAM,KAAM,EAAE,UAAU,OAAO;IAClC;GACF;GACA,wBAAwB,UAAU;GAClC,MAAM;EACR;EAOA,IAAI,GAAG,eAAe,UAAU,MAAM;GACpC,KAAKA,IAAI,QAAQ,yCAAyC;IACxD;IACA;IACA,YAAY,GAAG;GACjB,CAAC;GACD,wBAAwB,oBAAoB;GAC5C;EACF;EAEA,MAAM,aAAa,KAAKC,aAAa,WAAW,aAAa;EAC7D,MAAM,qBAAqB,WAAW;EACtC,MAAM,QAAQ,mBAAmB,cAAc;EAQ/C,IACE,MAAM,eAAe,KAAA,KACrB,MAAM,WAAW,OAAO,gBACxB;GACA,MAAM,QAAQ,IAAI,cAAc;IAC9B,MAAM;IACN,SACE;IACF,QAAQ;GACV,CAAC;GACD,wBAAwB,eAAe;GACvC,UAAU,KAAKD,KAAK,IAAI,MAAM,SAAS;GACvC,GAAG,MAAM,KAAM,MAAM,OAAO;GAC5B;EACF;EAGA,MAAM,WAAW,KAAKI,aAAa,IAAI,QAAQ;EAC/C,IAAI,UAAU;GACZ,KAAKJ,IAAI,QACP,UAAU,SAAS,gDACrB;GACA,SAAS,MAAM,eAAe,OAAO,MAAM;EAC7C;EAEA,mBAAmB,mBACjB;GAAC;GAAU,MAAM,OAAO;EAAI,GAC5B,QACA,WACF;EAEA,MAAM,UAAU,KAAKE,WAAW,WAAW,aAAa;EACxD,MAAM,SAAS,KAAKC,UAAU,WAAW,aAAa;EAEtD,SAAS,IAAI;EACb,QAAQ,IAAI;EAEZ,IAAI;EACJ,IAAI;GACF,aAAa,IAAI,WACf,KAAKH,KACL,QACA,IACA,KAAKW,QAAQ,oBACb,IAAI,uBACF,KAAKX,KACL,QACA,oBACA,YACA,SACA,MACF,SACM;IACJ,mBAAmB,gBAAgB;KACjC;KACA,MAAM,OAAO;IACf,CAAC;IACD,IAAI,KAAKI,aAAa,IAAI,QAAQ,MAAM,YACtC,KAAKA,aAAa,OAAO,QAAQ;IAInC,SAAS,MAAM;IACf,QAAQ,MAAM;GAChB,CACF;EACF,SAAS,GAAG;GACV,wBAAwB,UAAU;GAClC,mBAAmB,gBAAgB;IAAC;IAAU,MAAM,OAAO;GAAI,CAAC;GAChE,SAAS,MAAM;GACf,QAAQ,MAAM;GACd,MAAM;EACR;EAEA,KAAKA,aAAa,IAAI,UAAU,UAAU;EAE1C,IAAI,WAAW,KAAK,GAAG;GACrB,8BAA8B;GAC9B,wBAAwB;EAC1B,OACE,wBAAwB,kBAAkB;EAG5C,IAAI,OAAO,mBAAmB;GAC5B,KAAKJ,IAAI,QACP,oDACA,OAAO,eACP,OAAO,QACT;GACA,MAAM,WAAW,qBACf,KAAK,UAAU,OAAO,iBAAiB,CACzC;EACF;CACF;CAEA,qBAA8B,OAC5B,IACA,WACG;EACH,IAAI;GACF,MAAM,KAAK8B,kBAAkB,IAAI,MAAM;EACzC,UAAU;GACR,KAAKT,gBAAgB,OAAO,eAAe,OAAO,UAAU;EAC9D;CACF;CAEA,gBAAgB,eAAuB,YAAsC;EAC3E,MAAM,aAAa,KAAKhB,oBAAoB,IAAI,aAAa,KAAK,KAAK;EACvE,IAAI,aAAa,GAAG;GAClB,KAAKA,oBAAoB,OAAO,aAAa;GAC7C,IAAI,CAAC,KAAKJ,aAAa,WAAW,aAAa,GAAG;IAChD,MAAM,MACJ,KAAKK,wBAAwB,IAAI,aAAa,KAAK;IACrD,KAAKE,QAAQ,KAA+B,CAC1C,qBACA;KACE;KACA,QAAQ;KACR,GAAI,QAAQ,KAAA,IAAY,EAAC,YAAY,IAAG,IAAI,CAAC;IAC/C,CACF,CAAC;IACD,KAAKF,wBAAwB,OAAO,aAAa;GACnD;EACF,OACE,KAAKD,oBAAoB,IAAI,eAAe,SAAS;CAEzD;CAEA,MAAM;EACJ,OAAO,KAAKK,SAAS;CACvB;;;;;;;CAQA,MAAM,QAAQ;EACZ,MAAM,QAAQ,KAAK,IAAI;EACvB,KAAKV,IAAI,OAAO,YAAY,KAAKC,aAAa,KAAK,cAAc;EAEjE,KAAKM,kBAAkB,YAAY,CAAC;EAEpC,OAAO,KAAKN,aAAa,MAAM;GAC7B,MAAM,KAAKM,kBAAkB;GAG7B,KAAK,MAAM,MAAM,KAAKN,aAAa,YAAY,GAAG;IAChD,KAAKD,IAAI,QAAQ,wBAAwB,GAAG,GAAG,UAAU;IAGzD,GAAQ,KAAK;IACb;GACF;EACF;EACA,KAAKA,IAAI,OAAO,sBAAsB,KAAK,IAAI,IAAI,MAAM,KAAK;CAChE;CAEA,OAAO;EACL,cAAc,KAAKwB,4BAA4B;EAC/C,cAAc,KAAKE,uBAAuB;EAC1C,KAAKjB,KAAK,MAAM;EAChB,KAAKC,SAAS,QAAQ;EACtB,OAAO;CACT;AACF"}