@rocicorp/zero 1.12.0-canary.12 → 1.12.0-canary.13

package/out/zero-cache/src/workers/connection.js.map

@@ -1 +1 @@
-{"version":3,"file":"connection.js","names":["valita.parse","ErrorKind.VersionNotSupported","ErrorOrigin.ZeroCache","ErrorKind.InvalidMessage","WebSocket","ErrorKind.Internal"],"sources":["../../../../../zero-cache/src/workers/connection.ts"],"sourcesContent":["import {pipeline, Readable, Writable} from 'node:stream';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport type {CloseEvent, Data, ErrorEvent} from 'ws';\nimport WebSocket, {createWebSocketStream} from 'ws';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport * as valita from '../../../shared/src/valita.ts';\nimport type {ConnectedMessage} from '../../../zero-protocol/src/connect.ts';\nimport type {Downstream} from '../../../zero-protocol/src/down.ts';\nimport {ErrorKind} from '../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../zero-protocol/src/error-origin.ts';\nimport type {ErrorBody} from '../../../zero-protocol/src/error.ts';\nimport {\n  isProtocolError,\n  type ProtocolError,\n} from '../../../zero-protocol/src/error.ts';\nimport {\n  POKE_CHUNK_PROTOCOL_VERSION,\n  type PokeChunk,\n  type PokeEndMessage,\n  type PokePartMessage,\n} from '../../../zero-protocol/src/poke.ts';\nimport {\n  MIN_SERVER_SUPPORTED_SYNC_PROTOCOL,\n  PROTOCOL_VERSION,\n} from '../../../zero-protocol/src/protocol-version.ts';\nimport {\n  upstreamSchemaWithUnparsedAnalyzeQuery,\n  type UpstreamWithUnparsedAnalyzeQuery,\n} from '../../../zero-protocol/src/up.ts';\nimport {getOrCreateCounter} from '../observability/metrics.ts';\nimport type {ViewSyncerDownstream} from '../types/downstream.ts';\nimport {\n  ProtocolErrorWithLevel,\n  getLogLevel,\n  wrapWithProtocolError,\n} from '../types/error-with-level.ts';\nimport {PokeChunkEncoder} from '../types/poke-chunk.ts';\nimport type {Source} from '../types/streams.ts';\nimport type {ConnectParams} from './connect-params.ts';\n\nexport type HandlerResult =\n  | {\n      type: 'ok';\n    }\n  | {\n      type: 'fatal';\n      error: ErrorBody;\n    }\n  | {\n      type: 'transient';\n      errors: ErrorBody[];\n    }\n  | StreamResult;\n\nexport type StreamResult =\n  | {\n      type: 'stream';\n      source: 'viewSyncer';\n      stream: Source<ViewSyncerDownstream>;\n    }\n  | {\n      type: 'stream';\n      source: 'pusher';\n      stream: Source<Downstream>;\n    };\n\nexport interface MessageHandler {\n  handleMessage(\n    msg: UpstreamWithUnparsedAnalyzeQuery,\n  ): Promise<HandlerResult[]>;\n}\n\nfunction hasOwn(value: unknown, property: string): boolean {\n  return (\n    typeof value === 'object' &&\n    value !== null &&\n    Object.hasOwn(value, property)\n  );\n}\n\nfunction containsLegacyQuery(message: unknown): boolean {\n  if (!Array.isArray(message)) {\n    return false;\n  }\n\n  const body = message[1];\n  if (typeof body !== 'object' || body === null) {\n    return false;\n  }\n\n  if (\n    message[0] !== 'initConnection' &&\n    message[0] !== 'changeDesiredQueries'\n  ) {\n    return false;\n  }\n\n  const patch = body['desiredQueriesPatch'];\n  return (\n    Array.isArray(patch) && patch.some(operation => hasOwn(operation, 'ast'))\n  );\n}\n\nconst LEGACY_QUERIES_DISABLED_MESSAGE =\n  'Legacy queries are disabled by this Zero server. This client must use custom queries instead of sending query ASTs directly. To temporarily restore compatibility, set ZERO_ALLOW_LEGACY_QUERIES=true on zero-cache.';\n\n// Exported for testing purposes.\n//\n// Client messages are parsed leniently, so a newer client can add message\n// kinds and fields:\n// - A message of a kind this server doesn't know parses as an\n//   UnrecognizedTuple, for the connection to drop. Malformed messages still\n//   throw.\n// - Messages are parsed in strip mode, so fields this server doesn't know are\n//   dropped. Strip rather than passthrough: nothing this server doesn't know\n//   reaches the CVR or the API server, and valita's passthrough drops unknown\n//   fields anyway on any object with a transformed field below it (such as\n//   jsonSchema args). ASTs stay strict in the schema, because a server that\n//   ignored an AST field would run a different query.\nexport function parseUpstreamMessage(\n  value: unknown,\n  allowLegacyQueries: boolean,\n): UpstreamWithUnparsedAnalyzeQuery | valita.UnrecognizedTuple {\n  if (!allowLegacyQueries && containsLegacyQuery(value)) {\n    throw new Error(LEGACY_QUERIES_DISABLED_MESSAGE);\n  }\n  return valita.parse(value, upstreamSchemaWithUnparsedAnalyzeQuery, 'strip');\n}\n\n// Ensures that a downstream message is sent at least every interval, sending a\n// 'pong' if necessary. This is set to be slightly longer than the client-side\n// PING_INTERVAL of 5 seconds, so that in the common case, 'pong's are sent in\n// response to client-initiated 'ping's. However, if the inbound stream is\n// backed up because a command is taking a long time to process, the pings\n// will be stuck in the queue (i.e. back-pressured), in which case pongs will\n// be manually sent to notify the client of server liveness.\n//\n// This is equivalent to what is done for Postgres keepalives on the\n// replication stream (which can similarly be back-pressured):\n// https://github.com/rocicorp/mono/blob/f98cb369a2dbb15650328859c732db358f187ef0/packages/zero-cache/src/services/change-source/pg/logical-replication/stream.ts#L21\nconst DOWNSTREAM_MSG_INTERVAL_MS = 6_000;\nexport const WEBSOCKET_SEND_TIMEOUT_MS = 10_000;\nconst PROTOCOL_VERSION_ATTRIBUTE = 'protocol.version';\nconst EVENT_TYPE_ATTRIBUTE = 'event.type';\n\n/**\n * Represents a connection between the client and server.\n *\n * Handles incoming messages on the connection and dispatches\n * them to the correct service.\n *\n * Listens to the ViewSyncer and sends messages to the client.\n */\nexport class Connection {\n  readonly #ws: WebSocket;\n  readonly #wsID: string;\n  readonly #protocolVersion: number;\n  readonly #lc: LogContext;\n  readonly #onClose: () => void;\n  readonly #messageHandler: MessageHandler;\n  readonly #allowLegacyQueries: boolean;\n  readonly #downstreamSender: DownstreamSender;\n  readonly #downstreamMsgTimer: NodeJS.Timeout | undefined;\n  readonly #webSocketErrors = getOrCreateCounter(\n    'sync',\n    'websocket.errors',\n    'Client WebSocket error events.',\n  );\n\n  #viewSyncerOutboundStream: Source<ViewSyncerDownstream> | undefined;\n  #pusherOutboundStream: Source<Downstream> | undefined;\n  #closed = false;\n  #loggedUnknownMessage = false;\n\n  constructor(\n    lc: LogContext,\n    connectParams: ConnectParams,\n    ws: WebSocket,\n    allowLegacyQueries: boolean,\n    messageHandler: MessageHandler,\n    onClose: () => void,\n  ) {\n    const {clientGroupID, clientID, wsID, protocolVersion} = connectParams;\n    this.#messageHandler = messageHandler;\n    this.#allowLegacyQueries = allowLegacyQueries;\n\n    this.#ws = ws;\n    this.#wsID = wsID;\n    this.#protocolVersion = protocolVersion;\n\n    this.#lc = lc\n      .withContext('connection')\n      .withContext('clientID', clientID)\n      .withContext('clientGroupID', clientGroupID)\n      .withContext('wsID', wsID);\n    this.#downstreamSender = new DownstreamSender(\n      this.#lc,\n      ws,\n      protocolVersion,\n    );\n    this.#lc.debug?.('new connection');\n    this.#onClose = onClose;\n\n    this.#ws.addEventListener('close', this.#handleClose);\n    this.#ws.addEventListener('error', this.#handleError);\n\n    this.#proxyInbound();\n    this.#downstreamMsgTimer = setInterval(\n      this.#maybeSendPong,\n      DOWNSTREAM_MSG_INTERVAL_MS / 2,\n    );\n  }\n\n  /**\n   * Checks the protocol version and errors for unsupported protocols,\n   * sending the initial `connected` response on success.\n   *\n   * This is early in the connection lifecycle because {@link #handleMessage}\n   * will only parse messages with schema(s) of supported protocol versions.\n   */\n  init(): boolean {\n    if (\n      this.#protocolVersion > PROTOCOL_VERSION ||\n      this.#protocolVersion < MIN_SERVER_SUPPORTED_SYNC_PROTOCOL\n    ) {\n      this.#closeWithError({\n        kind: ErrorKind.VersionNotSupported,\n        message: `server is at sync protocol v${PROTOCOL_VERSION} and does not support v${\n          this.#protocolVersion\n        }. The ${\n          this.#protocolVersion > PROTOCOL_VERSION ? 'server' : 'client'\n        } must be updated to a newer release.`,\n        origin: ErrorOrigin.ZeroCache,\n      });\n    } else {\n      const connectedMessage: ConnectedMessage = [\n        'connected',\n        {wsid: this.#wsID, timestamp: Date.now()},\n      ];\n      this.send(connectedMessage, 'ignore-backpressure');\n      return true;\n    }\n    return false;\n  }\n\n  get wsID(): string {\n    return this.#wsID;\n  }\n\n  close(reason: string, ...args: unknown[]) {\n    if (this.#closed) {\n      return;\n    }\n    this.#closed = true;\n    this.#lc.info?.(`closing connection: ${reason}`, ...args);\n    this.#ws.removeEventListener('close', this.#handleClose);\n    this.#ws.removeEventListener('error', this.#handleError);\n    this.#viewSyncerOutboundStream?.cancel();\n    this.#viewSyncerOutboundStream = undefined;\n    this.#pusherOutboundStream?.cancel();\n    this.#pusherOutboundStream = undefined;\n    this.#onClose();\n    if (this.#ws.readyState !== this.#ws.CLOSED) {\n      this.#ws.close();\n    }\n    clearTimeout(this.#downstreamMsgTimer);\n\n    // spin down services if we have\n    // no more client connections for the client group?\n  }\n\n  handleInitConnection(initConnectionMsg: string) {\n    return this.#handleMessage({data: initConnectionMsg});\n  }\n\n  #handleMessage = async (event: {data: Data}) => {\n    const data = event.data.toString();\n    if (this.#closed) {\n      this.#lc.debug?.('Ignoring message received after closed', data);\n      return;\n    }\n\n    let msg;\n    try {\n      const value = JSON.parse(data);\n      msg = parseUpstreamMessage(value, this.#allowLegacyQueries);\n    } catch (e) {\n      const errorBody = {\n        kind: ErrorKind.InvalidMessage,\n        message: String(e),\n        origin: ErrorOrigin.ZeroCache,\n      } as const;\n      this.#closeWithError(\n        errorBody,\n        new ProtocolErrorWithLevel(errorBody, 'warn'),\n      );\n      return;\n    }\n\n    if (!upstreamSchemaWithUnparsedAnalyzeQuery.isKnown(msg)) {\n      // A message kind from a newer client.\n      if (!this.#loggedUnknownMessage) {\n        this.#loggedUnknownMessage = true;\n        this.#lc.info?.(\n          `Ignoring messages of unknown type, starting with ${JSON.stringify(msg[0])}`,\n        );\n      }\n      return;\n    }\n\n    try {\n      const msgType = msg[0];\n      if (msgType === 'ping') {\n        this.send(['pong', {}], 'ignore-backpressure');\n        return;\n      }\n\n      const result = await this.#messageHandler.handleMessage(msg);\n      for (const r of result) {\n        this.#handleMessageResult(r);\n      }\n    } catch (e) {\n      this.#closeWithThrown(e);\n    }\n  };\n\n  #handleMessageResult(result: HandlerResult): void {\n    switch (result.type) {\n      case 'fatal':\n        this.#closeWithError(result.error);\n        break;\n      case 'ok':\n        break;\n      case 'stream': {\n        switch (result.source) {\n          case 'viewSyncer':\n            assert(\n              this.#viewSyncerOutboundStream === undefined,\n              'Outbound stream already set for this connection!',\n            );\n            this.#viewSyncerOutboundStream = result.stream;\n            break;\n          case 'pusher':\n            assert(\n              this.#pusherOutboundStream === undefined,\n              'Outbound stream already set for this connection!',\n            );\n            this.#pusherOutboundStream = result.stream;\n            break;\n        }\n        if (result.source === 'viewSyncer') {\n          this.#proxyOutbound(result.stream, (downstream, callback) =>\n            this.send(downstream.message, callback, downstream.serialized),\n          );\n        } else {\n          this.#proxyOutbound(result.stream, (downstream, callback) =>\n            this.send(downstream, callback),\n          );\n        }\n        break;\n      }\n      case 'transient': {\n        for (const error of result.errors) {\n          this.sendError(error);\n        }\n      }\n    }\n  }\n\n  #handleClose = (e: CloseEvent) => {\n    const {code, reason, wasClean} = e;\n    if (!wasClean) {\n      this.#recordWebSocketError('unclean_close');\n    }\n    this.close('WebSocket close event', {code, reason, wasClean});\n  };\n\n  #handleError = (e: ErrorEvent) => {\n    this.#recordWebSocketError('error_event');\n    this.#lc.warn?.('WebSocket error event', e.message, e.error);\n  };\n\n  #recordWebSocketError(eventType: string) {\n    this.#webSocketErrors.add(1, {\n      [PROTOCOL_VERSION_ATTRIBUTE]: this.#protocolVersion,\n      [EVENT_TYPE_ATTRIBUTE]: eventType,\n    });\n  }\n\n  #proxyInbound() {\n    pipeline(\n      createWebSocketStream(this.#ws),\n      new Writable({\n        write: (data, _encoding, callback) => {\n          this.#handleMessage({data}).then(() => callback(), callback);\n        },\n      }),\n      // The done callback is not used, as #handleClose and #handleError,\n      // configured on the underlying WebSocket, provide more complete\n      // information.\n      () => {},\n    );\n  }\n\n  #proxyOutbound<T>(\n    outboundStream: Source<T>,\n    sendMessage: (\n      downstream: T,\n      callback: (err?: Error | null) => void,\n    ) => void,\n  ) {\n    // Note: createWebSocketStream() is avoided here in order to control\n    //       exception handling with #closeWithThrown(). If the Writable\n    //       from createWebSocketStream() were instead used, exceptions\n    //       from the outboundStream result in the Writable closing the\n    //       the websocket before the error message can be sent.\n    pipeline(\n      Readable.from(outboundStream),\n      new Writable({\n        objectMode: true,\n        write: (downstream: T, _encoding, callback) =>\n          sendMessage(downstream, callback),\n      }),\n      e =>\n        e\n          ? this.#closeWithThrown(e)\n          : this.close(`downstream closed by ViewSyncer`),\n    );\n  }\n\n  #closeWithThrown(e: unknown) {\n    const errorBody =\n      findProtocolError(e)?.errorBody ?? wrapWithProtocolError(e).errorBody;\n\n    this.#closeWithError(errorBody, e);\n  }\n\n  #closeWithError(errorBody: ErrorBody, thrown?: unknown) {\n    this.sendError(errorBody, thrown);\n    this.close(\n      `${errorBody.kind} (${errorBody.origin}): ${errorBody.message}`,\n      errorBody,\n    );\n  }\n\n  #lastDownstreamMsgTime = Date.now();\n\n  #maybeSendPong = () => {\n    if (Date.now() - this.#lastDownstreamMsgTime > DOWNSTREAM_MSG_INTERVAL_MS) {\n      this.#lc.debug?.('manually sending pong');\n      this.send(['pong', {}], 'ignore-backpressure');\n    }\n  };\n\n  send(\n    data: Downstream,\n    callback: ((err?: Error | null) => void) | 'ignore-backpressure',\n    serialized?: string | undefined,\n  ) {\n    this.#lastDownstreamMsgTime = Date.now();\n    this.#downstreamSender.send(data, callback, serialized);\n  }\n\n  sendError(errorBody: ErrorBody, thrown?: unknown) {\n    sendError(this.#lc, this.#ws, errorBody, thrown);\n  }\n}\n\nexport type WebSocketLike = Pick<WebSocket, 'readyState'> & {\n  send(data: string | PokeChunk, cb?: (err?: Error) => void): void;\n};\n\n// Exported for testing purposes.\nexport function send(\n  lc: LogContext,\n  ws: WebSocketLike,\n  data: Downstream,\n  callback: ((err?: Error | null) => void) | 'ignore-backpressure',\n  serialized?: string | undefined,\n) {\n  if (ws.readyState === WebSocket.OPEN) {\n    serialized ??= JSON.stringify(data);\n    if (callback === 'ignore-backpressure') {\n      ws.send(serialized);\n      return;\n    }\n\n    let completed = false;\n    const timer = setTimeout(() => {\n      complete(webSocketSendTimeoutError());\n    }, WEBSOCKET_SEND_TIMEOUT_MS);\n    timer.unref();\n\n    const complete = (error?: Error | null) => {\n      if (completed) {\n        return;\n      }\n      completed = true;\n      clearTimeout(timer);\n      callback(error);\n    };\n\n    ws.send(serialized, complete);\n  } else {\n    lc.debug?.(`Dropping outbound message on ws (state: ${ws.readyState})`, {\n      dropped: data,\n    });\n    if (callback !== 'ignore-backpressure') {\n      callback(\n        new ProtocolErrorWithLevel(\n          {\n            kind: ErrorKind.Internal,\n            message: 'WebSocket closed',\n            origin: ErrorOrigin.ZeroCache,\n          },\n          'info',\n        ),\n      );\n    }\n  }\n}\n\n/**\n * Sends downstream messages in the format supported by a connection's sync\n * protocol version. Keep version forks contained here so the view-syncer can\n * continue producing one canonical poke representation.\n *\n * Exported for compatibility testing.\n */\nexport class DownstreamSender {\n  readonly #lc: LogContext;\n  readonly #ws: WebSocketLike;\n  readonly #protocolVersion: number;\n  #pokeChunkEncoder: PokeChunkEncoder | undefined;\n\n  constructor(lc: LogContext, ws: WebSocketLike, protocolVersion: number) {\n    this.#lc = lc;\n    this.#ws = ws;\n    this.#protocolVersion = protocolVersion;\n  }\n\n  send(\n    data: Downstream,\n    callback: ((err?: Error | null) => void) | 'ignore-backpressure',\n    serialized?: string | undefined,\n  ): void {\n    if (this.#protocolVersion >= POKE_CHUNK_PROTOCOL_VERSION) {\n      switch (data[0]) {\n        case 'pokeStart':\n          assert(\n            this.#pokeChunkEncoder === undefined,\n            'cannot start a poke while another poke is in progress',\n          );\n          this.#pokeChunkEncoder = new PokeChunkEncoder();\n          break;\n        case 'pokePart':\n          this.#sendPokePart(data, callback, serialized);\n          return;\n        case 'pokeEnd':\n          this.#sendPokeEnd(data, callback);\n          return;\n      }\n    }\n    send(this.#lc, this.#ws, data, callback, serialized);\n  }\n\n  #sendPokePart(\n    pokePart: PokePartMessage,\n    callback: ((err?: Error | null) => void) | 'ignore-backpressure',\n    serialized?: string | undefined,\n  ): void {\n    const encoder = this.#pokeChunkEncoder;\n    assert(encoder, 'pokePart received without a pokeStart');\n\n    serialized ??= JSON.stringify(pokePart);\n    assert(\n      serialized.startsWith(POKE_PART_PREFIX) && serialized.endsWith(']'),\n      'invalid serialized pokePart',\n    );\n    void encoder\n      .addPatch(serialized.slice(POKE_PART_PREFIX.length, -1), chunk =>\n        sendBinary(this.#lc, this.#ws, chunk),\n      )\n      .then(\n        () => invokeCallback(callback),\n        error => invokeCallback(callback, toError(error)),\n      );\n  }\n\n  #sendPokeEnd(\n    pokeEnd: PokeEndMessage,\n    callback: ((err?: Error | null) => void) | 'ignore-backpressure',\n  ): void {\n    const encoder = this.#pokeChunkEncoder;\n    assert(encoder, 'pokeEnd received without a pokeStart');\n    this.#pokeChunkEncoder = undefined;\n\n    if (pokeEnd[1].cancel) {\n      encoder.cancel();\n      send(this.#lc, this.#ws, pokeEnd, callback);\n      return;\n    }\n\n    void encoder\n      .finish(chunk => sendBinary(this.#lc, this.#ws, chunk))\n      .then(() => sendAsync(this.#lc, this.#ws, pokeEnd))\n      .then(\n        () => invokeCallback(callback),\n        error => invokeCallback(callback, toError(error)),\n      );\n  }\n}\n\nconst POKE_PART_PREFIX = '[\"pokePart\",';\n\nfunction sendAsync(\n  lc: LogContext,\n  ws: WebSocketLike,\n  data: Downstream,\n): Promise<void> {\n  return new Promise((resolve, reject) => {\n    send(lc, ws, data, error => (error ? reject(error) : resolve()));\n  });\n}\n\nfunction sendBinary(\n  lc: LogContext,\n  ws: WebSocketLike,\n  chunk: PokeChunk,\n): Promise<void> {\n  return new Promise((resolve, reject) => {\n    if (ws.readyState !== WebSocket.OPEN) {\n      const error = webSocketClosedError();\n      lc.debug?.(\n        `Dropping outbound binary message on ws (state: ${ws.readyState})`,\n      );\n      reject(error);\n      return;\n    }\n    let completed = false;\n    const timer = setTimeout(\n      () => complete(webSocketSendTimeoutError()),\n      WEBSOCKET_SEND_TIMEOUT_MS,\n    );\n    timer.unref();\n\n    const complete = (error?: Error | null) => {\n      if (completed) {\n        return;\n      }\n      completed = true;\n      clearTimeout(timer);\n      error ? reject(error) : resolve();\n    };\n\n    ws.send(chunk, complete);\n  });\n}\n\nfunction invokeCallback(\n  callback: ((err?: Error | null) => void) | 'ignore-backpressure',\n  error?: Error,\n): void {\n  if (callback !== 'ignore-backpressure') {\n    callback(error);\n  }\n}\n\nfunction toError(error: unknown): Error {\n  return error instanceof Error ? error : new Error(String(error));\n}\n\nfunction webSocketClosedError(): ProtocolErrorWithLevel {\n  return new ProtocolErrorWithLevel(\n    {\n      kind: ErrorKind.Internal,\n      message: 'WebSocket closed',\n      origin: ErrorOrigin.ZeroCache,\n    },\n    'info',\n  );\n}\n\nfunction webSocketSendTimeoutError(): ProtocolErrorWithLevel {\n  return new ProtocolErrorWithLevel(\n    {\n      kind: ErrorKind.Internal,\n      message: `WebSocket send timed out after ${WEBSOCKET_SEND_TIMEOUT_MS} ms`,\n      origin: ErrorOrigin.ZeroCache,\n    },\n    'info',\n  );\n}\n\nexport function sendError(\n  lc: LogContext,\n  ws: WebSocket,\n  errorBody: ErrorBody,\n  thrown?: unknown,\n) {\n  lc = lc.withContext('errorKind', errorBody.kind);\n\n  let logLevel: LogLevel;\n\n  // If the thrown error is a ProtocolErrorWithLevel, its explicit logLevel takes precedence\n  if (thrown instanceof ProtocolErrorWithLevel) {\n    logLevel = thrown.logLevel;\n  }\n  // Errors with errno or transient socket codes are low-level, transient I/O issues\n  // (e.g., EPIPE, ECONNRESET) and should be warnings, not errors\n  else if (\n    hasErrno(thrown) ||\n    hasTransientSocketCode(thrown) ||\n    isTransientSocketMessage(errorBody.message)\n  ) {\n    logLevel = 'warn';\n  }\n  // Fallback: check errorBody.kind for errors that weren't thrown as ProtocolErrorWithLevel\n  else if (\n    errorBody.kind === ErrorKind.ClientNotFound ||\n    errorBody.kind === ErrorKind.TransformFailed\n  ) {\n    logLevel = 'warn';\n  } else {\n    logLevel = thrown ? getLogLevel(thrown) : 'info';\n  }\n\n  lc[logLevel]?.('Sending error on WebSocket', errorBody, thrown ?? '');\n  send(lc, ws, ['error', errorBody], 'ignore-backpressure');\n}\n\nfunction findProtocolError(error: unknown): ProtocolError | undefined {\n  if (isProtocolError(error)) {\n    return error;\n  }\n  if (error instanceof Error && error.cause) {\n    return findProtocolError(error.cause);\n  }\n  return undefined;\n}\n\nfunction hasErrno(error: unknown): boolean {\n  return Boolean(\n    error &&\n    typeof error === 'object' &&\n    'errno' in error &&\n    typeof (error as {errno: unknown}).errno !== 'undefined',\n  );\n}\n\n// System error codes that indicate transient socket conditions.\n// These are checked via the `code` property on errors.\nconst TRANSIENT_SOCKET_ERROR_CODES = new Set([\n  'EPIPE',\n  'ECONNRESET',\n  'ECANCELED',\n]);\n\n// Error messages that indicate transient socket conditions but don't have\n// standard error codes (e.g., WebSocket library errors).\nconst TRANSIENT_SOCKET_MESSAGE_PATTERNS = [\n  'socket was closed while data was being compressed',\n];\n\nfunction hasTransientSocketCode(error: unknown): boolean {\n  if (!error || typeof error !== 'object') {\n    return false;\n  }\n  const maybeCode =\n    'code' in error ? String((error as {code?: unknown}).code) : undefined;\n  return Boolean(\n    maybeCode && TRANSIENT_SOCKET_ERROR_CODES.has(maybeCode.toUpperCase()),\n  );\n}\n\nfunction isTransientSocketMessage(message: string | undefined): boolean {\n  if (!message) {\n    return false;\n  }\n  const lower = message.toLowerCase();\n  return TRANSIENT_SOCKET_MESSAGE_PATTERNS.some(pattern =>\n    lower.includes(pattern),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;AAwEA,SAAS,OAAO,OAAgB,UAA2B;CACzD,OACE,OAAO,UAAU,YACjB,UAAU,QACV,OAAO,OAAO,OAAO,QAAQ;AAEjC;AAEA,SAAS,oBAAoB,SAA2B;CACtD,IAAI,CAAC,MAAM,QAAQ,OAAO,GACxB,OAAO;CAGT,MAAM,OAAO,QAAQ;CACrB,IAAI,OAAO,SAAS,YAAY,SAAS,MACvC,OAAO;CAGT,IACE,QAAQ,OAAO,oBACf,QAAQ,OAAO,wBAEf,OAAO;CAGT,MAAM,QAAQ,KAAK;CACnB,OACE,MAAM,QAAQ,KAAK,KAAK,MAAM,MAAK,cAAa,OAAO,WAAW,KAAK,CAAC;AAE5E;AAEA,IAAM,kCACJ;AAeF,SAAgB,qBACd,OACA,oBAC6D;CAC7D,IAAI,CAAC,sBAAsB,oBAAoB,KAAK,GAClD,MAAM,IAAI,MAAM,+BAA+B;CAEjD,OAAOA,MAAa,OAAO,wCAAwC,OAAO;AAC5E;AAaA,IAAM,6BAA6B;AACnC,IAAa,4BAA4B;AACzC,IAAM,6BAA6B;AACnC,IAAM,uBAAuB;;;;;;;;;AAU7B,IAAa,aAAb,MAAwB;CACtB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA,mBAA4B,mBAC1B,QACA,oBACA,gCACF;CAEA;CACA;CACA,UAAU;CACV,wBAAwB;CAExB,YACE,IACA,eACA,IACA,oBACA,gBACA,SACA;EACA,MAAM,EAAC,eAAe,UAAU,MAAM,oBAAmB;EACzD,KAAK,kBAAkB;EACvB,KAAK,sBAAsB;EAE3B,KAAK,MAAM;EACX,KAAK,QAAQ;EACb,KAAK,mBAAmB;EAExB,KAAK,MAAM,GACR,YAAY,YAAY,CAAC,CACzB,YAAY,YAAY,QAAQ,CAAC,CACjC,YAAY,iBAAiB,aAAa,CAAC,CAC3C,YAAY,QAAQ,IAAI;EAC3B,KAAK,oBAAoB,IAAI,iBAC3B,KAAK,KACL,IACA,eACF;EACA,KAAK,IAAI,QAAQ,gBAAgB;EACjC,KAAK,WAAW;EAEhB,KAAK,IAAI,iBAAiB,SAAS,KAAK,YAAY;EACpD,KAAK,IAAI,iBAAiB,SAAS,KAAK,YAAY;EAEpD,KAAK,cAAc;EACnB,KAAK,sBAAsB,YACzB,KAAK,gBACL,6BAA6B,CAC/B;CACF;;;;;;;;CASA,OAAgB;EACd,IACE,KAAK,mBAAA,MACL,KAAK,mBAAA,IAEL,KAAK,gBAAgB;GACnB,MAAMC;GACN,SAAS,wDACP,KAAK,iBACN,QACC,KAAK,mBAAA,KAAsC,WAAW,SACvD;GACD,QAAQC;EACV,CAAC;OACI;GACL,MAAM,mBAAqC,CACzC,aACA;IAAC,MAAM,KAAK;IAAO,WAAW,KAAK,IAAI;GAAC,CAC1C;GACA,KAAK,KAAK,kBAAkB,qBAAqB;GACjD,OAAO;EACT;EACA,OAAO;CACT;CAEA,IAAI,OAAe;EACjB,OAAO,KAAK;CACd;CAEA,MAAM,QAAgB,GAAG,MAAiB;EACxC,IAAI,KAAK,SACP;EAEF,KAAK,UAAU;EACf,KAAK,IAAI,OAAO,uBAAuB,UAAU,GAAG,IAAI;EACxD,KAAK,IAAI,oBAAoB,SAAS,KAAK,YAAY;EACvD,KAAK,IAAI,oBAAoB,SAAS,KAAK,YAAY;EACvD,KAAK,2BAA2B,OAAO;EACvC,KAAK,4BAA4B,KAAA;EACjC,KAAK,uBAAuB,OAAO;EACnC,KAAK,wBAAwB,KAAA;EAC7B,KAAK,SAAS;EACd,IAAI,KAAK,IAAI,eAAe,KAAK,IAAI,QACnC,KAAK,IAAI,MAAM;EAEjB,aAAa,KAAK,mBAAmB;CAIvC;CAEA,qBAAqB,mBAA2B;EAC9C,OAAO,KAAK,eAAe,EAAC,MAAM,kBAAiB,CAAC;CACtD;CAEA,iBAAiB,OAAO,UAAwB;EAC9C,MAAM,OAAO,MAAM,KAAK,SAAS;EACjC,IAAI,KAAK,SAAS;GAChB,KAAK,IAAI,QAAQ,0CAA0C,IAAI;GAC/D;EACF;EAEA,IAAI;EACJ,IAAI;GAEF,MAAM,qBADQ,KAAK,MAAM,IACE,GAAO,KAAK,mBAAmB;EAC5D,SAAS,GAAG;GACV,MAAM,YAAY;IAChB,MAAMC;IACN,SAAS,OAAO,CAAC;IACjB,QAAQD;GACV;GACA,KAAK,gBACH,WACA,IAAI,uBAAuB,WAAW,MAAM,CAC9C;GACA;EACF;EAEA,IAAI,CAAC,uCAAuC,QAAQ,GAAG,GAAG;GAExD,IAAI,CAAC,KAAK,uBAAuB;IAC/B,KAAK,wBAAwB;IAC7B,KAAK,IAAI,OACP,oDAAoD,KAAK,UAAU,IAAI,EAAE,GAC3E;GACF;GACA;EACF;EAEA,IAAI;GAEF,IADgB,IAAI,OACJ,QAAQ;IACtB,KAAK,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,qBAAqB;IAC7C;GACF;GAEA,MAAM,SAAS,MAAM,KAAK,gBAAgB,cAAc,GAAG;GAC3D,KAAK,MAAM,KAAK,QACd,KAAK,qBAAqB,CAAC;EAE/B,SAAS,GAAG;GACV,KAAK,iBAAiB,CAAC;EACzB;CACF;CAEA,qBAAqB,QAA6B;EAChD,QAAQ,OAAO,MAAf;GACE,KAAK;IACH,KAAK,gBAAgB,OAAO,KAAK;IACjC;GACF,KAAK,MACH;GACF,KAAK;IACH,QAAQ,OAAO,QAAf;KACE,KAAK;MACH,OACE,KAAK,8BAA8B,KAAA,GACnC,kDACF;MACA,KAAK,4BAA4B,OAAO;MACxC;KACF,KAAK;MACH,OACE,KAAK,0BAA0B,KAAA,GAC/B,kDACF;MACA,KAAK,wBAAwB,OAAO;IAExC;IACA,IAAI,OAAO,WAAW,cACpB,KAAK,eAAe,OAAO,SAAS,YAAY,aAC9C,KAAK,KAAK,WAAW,SAAS,UAAU,WAAW,UAAU,CAC/D;SAEA,KAAK,eAAe,OAAO,SAAS,YAAY,aAC9C,KAAK,KAAK,YAAY,QAAQ,CAChC;IAEF;GAEF,KAAK,aACH,KAAK,MAAM,SAAS,OAAO,QACzB,KAAK,UAAU,KAAK;EAG1B;CACF;CAEA,gBAAgB,MAAkB;EAChC,MAAM,EAAC,MAAM,QAAQ,aAAY;EACjC,IAAI,CAAC,UACH,KAAK,sBAAsB,eAAe;EAE5C,KAAK,MAAM,yBAAyB;GAAC;GAAM;GAAQ;EAAQ,CAAC;CAC9D;CAEA,gBAAgB,MAAkB;EAChC,KAAK,sBAAsB,aAAa;EACxC,KAAK,IAAI,OAAO,yBAAyB,EAAE,SAAS,EAAE,KAAK;CAC7D;CAEA,sBAAsB,WAAmB;EACvC,KAAK,iBAAiB,IAAI,GAAG;IAC1B,6BAA6B,KAAK;IAClC,uBAAuB;EAC1B,CAAC;CACH;CAEA,gBAAgB;EACd,SACE,sBAAsB,KAAK,GAAG,GAC9B,IAAI,SAAS,EACX,QAAQ,MAAM,WAAW,aAAa;GACpC,KAAK,eAAe,EAAC,KAAI,CAAC,CAAC,CAAC,WAAW,SAAS,GAAG,QAAQ;EAC7D,EACF,CAAC,SAIK,CAAC,CACT;CACF;CAEA,eACE,gBACA,aAIA;EAMA,SACE,SAAS,KAAK,cAAc,GAC5B,IAAI,SAAS;GACX,YAAY;GACZ,QAAQ,YAAe,WAAW,aAChC,YAAY,YAAY,QAAQ;EACpC,CAAC,IACD,MACE,IACI,KAAK,iBAAiB,CAAC,IACvB,KAAK,MAAM,iCAAiC,CACpD;CACF;CAEA,iBAAiB,GAAY;EAC3B,MAAM,YACJ,kBAAkB,CAAC,CAAC,EAAE,aAAa,sBAAsB,CAAC,CAAC,CAAC;EAE9D,KAAK,gBAAgB,WAAW,CAAC;CACnC;CAEA,gBAAgB,WAAsB,QAAkB;EACtD,KAAK,UAAU,WAAW,MAAM;EAChC,KAAK,MACH,GAAG,UAAU,KAAK,IAAI,UAAU,OAAO,KAAK,UAAU,WACtD,SACF;CACF;CAEA,yBAAyB,KAAK,IAAI;CAElC,uBAAuB;EACrB,IAAI,KAAK,IAAI,IAAI,KAAK,yBAAyB,4BAA4B;GACzE,KAAK,IAAI,QAAQ,uBAAuB;GACxC,KAAK,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,qBAAqB;EAC/C;CACF;CAEA,KACE,MACA,UACA,YACA;EACA,KAAK,yBAAyB,KAAK,IAAI;EACvC,KAAK,kBAAkB,KAAK,MAAM,UAAU,UAAU;CACxD;CAEA,UAAU,WAAsB,QAAkB;EAChD,UAAU,KAAK,KAAK,KAAK,KAAK,WAAW,MAAM;CACjD;AACF;AAOA,SAAgB,KACd,IACA,IACA,MACA,UACA,YACA;CACA,IAAI,GAAG,eAAeE,YAAU,MAAM;EACpC,eAAe,KAAK,UAAU,IAAI;EAClC,IAAI,aAAa,uBAAuB;GACtC,GAAG,KAAK,UAAU;GAClB;EACF;EAEA,IAAI,YAAY;EAChB,MAAM,QAAQ,iBAAiB;GAC7B,SAAS,0BAA0B,CAAC;EACtC,GAAG,yBAAyB;EAC5B,MAAM,MAAM;EAEZ,MAAM,YAAY,UAAyB;GACzC,IAAI,WACF;GAEF,YAAY;GACZ,aAAa,KAAK;GAClB,SAAS,KAAK;EAChB;EAEA,GAAG,KAAK,YAAY,QAAQ;CAC9B,OAAO;EACL,GAAG,QAAQ,2CAA2C,GAAG,WAAW,IAAI,EACtE,SAAS,KACX,CAAC;EACD,IAAI,aAAa,uBACf,SACE,IAAI,uBACF;GACE,MAAMC;GACN,SAAS;GACT,QAAQH;EACV,GACA,MACF,CACF;CAEJ;AACF;;;;;;;;AASA,IAAa,mBAAb,MAA8B;CAC5B;CACA;CACA;CACA;CAEA,YAAY,IAAgB,IAAmB,iBAAyB;EACtE,KAAK,MAAM;EACX,KAAK,MAAM;EACX,KAAK,mBAAmB;CAC1B;CAEA,KACE,MACA,UACA,YACM;EACN,IAAI,KAAK,oBAAA,IACP,QAAQ,KAAK,IAAb;GACE,KAAK;IACH,OACE,KAAK,sBAAsB,KAAA,GAC3B,uDACF;IACA,KAAK,oBAAoB,IAAI,iBAAiB;IAC9C;GACF,KAAK;IACH,KAAK,cAAc,MAAM,UAAU,UAAU;IAC7C;GACF,KAAK;IACH,KAAK,aAAa,MAAM,QAAQ;IAChC;EACJ;EAEF,KAAK,KAAK,KAAK,KAAK,KAAK,MAAM,UAAU,UAAU;CACrD;CAEA,cACE,UACA,UACA,YACM;EACN,MAAM,UAAU,KAAK;EACrB,OAAO,SAAS,uCAAuC;EAEvD,eAAe,KAAK,UAAU,QAAQ;EACtC,OACE,WAAW,WAAW,gBAAgB,KAAK,WAAW,SAAS,GAAG,GAClE,6BACF;EACA,QACG,SAAS,WAAW,MAAM,iBAAiB,QAAQ,EAAE,IAAG,UACvD,WAAW,KAAK,KAAK,KAAK,KAAK,KAAK,CACtC,CAAC,CACA,WACO,eAAe,QAAQ,IAC7B,UAAS,eAAe,UAAU,QAAQ,KAAK,CAAC,CAClD;CACJ;CAEA,aACE,SACA,UACM;EACN,MAAM,UAAU,KAAK;EACrB,OAAO,SAAS,sCAAsC;EACtD,KAAK,oBAAoB,KAAA;EAEzB,IAAI,QAAQ,EAAE,CAAC,QAAQ;GACrB,QAAQ,OAAO;GACf,KAAK,KAAK,KAAK,KAAK,KAAK,SAAS,QAAQ;GAC1C;EACF;EAEA,QACG,QAAO,UAAS,WAAW,KAAK,KAAK,KAAK,KAAK,KAAK,CAAC,CAAC,CACtD,WAAW,UAAU,KAAK,KAAK,KAAK,KAAK,OAAO,CAAC,CAAC,CAClD,WACO,eAAe,QAAQ,IAC7B,UAAS,eAAe,UAAU,QAAQ,KAAK,CAAC,CAClD;CACJ;AACF;AAEA,IAAM,mBAAmB;AAEzB,SAAS,UACP,IACA,IACA,MACe;CACf,OAAO,IAAI,SAAS,SAAS,WAAW;EACtC,KAAK,IAAI,IAAI,OAAM,UAAU,QAAQ,OAAO,KAAK,IAAI,QAAQ,CAAE;CACjE,CAAC;AACH;AAEA,SAAS,WACP,IACA,IACA,OACe;CACf,OAAO,IAAI,SAAS,SAAS,WAAW;EACtC,IAAI,GAAG,eAAeE,YAAU,MAAM;GACpC,MAAM,QAAQ,qBAAqB;GACnC,GAAG,QACD,kDAAkD,GAAG,WAAW,EAClE;GACA,OAAO,KAAK;GACZ;EACF;EACA,IAAI,YAAY;EAChB,MAAM,QAAQ,iBACN,SAAS,0BAA0B,CAAC,GAC1C,yBACF;EACA,MAAM,MAAM;EAEZ,MAAM,YAAY,UAAyB;GACzC,IAAI,WACF;GAEF,YAAY;GACZ,aAAa,KAAK;GAClB,QAAQ,OAAO,KAAK,IAAI,QAAQ;EAClC;EAEA,GAAG,KAAK,OAAO,QAAQ;CACzB,CAAC;AACH;AAEA,SAAS,eACP,UACA,OACM;CACN,IAAI,aAAa,uBACf,SAAS,KAAK;AAElB;AAEA,SAAS,QAAQ,OAAuB;CACtC,OAAO,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;AACjE;AAEA,SAAS,uBAA+C;CACtD,OAAO,IAAI,uBACT;EACE,MAAMC;EACN,SAAS;EACT,QAAQH;CACV,GACA,MACF;AACF;AAEA,SAAS,4BAAoD;CAC3D,OAAO,IAAI,uBACT;EACE,MAAMG;EACN,SAAS,kCAAkC,0BAA0B;EACrE,QAAQH;CACV,GACA,MACF;AACF;AAEA,SAAgB,UACd,IACA,IACA,WACA,QACA;CACA,KAAK,GAAG,YAAY,aAAa,UAAU,IAAI;CAE/C,IAAI;CAGJ,IAAI,kBAAkB,wBACpB,WAAW,OAAO;MAIf,IACH,SAAS,MAAM,KACf,uBAAuB,MAAM,KAC7B,yBAAyB,UAAU,OAAO,GAE1C,WAAW;MAGR,IACH,UAAU,SAAS,oBACnB,UAAU,SAAS,mBAEnB,WAAW;MAEX,WAAW,SAAS,YAAY,MAAM,IAAI;CAG5C,GAAG,SAAS,GAAG,8BAA8B,WAAW,UAAU,EAAE;CACpE,KAAK,IAAI,IAAI,CAAC,SAAS,SAAS,GAAG,qBAAqB;AAC1D;AAEA,SAAS,kBAAkB,OAA2C;CACpE,IAAI,gBAAgB,KAAK,GACvB,OAAO;CAET,IAAI,iBAAiB,SAAS,MAAM,OAClC,OAAO,kBAAkB,MAAM,KAAK;AAGxC;AAEA,SAAS,SAAS,OAAyB;CACzC,OAAO,QACL,SACA,OAAO,UAAU,YACjB,WAAW,SACX,OAAQ,MAA2B,UAAU,WAC/C;AACF;AAIA,IAAM,+CAA+B,IAAI,IAAI;CAC3C;CACA;CACA;AACF,CAAC;AAID,IAAM,oCAAoC,CACxC,mDACF;AAEA,SAAS,uBAAuB,OAAyB;CACvD,IAAI,CAAC,SAAS,OAAO,UAAU,UAC7B,OAAO;CAET,MAAM,YACJ,UAAU,QAAQ,OAAQ,MAA2B,IAAI,IAAI,KAAA;CAC/D,OAAO,QACL,aAAa,6BAA6B,IAAI,UAAU,YAAY,CAAC,CACvE;AACF;AAEA,SAAS,yBAAyB,SAAsC;CACtE,IAAI,CAAC,SACH,OAAO;CAET,MAAM,QAAQ,QAAQ,YAAY;CAClC,OAAO,kCAAkC,MAAK,YAC5C,MAAM,SAAS,OAAO,CACxB;AACF"}
+{"version":3,"file":"connection.js","names":["valita.parse","ErrorKind.VersionNotSupported","ErrorOrigin.ZeroCache","ErrorKind.InvalidMessage","WebSocket","ErrorKind.Internal"],"sources":["../../../../../zero-cache/src/workers/connection.ts"],"sourcesContent":["import {pipeline, Readable, Writable} from 'node:stream';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport type {CloseEvent, Data, ErrorEvent} from 'ws';\nimport WebSocket, {createWebSocketStream} from 'ws';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport * as valita from '../../../shared/src/valita.ts';\nimport type {ConnectedMessage} from '../../../zero-protocol/src/connect.ts';\nimport type {Downstream} from '../../../zero-protocol/src/down.ts';\nimport {ErrorKind} from '../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../zero-protocol/src/error-origin.ts';\nimport type {ErrorBody} from '../../../zero-protocol/src/error.ts';\nimport {\n  isProtocolError,\n  type ProtocolError,\n} from '../../../zero-protocol/src/error.ts';\nimport {\n  POKE_CHUNK_PROTOCOL_VERSION,\n  type PokeChunk,\n  type PokeEndMessage,\n  type PokePartMessage,\n} from '../../../zero-protocol/src/poke.ts';\nimport {\n  MIN_SERVER_SUPPORTED_SYNC_PROTOCOL,\n  PROTOCOL_VERSION,\n} from '../../../zero-protocol/src/protocol-version.ts';\nimport {\n  upstreamSchemaWithUnparsedAnalyzeQuery,\n  type UpstreamWithUnparsedAnalyzeQuery,\n} from '../../../zero-protocol/src/up.ts';\nimport {getOrCreateCounter} from '../observability/metrics.ts';\nimport {clientGroupLogID} from '../types/client-group-log-id.ts';\nimport type {ViewSyncerDownstream} from '../types/downstream.ts';\nimport {\n  ProtocolErrorWithLevel,\n  getLogLevel,\n  wrapWithProtocolError,\n} from '../types/error-with-level.ts';\nimport {PokeChunkEncoder} from '../types/poke-chunk.ts';\nimport type {Source} from '../types/streams.ts';\nimport type {ConnectParams} from './connect-params.ts';\n\nexport type HandlerResult =\n  | {\n      type: 'ok';\n    }\n  | {\n      type: 'fatal';\n      error: ErrorBody;\n    }\n  | {\n      type: 'transient';\n      errors: ErrorBody[];\n    }\n  | StreamResult;\n\nexport type StreamResult =\n  | {\n      type: 'stream';\n      source: 'viewSyncer';\n      stream: Source<ViewSyncerDownstream>;\n    }\n  | {\n      type: 'stream';\n      source: 'pusher';\n      stream: Source<Downstream>;\n    };\n\nexport interface MessageHandler {\n  handleMessage(\n    msg: UpstreamWithUnparsedAnalyzeQuery,\n  ): Promise<HandlerResult[]>;\n}\n\nfunction hasOwn(value: unknown, property: string): boolean {\n  return (\n    typeof value === 'object' &&\n    value !== null &&\n    Object.hasOwn(value, property)\n  );\n}\n\nfunction containsLegacyQuery(message: unknown): boolean {\n  if (!Array.isArray(message)) {\n    return false;\n  }\n\n  const body = message[1];\n  if (typeof body !== 'object' || body === null) {\n    return false;\n  }\n\n  if (\n    message[0] !== 'initConnection' &&\n    message[0] !== 'changeDesiredQueries'\n  ) {\n    return false;\n  }\n\n  const patch = body['desiredQueriesPatch'];\n  return (\n    Array.isArray(patch) && patch.some(operation => hasOwn(operation, 'ast'))\n  );\n}\n\nconst LEGACY_QUERIES_DISABLED_MESSAGE =\n  'Legacy queries are disabled by this Zero server. This client must use custom queries instead of sending query ASTs directly. To temporarily restore compatibility, set ZERO_ALLOW_LEGACY_QUERIES=true on zero-cache.';\n\n// Exported for testing purposes.\n//\n// Client messages are parsed leniently, so a newer client can add message\n// kinds and fields:\n// - A message of a kind this server doesn't know parses as an\n//   UnrecognizedTuple, for the connection to drop. Malformed messages still\n//   throw.\n// - Messages are parsed in strip mode, so fields this server doesn't know are\n//   dropped. Strip rather than passthrough: nothing this server doesn't know\n//   reaches the CVR or the API server, and valita's passthrough drops unknown\n//   fields anyway on any object with a transformed field below it (such as\n//   jsonSchema args). ASTs stay strict in the schema, because a server that\n//   ignored an AST field would run a different query.\nexport function parseUpstreamMessage(\n  value: unknown,\n  allowLegacyQueries: boolean,\n): UpstreamWithUnparsedAnalyzeQuery | valita.UnrecognizedTuple {\n  if (!allowLegacyQueries && containsLegacyQuery(value)) {\n    throw new Error(LEGACY_QUERIES_DISABLED_MESSAGE);\n  }\n  return valita.parse(value, upstreamSchemaWithUnparsedAnalyzeQuery, 'strip');\n}\n\n// Ensures that a downstream message is sent at least every interval, sending a\n// 'pong' if necessary. This is set to be slightly longer than the client-side\n// PING_INTERVAL of 5 seconds, so that in the common case, 'pong's are sent in\n// response to client-initiated 'ping's. However, if the inbound stream is\n// backed up because a command is taking a long time to process, the pings\n// will be stuck in the queue (i.e. back-pressured), in which case pongs will\n// be manually sent to notify the client of server liveness.\n//\n// This is equivalent to what is done for Postgres keepalives on the\n// replication stream (which can similarly be back-pressured):\n// https://github.com/rocicorp/mono/blob/f98cb369a2dbb15650328859c732db358f187ef0/packages/zero-cache/src/services/change-source/pg/logical-replication/stream.ts#L21\nconst DOWNSTREAM_MSG_INTERVAL_MS = 6_000;\nexport const WEBSOCKET_SEND_TIMEOUT_MS = 10_000;\nconst PROTOCOL_VERSION_ATTRIBUTE = 'protocol.version';\nconst EVENT_TYPE_ATTRIBUTE = 'event.type';\n\n/**\n * Represents a connection between the client and server.\n *\n * Handles incoming messages on the connection and dispatches\n * them to the correct service.\n *\n * Listens to the ViewSyncer and sends messages to the client.\n */\nexport class Connection {\n  readonly #ws: WebSocket;\n  readonly #wsID: string;\n  readonly #protocolVersion: number;\n  readonly #lc: LogContext;\n  readonly #onClose: () => void;\n  readonly #messageHandler: MessageHandler;\n  readonly #allowLegacyQueries: boolean;\n  readonly #downstreamSender: DownstreamSender;\n  readonly #downstreamMsgTimer: NodeJS.Timeout | undefined;\n  readonly #webSocketErrors = getOrCreateCounter(\n    'sync',\n    'websocket.errors',\n    'Client WebSocket error events.',\n  );\n\n  #viewSyncerOutboundStream: Source<ViewSyncerDownstream> | undefined;\n  #pusherOutboundStream: Source<Downstream> | undefined;\n  #closed = false;\n  #loggedUnknownMessage = false;\n\n  constructor(\n    lc: LogContext,\n    connectParams: ConnectParams,\n    ws: WebSocket,\n    allowLegacyQueries: boolean,\n    messageHandler: MessageHandler,\n    onClose: () => void,\n  ) {\n    const {clientGroupID, clientID, wsID, protocolVersion} = connectParams;\n    this.#messageHandler = messageHandler;\n    this.#allowLegacyQueries = allowLegacyQueries;\n\n    this.#ws = ws;\n    this.#wsID = wsID;\n    this.#protocolVersion = protocolVersion;\n\n    this.#lc = lc\n      .withContext('connection')\n      .withContext('clientID', clientID)\n      .withContext('clientGroupID', clientGroupLogID(clientGroupID))\n      .withContext('wsID', wsID);\n    this.#downstreamSender = new DownstreamSender(\n      this.#lc,\n      ws,\n      protocolVersion,\n    );\n    this.#lc.debug?.('new connection');\n    this.#onClose = onClose;\n\n    this.#ws.addEventListener('close', this.#handleClose);\n    this.#ws.addEventListener('error', this.#handleError);\n\n    this.#proxyInbound();\n    this.#downstreamMsgTimer = setInterval(\n      this.#maybeSendPong,\n      DOWNSTREAM_MSG_INTERVAL_MS / 2,\n    );\n  }\n\n  /**\n   * Checks the protocol version and errors for unsupported protocols,\n   * sending the initial `connected` response on success.\n   *\n   * This is early in the connection lifecycle because {@link #handleMessage}\n   * will only parse messages with schema(s) of supported protocol versions.\n   */\n  init(): boolean {\n    if (\n      this.#protocolVersion > PROTOCOL_VERSION ||\n      this.#protocolVersion < MIN_SERVER_SUPPORTED_SYNC_PROTOCOL\n    ) {\n      this.#closeWithError({\n        kind: ErrorKind.VersionNotSupported,\n        message: `server is at sync protocol v${PROTOCOL_VERSION} and does not support v${\n          this.#protocolVersion\n        }. The ${\n          this.#protocolVersion > PROTOCOL_VERSION ? 'server' : 'client'\n        } must be updated to a newer release.`,\n        origin: ErrorOrigin.ZeroCache,\n      });\n    } else {\n      const connectedMessage: ConnectedMessage = [\n        'connected',\n        {wsid: this.#wsID, timestamp: Date.now()},\n      ];\n      this.send(connectedMessage, 'ignore-backpressure');\n      return true;\n    }\n    return false;\n  }\n\n  get wsID(): string {\n    return this.#wsID;\n  }\n\n  close(reason: string, ...args: unknown[]) {\n    if (this.#closed) {\n      return;\n    }\n    this.#closed = true;\n    this.#lc.info?.(`closing connection: ${reason}`, ...args);\n    this.#ws.removeEventListener('close', this.#handleClose);\n    this.#ws.removeEventListener('error', this.#handleError);\n    this.#viewSyncerOutboundStream?.cancel();\n    this.#viewSyncerOutboundStream = undefined;\n    this.#pusherOutboundStream?.cancel();\n    this.#pusherOutboundStream = undefined;\n    this.#onClose();\n    if (this.#ws.readyState !== this.#ws.CLOSED) {\n      this.#ws.close();\n    }\n    clearTimeout(this.#downstreamMsgTimer);\n\n    // spin down services if we have\n    // no more client connections for the client group?\n  }\n\n  handleInitConnection(initConnectionMsg: string) {\n    return this.#handleMessage({data: initConnectionMsg});\n  }\n\n  #handleMessage = async (event: {data: Data}) => {\n    const data = event.data.toString();\n    if (this.#closed) {\n      this.#lc.debug?.('Ignoring message received after closed', data);\n      return;\n    }\n\n    let msg;\n    try {\n      const value = JSON.parse(data);\n      msg = parseUpstreamMessage(value, this.#allowLegacyQueries);\n    } catch (e) {\n      const errorBody = {\n        kind: ErrorKind.InvalidMessage,\n        message: String(e),\n        origin: ErrorOrigin.ZeroCache,\n      } as const;\n      this.#closeWithError(\n        errorBody,\n        new ProtocolErrorWithLevel(errorBody, 'warn'),\n      );\n      return;\n    }\n\n    if (!upstreamSchemaWithUnparsedAnalyzeQuery.isKnown(msg)) {\n      // A message kind from a newer client.\n      if (!this.#loggedUnknownMessage) {\n        this.#loggedUnknownMessage = true;\n        this.#lc.info?.(\n          `Ignoring messages of unknown type, starting with ${JSON.stringify(msg[0])}`,\n        );\n      }\n      return;\n    }\n\n    try {\n      const msgType = msg[0];\n      if (msgType === 'ping') {\n        this.send(['pong', {}], 'ignore-backpressure');\n        return;\n      }\n\n      const result = await this.#messageHandler.handleMessage(msg);\n      for (const r of result) {\n        this.#handleMessageResult(r);\n      }\n    } catch (e) {\n      this.#closeWithThrown(e);\n    }\n  };\n\n  #handleMessageResult(result: HandlerResult): void {\n    switch (result.type) {\n      case 'fatal':\n        this.#closeWithError(result.error);\n        break;\n      case 'ok':\n        break;\n      case 'stream': {\n        switch (result.source) {\n          case 'viewSyncer':\n            assert(\n              this.#viewSyncerOutboundStream === undefined,\n              'Outbound stream already set for this connection!',\n            );\n            this.#viewSyncerOutboundStream = result.stream;\n            break;\n          case 'pusher':\n            assert(\n              this.#pusherOutboundStream === undefined,\n              'Outbound stream already set for this connection!',\n            );\n            this.#pusherOutboundStream = result.stream;\n            break;\n        }\n        if (result.source === 'viewSyncer') {\n          this.#proxyOutbound(result.stream, (downstream, callback) =>\n            this.send(downstream.message, callback, downstream.serialized),\n          );\n        } else {\n          this.#proxyOutbound(result.stream, (downstream, callback) =>\n            this.send(downstream, callback),\n          );\n        }\n        break;\n      }\n      case 'transient': {\n        for (const error of result.errors) {\n          this.sendError(error);\n        }\n      }\n    }\n  }\n\n  #handleClose = (e: CloseEvent) => {\n    const {code, reason, wasClean} = e;\n    if (!wasClean) {\n      this.#recordWebSocketError('unclean_close');\n    }\n    this.close('WebSocket close event', {code, reason, wasClean});\n  };\n\n  #handleError = (e: ErrorEvent) => {\n    this.#recordWebSocketError('error_event');\n    this.#lc.warn?.('WebSocket error event', e.message, e.error);\n  };\n\n  #recordWebSocketError(eventType: string) {\n    this.#webSocketErrors.add(1, {\n      [PROTOCOL_VERSION_ATTRIBUTE]: this.#protocolVersion,\n      [EVENT_TYPE_ATTRIBUTE]: eventType,\n    });\n  }\n\n  #proxyInbound() {\n    pipeline(\n      createWebSocketStream(this.#ws),\n      new Writable({\n        write: (data, _encoding, callback) => {\n          this.#handleMessage({data}).then(() => callback(), callback);\n        },\n      }),\n      // The done callback is not used, as #handleClose and #handleError,\n      // configured on the underlying WebSocket, provide more complete\n      // information.\n      () => {},\n    );\n  }\n\n  #proxyOutbound<T>(\n    outboundStream: Source<T>,\n    sendMessage: (\n      downstream: T,\n      callback: (err?: Error | null) => void,\n    ) => void,\n  ) {\n    // Note: createWebSocketStream() is avoided here in order to control\n    //       exception handling with #closeWithThrown(). If the Writable\n    //       from createWebSocketStream() were instead used, exceptions\n    //       from the outboundStream result in the Writable closing the\n    //       the websocket before the error message can be sent.\n    pipeline(\n      Readable.from(outboundStream),\n      new Writable({\n        objectMode: true,\n        write: (downstream: T, _encoding, callback) =>\n          sendMessage(downstream, callback),\n      }),\n      e =>\n        e\n          ? this.#closeWithThrown(e)\n          : this.close(`downstream closed by ViewSyncer`),\n    );\n  }\n\n  #closeWithThrown(e: unknown) {\n    const errorBody =\n      findProtocolError(e)?.errorBody ?? wrapWithProtocolError(e).errorBody;\n\n    this.#closeWithError(errorBody, e);\n  }\n\n  #closeWithError(errorBody: ErrorBody, thrown?: unknown) {\n    this.sendError(errorBody, thrown);\n    this.close(\n      `${errorBody.kind} (${errorBody.origin}): ${errorBody.message}`,\n      errorBody,\n    );\n  }\n\n  #lastDownstreamMsgTime = Date.now();\n\n  #maybeSendPong = () => {\n    if (Date.now() - this.#lastDownstreamMsgTime > DOWNSTREAM_MSG_INTERVAL_MS) {\n      this.#lc.debug?.('manually sending pong');\n      this.send(['pong', {}], 'ignore-backpressure');\n    }\n  };\n\n  send(\n    data: Downstream,\n    callback: ((err?: Error | null) => void) | 'ignore-backpressure',\n    serialized?: string | undefined,\n  ) {\n    this.#lastDownstreamMsgTime = Date.now();\n    this.#downstreamSender.send(data, callback, serialized);\n  }\n\n  sendError(errorBody: ErrorBody, thrown?: unknown) {\n    sendError(this.#lc, this.#ws, errorBody, thrown);\n  }\n}\n\nexport type WebSocketLike = Pick<WebSocket, 'readyState'> & {\n  send(data: string | PokeChunk, cb?: (err?: Error) => void): void;\n};\n\n// Exported for testing purposes.\nexport function send(\n  lc: LogContext,\n  ws: WebSocketLike,\n  data: Downstream,\n  callback: ((err?: Error | null) => void) | 'ignore-backpressure',\n  serialized?: string | undefined,\n) {\n  if (ws.readyState === WebSocket.OPEN) {\n    serialized ??= JSON.stringify(data);\n    if (callback === 'ignore-backpressure') {\n      ws.send(serialized);\n      return;\n    }\n\n    let completed = false;\n    const timer = setTimeout(() => {\n      complete(webSocketSendTimeoutError());\n    }, WEBSOCKET_SEND_TIMEOUT_MS);\n    timer.unref();\n\n    const complete = (error?: Error | null) => {\n      if (completed) {\n        return;\n      }\n      completed = true;\n      clearTimeout(timer);\n      callback(error);\n    };\n\n    ws.send(serialized, complete);\n  } else {\n    lc.debug?.(`Dropping outbound message on ws (state: ${ws.readyState})`, {\n      dropped: data,\n    });\n    if (callback !== 'ignore-backpressure') {\n      callback(\n        new ProtocolErrorWithLevel(\n          {\n            kind: ErrorKind.Internal,\n            message: 'WebSocket closed',\n            origin: ErrorOrigin.ZeroCache,\n          },\n          'info',\n        ),\n      );\n    }\n  }\n}\n\n/**\n * Sends downstream messages in the format supported by a connection's sync\n * protocol version. Keep version forks contained here so the view-syncer can\n * continue producing one canonical poke representation.\n *\n * Exported for compatibility testing.\n */\nexport class DownstreamSender {\n  readonly #lc: LogContext;\n  readonly #ws: WebSocketLike;\n  readonly #protocolVersion: number;\n  #pokeChunkEncoder: PokeChunkEncoder | undefined;\n  /** The last finished poke's buffer, so the next poke allocates nothing. */\n  #idlePokeChunkBuffer: Uint8Array | undefined;\n\n  constructor(lc: LogContext, ws: WebSocketLike, protocolVersion: number) {\n    this.#lc = lc;\n    this.#ws = ws;\n    this.#protocolVersion = protocolVersion;\n  }\n\n  send(\n    data: Downstream,\n    callback: ((err?: Error | null) => void) | 'ignore-backpressure',\n    serialized?: string | undefined,\n  ): void {\n    if (this.#protocolVersion >= POKE_CHUNK_PROTOCOL_VERSION) {\n      switch (data[0]) {\n        case 'pokeStart':\n          assert(\n            this.#pokeChunkEncoder === undefined,\n            'cannot start a poke while another poke is in progress',\n          );\n          this.#pokeChunkEncoder = new PokeChunkEncoder(\n            this.#idlePokeChunkBuffer,\n          );\n          this.#idlePokeChunkBuffer = undefined;\n          break;\n        case 'pokePart':\n          this.#sendPokePart(data, callback, serialized);\n          return;\n        case 'pokeEnd':\n          this.#sendPokeEnd(data, callback);\n          return;\n      }\n    }\n    send(this.#lc, this.#ws, data, callback, serialized);\n  }\n\n  #sendPokePart(\n    pokePart: PokePartMessage,\n    callback: ((err?: Error | null) => void) | 'ignore-backpressure',\n    serialized?: string | undefined,\n  ): void {\n    const encoder = this.#pokeChunkEncoder;\n    assert(encoder, 'pokePart received without a pokeStart');\n\n    serialized ??= JSON.stringify(pokePart);\n    assert(\n      serialized.startsWith(POKE_PART_PREFIX) && serialized.endsWith(']'),\n      'invalid serialized pokePart',\n    );\n    void encoder\n      .addPatch(serialized.slice(POKE_PART_PREFIX.length, -1), chunk =>\n        sendBinary(this.#lc, this.#ws, chunk),\n      )\n      .then(\n        () => invokeCallback(callback),\n        error => invokeCallback(callback, toError(error)),\n      );\n  }\n\n  #sendPokeEnd(\n    pokeEnd: PokeEndMessage,\n    callback: ((err?: Error | null) => void) | 'ignore-backpressure',\n  ): void {\n    const encoder = this.#pokeChunkEncoder;\n    assert(encoder, 'pokeEnd received without a pokeStart');\n    this.#pokeChunkEncoder = undefined;\n\n    if (pokeEnd[1].cancel) {\n      encoder.cancel();\n      send(this.#lc, this.#ws, pokeEnd, callback);\n      return;\n    }\n\n    void encoder\n      .finish(chunk => sendBinary(this.#lc, this.#ws, chunk))\n      .then(buffer => {\n        this.#idlePokeChunkBuffer = buffer;\n        return sendAsync(this.#lc, this.#ws, pokeEnd);\n      })\n      .then(\n        () => invokeCallback(callback),\n        error => invokeCallback(callback, toError(error)),\n      );\n  }\n}\n\nconst POKE_PART_PREFIX = '[\"pokePart\",';\n\nfunction sendAsync(\n  lc: LogContext,\n  ws: WebSocketLike,\n  data: Downstream,\n): Promise<void> {\n  return new Promise((resolve, reject) => {\n    send(lc, ws, data, error => (error ? reject(error) : resolve()));\n  });\n}\n\nfunction sendBinary(\n  lc: LogContext,\n  ws: WebSocketLike,\n  chunk: PokeChunk,\n): Promise<void> {\n  return new Promise((resolve, reject) => {\n    if (ws.readyState !== WebSocket.OPEN) {\n      const error = webSocketClosedError();\n      lc.debug?.(\n        `Dropping outbound binary message on ws (state: ${ws.readyState})`,\n      );\n      reject(error);\n      return;\n    }\n    let completed = false;\n    const timer = setTimeout(\n      () => complete(webSocketSendTimeoutError()),\n      WEBSOCKET_SEND_TIMEOUT_MS,\n    );\n    timer.unref();\n\n    const complete = (error?: Error | null) => {\n      if (completed) {\n        return;\n      }\n      completed = true;\n      clearTimeout(timer);\n      error ? reject(error) : resolve();\n    };\n\n    ws.send(chunk, complete);\n  });\n}\n\nfunction invokeCallback(\n  callback: ((err?: Error | null) => void) | 'ignore-backpressure',\n  error?: Error,\n): void {\n  if (callback !== 'ignore-backpressure') {\n    callback(error);\n  }\n}\n\nfunction toError(error: unknown): Error {\n  return error instanceof Error ? error : new Error(String(error));\n}\n\nfunction webSocketClosedError(): ProtocolErrorWithLevel {\n  return new ProtocolErrorWithLevel(\n    {\n      kind: ErrorKind.Internal,\n      message: 'WebSocket closed',\n      origin: ErrorOrigin.ZeroCache,\n    },\n    'info',\n  );\n}\n\nfunction webSocketSendTimeoutError(): ProtocolErrorWithLevel {\n  return new ProtocolErrorWithLevel(\n    {\n      kind: ErrorKind.Internal,\n      message: `WebSocket send timed out after ${WEBSOCKET_SEND_TIMEOUT_MS} ms`,\n      origin: ErrorOrigin.ZeroCache,\n    },\n    'info',\n  );\n}\n\nexport function sendError(\n  lc: LogContext,\n  ws: WebSocket,\n  errorBody: ErrorBody,\n  thrown?: unknown,\n) {\n  lc = lc.withContext('errorKind', errorBody.kind);\n\n  let logLevel: LogLevel;\n\n  // If the thrown error is a ProtocolErrorWithLevel, its explicit logLevel takes precedence\n  if (thrown instanceof ProtocolErrorWithLevel) {\n    logLevel = thrown.logLevel;\n  }\n  // Errors with errno or transient socket codes are low-level, transient I/O issues\n  // (e.g., EPIPE, ECONNRESET) and should be warnings, not errors\n  else if (\n    hasErrno(thrown) ||\n    hasTransientSocketCode(thrown) ||\n    isTransientSocketMessage(errorBody.message)\n  ) {\n    logLevel = 'warn';\n  }\n  // Fallback: check errorBody.kind for errors that weren't thrown as ProtocolErrorWithLevel\n  else if (\n    errorBody.kind === ErrorKind.ClientNotFound ||\n    errorBody.kind === ErrorKind.TransformFailed\n  ) {\n    logLevel = 'warn';\n  } else {\n    logLevel = thrown ? getLogLevel(thrown) : 'info';\n  }\n\n  lc[logLevel]?.('Sending error on WebSocket', errorBody, thrown ?? '');\n  send(lc, ws, ['error', errorBody], 'ignore-backpressure');\n}\n\nfunction findProtocolError(error: unknown): ProtocolError | undefined {\n  if (isProtocolError(error)) {\n    return error;\n  }\n  if (error instanceof Error && error.cause) {\n    return findProtocolError(error.cause);\n  }\n  return undefined;\n}\n\nfunction hasErrno(error: unknown): boolean {\n  return Boolean(\n    error &&\n    typeof error === 'object' &&\n    'errno' in error &&\n    typeof (error as {errno: unknown}).errno !== 'undefined',\n  );\n}\n\n// System error codes that indicate transient socket conditions.\n// These are checked via the `code` property on errors.\nconst TRANSIENT_SOCKET_ERROR_CODES = new Set([\n  'EPIPE',\n  'ECONNRESET',\n  'ECANCELED',\n]);\n\n// Error messages that indicate transient socket conditions but don't have\n// standard error codes (e.g., WebSocket library errors).\nconst TRANSIENT_SOCKET_MESSAGE_PATTERNS = [\n  'socket was closed while data was being compressed',\n];\n\nfunction hasTransientSocketCode(error: unknown): boolean {\n  if (!error || typeof error !== 'object') {\n    return false;\n  }\n  const maybeCode =\n    'code' in error ? String((error as {code?: unknown}).code) : undefined;\n  return Boolean(\n    maybeCode && TRANSIENT_SOCKET_ERROR_CODES.has(maybeCode.toUpperCase()),\n  );\n}\n\nfunction isTransientSocketMessage(message: string | undefined): boolean {\n  if (!message) {\n    return false;\n  }\n  const lower = message.toLowerCase();\n  return TRANSIENT_SOCKET_MESSAGE_PATTERNS.some(pattern =>\n    lower.includes(pattern),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;AAyEA,SAAS,OAAO,OAAgB,UAA2B;CACzD,OACE,OAAO,UAAU,YACjB,UAAU,QACV,OAAO,OAAO,OAAO,QAAQ;AAEjC;AAEA,SAAS,oBAAoB,SAA2B;CACtD,IAAI,CAAC,MAAM,QAAQ,OAAO,GACxB,OAAO;CAGT,MAAM,OAAO,QAAQ;CACrB,IAAI,OAAO,SAAS,YAAY,SAAS,MACvC,OAAO;CAGT,IACE,QAAQ,OAAO,oBACf,QAAQ,OAAO,wBAEf,OAAO;CAGT,MAAM,QAAQ,KAAK;CACnB,OACE,MAAM,QAAQ,KAAK,KAAK,MAAM,MAAK,cAAa,OAAO,WAAW,KAAK,CAAC;AAE5E;AAEA,IAAM,kCACJ;AAeF,SAAgB,qBACd,OACA,oBAC6D;CAC7D,IAAI,CAAC,sBAAsB,oBAAoB,KAAK,GAClD,MAAM,IAAI,MAAM,+BAA+B;CAEjD,OAAOA,MAAa,OAAO,wCAAwC,OAAO;AAC5E;AAaA,IAAM,6BAA6B;AACnC,IAAa,4BAA4B;AACzC,IAAM,6BAA6B;AACnC,IAAM,uBAAuB;;;;;;;;;AAU7B,IAAa,aAAb,MAAwB;CACtB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA,mBAA4B,mBAC1B,QACA,oBACA,gCACF;CAEA;CACA;CACA,UAAU;CACV,wBAAwB;CAExB,YACE,IACA,eACA,IACA,oBACA,gBACA,SACA;EACA,MAAM,EAAC,eAAe,UAAU,MAAM,oBAAmB;EACzD,KAAK,kBAAkB;EACvB,KAAK,sBAAsB;EAE3B,KAAK,MAAM;EACX,KAAK,QAAQ;EACb,KAAK,mBAAmB;EAExB,KAAK,MAAM,GACR,YAAY,YAAY,CAAC,CACzB,YAAY,YAAY,QAAQ,CAAC,CACjC,YAAY,iBAAiB,iBAAiB,aAAa,CAAC,CAAC,CAC7D,YAAY,QAAQ,IAAI;EAC3B,KAAK,oBAAoB,IAAI,iBAC3B,KAAK,KACL,IACA,eACF;EACA,KAAK,IAAI,QAAQ,gBAAgB;EACjC,KAAK,WAAW;EAEhB,KAAK,IAAI,iBAAiB,SAAS,KAAK,YAAY;EACpD,KAAK,IAAI,iBAAiB,SAAS,KAAK,YAAY;EAEpD,KAAK,cAAc;EACnB,KAAK,sBAAsB,YACzB,KAAK,gBACL,6BAA6B,CAC/B;CACF;;;;;;;;CASA,OAAgB;EACd,IACE,KAAK,mBAAA,MACL,KAAK,mBAAA,IAEL,KAAK,gBAAgB;GACnB,MAAMC;GACN,SAAS,wDACP,KAAK,iBACN,QACC,KAAK,mBAAA,KAAsC,WAAW,SACvD;GACD,QAAQC;EACV,CAAC;OACI;GACL,MAAM,mBAAqC,CACzC,aACA;IAAC,MAAM,KAAK;IAAO,WAAW,KAAK,IAAI;GAAC,CAC1C;GACA,KAAK,KAAK,kBAAkB,qBAAqB;GACjD,OAAO;EACT;EACA,OAAO;CACT;CAEA,IAAI,OAAe;EACjB,OAAO,KAAK;CACd;CAEA,MAAM,QAAgB,GAAG,MAAiB;EACxC,IAAI,KAAK,SACP;EAEF,KAAK,UAAU;EACf,KAAK,IAAI,OAAO,uBAAuB,UAAU,GAAG,IAAI;EACxD,KAAK,IAAI,oBAAoB,SAAS,KAAK,YAAY;EACvD,KAAK,IAAI,oBAAoB,SAAS,KAAK,YAAY;EACvD,KAAK,2BAA2B,OAAO;EACvC,KAAK,4BAA4B,KAAA;EACjC,KAAK,uBAAuB,OAAO;EACnC,KAAK,wBAAwB,KAAA;EAC7B,KAAK,SAAS;EACd,IAAI,KAAK,IAAI,eAAe,KAAK,IAAI,QACnC,KAAK,IAAI,MAAM;EAEjB,aAAa,KAAK,mBAAmB;CAIvC;CAEA,qBAAqB,mBAA2B;EAC9C,OAAO,KAAK,eAAe,EAAC,MAAM,kBAAiB,CAAC;CACtD;CAEA,iBAAiB,OAAO,UAAwB;EAC9C,MAAM,OAAO,MAAM,KAAK,SAAS;EACjC,IAAI,KAAK,SAAS;GAChB,KAAK,IAAI,QAAQ,0CAA0C,IAAI;GAC/D;EACF;EAEA,IAAI;EACJ,IAAI;GAEF,MAAM,qBADQ,KAAK,MAAM,IACE,GAAO,KAAK,mBAAmB;EAC5D,SAAS,GAAG;GACV,MAAM,YAAY;IAChB,MAAMC;IACN,SAAS,OAAO,CAAC;IACjB,QAAQD;GACV;GACA,KAAK,gBACH,WACA,IAAI,uBAAuB,WAAW,MAAM,CAC9C;GACA;EACF;EAEA,IAAI,CAAC,uCAAuC,QAAQ,GAAG,GAAG;GAExD,IAAI,CAAC,KAAK,uBAAuB;IAC/B,KAAK,wBAAwB;IAC7B,KAAK,IAAI,OACP,oDAAoD,KAAK,UAAU,IAAI,EAAE,GAC3E;GACF;GACA;EACF;EAEA,IAAI;GAEF,IADgB,IAAI,OACJ,QAAQ;IACtB,KAAK,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,qBAAqB;IAC7C;GACF;GAEA,MAAM,SAAS,MAAM,KAAK,gBAAgB,cAAc,GAAG;GAC3D,KAAK,MAAM,KAAK,QACd,KAAK,qBAAqB,CAAC;EAE/B,SAAS,GAAG;GACV,KAAK,iBAAiB,CAAC;EACzB;CACF;CAEA,qBAAqB,QAA6B;EAChD,QAAQ,OAAO,MAAf;GACE,KAAK;IACH,KAAK,gBAAgB,OAAO,KAAK;IACjC;GACF,KAAK,MACH;GACF,KAAK;IACH,QAAQ,OAAO,QAAf;KACE,KAAK;MACH,OACE,KAAK,8BAA8B,KAAA,GACnC,kDACF;MACA,KAAK,4BAA4B,OAAO;MACxC;KACF,KAAK;MACH,OACE,KAAK,0BAA0B,KAAA,GAC/B,kDACF;MACA,KAAK,wBAAwB,OAAO;IAExC;IACA,IAAI,OAAO,WAAW,cACpB,KAAK,eAAe,OAAO,SAAS,YAAY,aAC9C,KAAK,KAAK,WAAW,SAAS,UAAU,WAAW,UAAU,CAC/D;SAEA,KAAK,eAAe,OAAO,SAAS,YAAY,aAC9C,KAAK,KAAK,YAAY,QAAQ,CAChC;IAEF;GAEF,KAAK,aACH,KAAK,MAAM,SAAS,OAAO,QACzB,KAAK,UAAU,KAAK;EAG1B;CACF;CAEA,gBAAgB,MAAkB;EAChC,MAAM,EAAC,MAAM,QAAQ,aAAY;EACjC,IAAI,CAAC,UACH,KAAK,sBAAsB,eAAe;EAE5C,KAAK,MAAM,yBAAyB;GAAC;GAAM;GAAQ;EAAQ,CAAC;CAC9D;CAEA,gBAAgB,MAAkB;EAChC,KAAK,sBAAsB,aAAa;EACxC,KAAK,IAAI,OAAO,yBAAyB,EAAE,SAAS,EAAE,KAAK;CAC7D;CAEA,sBAAsB,WAAmB;EACvC,KAAK,iBAAiB,IAAI,GAAG;IAC1B,6BAA6B,KAAK;IAClC,uBAAuB;EAC1B,CAAC;CACH;CAEA,gBAAgB;EACd,SACE,sBAAsB,KAAK,GAAG,GAC9B,IAAI,SAAS,EACX,QAAQ,MAAM,WAAW,aAAa;GACpC,KAAK,eAAe,EAAC,KAAI,CAAC,CAAC,CAAC,WAAW,SAAS,GAAG,QAAQ;EAC7D,EACF,CAAC,SAIK,CAAC,CACT;CACF;CAEA,eACE,gBACA,aAIA;EAMA,SACE,SAAS,KAAK,cAAc,GAC5B,IAAI,SAAS;GACX,YAAY;GACZ,QAAQ,YAAe,WAAW,aAChC,YAAY,YAAY,QAAQ;EACpC,CAAC,IACD,MACE,IACI,KAAK,iBAAiB,CAAC,IACvB,KAAK,MAAM,iCAAiC,CACpD;CACF;CAEA,iBAAiB,GAAY;EAC3B,MAAM,YACJ,kBAAkB,CAAC,CAAC,EAAE,aAAa,sBAAsB,CAAC,CAAC,CAAC;EAE9D,KAAK,gBAAgB,WAAW,CAAC;CACnC;CAEA,gBAAgB,WAAsB,QAAkB;EACtD,KAAK,UAAU,WAAW,MAAM;EAChC,KAAK,MACH,GAAG,UAAU,KAAK,IAAI,UAAU,OAAO,KAAK,UAAU,WACtD,SACF;CACF;CAEA,yBAAyB,KAAK,IAAI;CAElC,uBAAuB;EACrB,IAAI,KAAK,IAAI,IAAI,KAAK,yBAAyB,4BAA4B;GACzE,KAAK,IAAI,QAAQ,uBAAuB;GACxC,KAAK,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,qBAAqB;EAC/C;CACF;CAEA,KACE,MACA,UACA,YACA;EACA,KAAK,yBAAyB,KAAK,IAAI;EACvC,KAAK,kBAAkB,KAAK,MAAM,UAAU,UAAU;CACxD;CAEA,UAAU,WAAsB,QAAkB;EAChD,UAAU,KAAK,KAAK,KAAK,KAAK,WAAW,MAAM;CACjD;AACF;AAOA,SAAgB,KACd,IACA,IACA,MACA,UACA,YACA;CACA,IAAI,GAAG,eAAeE,YAAU,MAAM;EACpC,eAAe,KAAK,UAAU,IAAI;EAClC,IAAI,aAAa,uBAAuB;GACtC,GAAG,KAAK,UAAU;GAClB;EACF;EAEA,IAAI,YAAY;EAChB,MAAM,QAAQ,iBAAiB;GAC7B,SAAS,0BAA0B,CAAC;EACtC,GAAG,yBAAyB;EAC5B,MAAM,MAAM;EAEZ,MAAM,YAAY,UAAyB;GACzC,IAAI,WACF;GAEF,YAAY;GACZ,aAAa,KAAK;GAClB,SAAS,KAAK;EAChB;EAEA,GAAG,KAAK,YAAY,QAAQ;CAC9B,OAAO;EACL,GAAG,QAAQ,2CAA2C,GAAG,WAAW,IAAI,EACtE,SAAS,KACX,CAAC;EACD,IAAI,aAAa,uBACf,SACE,IAAI,uBACF;GACE,MAAMC;GACN,SAAS;GACT,QAAQH;EACV,GACA,MACF,CACF;CAEJ;AACF;;;;;;;;AASA,IAAa,mBAAb,MAA8B;CAC5B;CACA;CACA;CACA;;CAEA;CAEA,YAAY,IAAgB,IAAmB,iBAAyB;EACtE,KAAK,MAAM;EACX,KAAK,MAAM;EACX,KAAK,mBAAmB;CAC1B;CAEA,KACE,MACA,UACA,YACM;EACN,IAAI,KAAK,oBAAA,IACP,QAAQ,KAAK,IAAb;GACE,KAAK;IACH,OACE,KAAK,sBAAsB,KAAA,GAC3B,uDACF;IACA,KAAK,oBAAoB,IAAI,iBAC3B,KAAK,oBACP;IACA,KAAK,uBAAuB,KAAA;IAC5B;GACF,KAAK;IACH,KAAK,cAAc,MAAM,UAAU,UAAU;IAC7C;GACF,KAAK;IACH,KAAK,aAAa,MAAM,QAAQ;IAChC;EACJ;EAEF,KAAK,KAAK,KAAK,KAAK,KAAK,MAAM,UAAU,UAAU;CACrD;CAEA,cACE,UACA,UACA,YACM;EACN,MAAM,UAAU,KAAK;EACrB,OAAO,SAAS,uCAAuC;EAEvD,eAAe,KAAK,UAAU,QAAQ;EACtC,OACE,WAAW,WAAW,gBAAgB,KAAK,WAAW,SAAS,GAAG,GAClE,6BACF;EACA,QACG,SAAS,WAAW,MAAM,iBAAiB,QAAQ,EAAE,IAAG,UACvD,WAAW,KAAK,KAAK,KAAK,KAAK,KAAK,CACtC,CAAC,CACA,WACO,eAAe,QAAQ,IAC7B,UAAS,eAAe,UAAU,QAAQ,KAAK,CAAC,CAClD;CACJ;CAEA,aACE,SACA,UACM;EACN,MAAM,UAAU,KAAK;EACrB,OAAO,SAAS,sCAAsC;EACtD,KAAK,oBAAoB,KAAA;EAEzB,IAAI,QAAQ,EAAE,CAAC,QAAQ;GACrB,QAAQ,OAAO;GACf,KAAK,KAAK,KAAK,KAAK,KAAK,SAAS,QAAQ;GAC1C;EACF;EAEA,QACG,QAAO,UAAS,WAAW,KAAK,KAAK,KAAK,KAAK,KAAK,CAAC,CAAC,CACtD,MAAK,WAAU;GACd,KAAK,uBAAuB;GAC5B,OAAO,UAAU,KAAK,KAAK,KAAK,KAAK,OAAO;EAC9C,CAAC,CAAC,CACD,WACO,eAAe,QAAQ,IAC7B,UAAS,eAAe,UAAU,QAAQ,KAAK,CAAC,CAClD;CACJ;AACF;AAEA,IAAM,mBAAmB;AAEzB,SAAS,UACP,IACA,IACA,MACe;CACf,OAAO,IAAI,SAAS,SAAS,WAAW;EACtC,KAAK,IAAI,IAAI,OAAM,UAAU,QAAQ,OAAO,KAAK,IAAI,QAAQ,CAAE;CACjE,CAAC;AACH;AAEA,SAAS,WACP,IACA,IACA,OACe;CACf,OAAO,IAAI,SAAS,SAAS,WAAW;EACtC,IAAI,GAAG,eAAeE,YAAU,MAAM;GACpC,MAAM,QAAQ,qBAAqB;GACnC,GAAG,QACD,kDAAkD,GAAG,WAAW,EAClE;GACA,OAAO,KAAK;GACZ;EACF;EACA,IAAI,YAAY;EAChB,MAAM,QAAQ,iBACN,SAAS,0BAA0B,CAAC,GAC1C,yBACF;EACA,MAAM,MAAM;EAEZ,MAAM,YAAY,UAAyB;GACzC,IAAI,WACF;GAEF,YAAY;GACZ,aAAa,KAAK;GAClB,QAAQ,OAAO,KAAK,IAAI,QAAQ;EAClC;EAEA,GAAG,KAAK,OAAO,QAAQ;CACzB,CAAC;AACH;AAEA,SAAS,eACP,UACA,OACM;CACN,IAAI,aAAa,uBACf,SAAS,KAAK;AAElB;AAEA,SAAS,QAAQ,OAAuB;CACtC,OAAO,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;AACjE;AAEA,SAAS,uBAA+C;CACtD,OAAO,IAAI,uBACT;EACE,MAAMC;EACN,SAAS;EACT,QAAQH;CACV,GACA,MACF;AACF;AAEA,SAAS,4BAAoD;CAC3D,OAAO,IAAI,uBACT;EACE,MAAMG;EACN,SAAS,kCAAkC,0BAA0B;EACrE,QAAQH;CACV,GACA,MACF;AACF;AAEA,SAAgB,UACd,IACA,IACA,WACA,QACA;CACA,KAAK,GAAG,YAAY,aAAa,UAAU,IAAI;CAE/C,IAAI;CAGJ,IAAI,kBAAkB,wBACpB,WAAW,OAAO;MAIf,IACH,SAAS,MAAM,KACf,uBAAuB,MAAM,KAC7B,yBAAyB,UAAU,OAAO,GAE1C,WAAW;MAGR,IACH,UAAU,SAAS,oBACnB,UAAU,SAAS,mBAEnB,WAAW;MAEX,WAAW,SAAS,YAAY,MAAM,IAAI;CAG5C,GAAG,SAAS,GAAG,8BAA8B,WAAW,UAAU,EAAE;CACpE,KAAK,IAAI,IAAI,CAAC,SAAS,SAAS,GAAG,qBAAqB;AAC1D;AAEA,SAAS,kBAAkB,OAA2C;CACpE,IAAI,gBAAgB,KAAK,GACvB,OAAO;CAET,IAAI,iBAAiB,SAAS,MAAM,OAClC,OAAO,kBAAkB,MAAM,KAAK;AAGxC;AAEA,SAAS,SAAS,OAAyB;CACzC,OAAO,QACL,SACA,OAAO,UAAU,YACjB,WAAW,SACX,OAAQ,MAA2B,UAAU,WAC/C;AACF;AAIA,IAAM,+CAA+B,IAAI,IAAI;CAC3C;CACA;CACA;AACF,CAAC;AAID,IAAM,oCAAoC,CACxC,mDACF;AAEA,SAAS,uBAAuB,OAAyB;CACvD,IAAI,CAAC,SAAS,OAAO,UAAU,UAC7B,OAAO;CAET,MAAM,YACJ,UAAU,QAAQ,OAAQ,MAA2B,IAAI,IAAI,KAAA;CAC/D,OAAO,QACL,aAAa,6BAA6B,IAAI,UAAU,YAAY,CAAC,CACvE;AACF;AAEA,SAAS,yBAAyB,SAAsC;CACtE,IAAI,CAAC,SACH,OAAO;CAET,MAAM,QAAQ,QAAQ,YAAY;CAClC,OAAO,kCAAkC,MAAK,YAC5C,MAAM,SAAS,OAAO,CACxB;AACF"}

package/out/zero-cache/src/workers/rehome.d.ts

@@ -1,6 +1,16 @@
 import type { LogContext } from '@rocicorp/logger';
 import type { WebSocket } from 'ws';
 import type { ZeroConfig } from '../config/zero-config.ts';
+/**
+ * The range of the `maxBackoffMs` sent with a draining Rehome. Without it, the
+ * client waits its full reconnect interval (5 seconds), when all it needs to
+ * do is try again and be routed to another server. The client does not add
+ * jitter, so it is chosen per connection here, spreading out the clients that
+ * arrive together and any that a load balancer sends back before it stops
+ * routing to this server.
+ */
+export declare const MIN_REHOME_BACKOFF_MS = 250;
+export declare const MAX_REHOME_BACKOFF_MS = 1000;
 /**
  * Returns the `Set-Cookie` header line that expires the configured load
  * balancer affinity cookie, or `undefined` if none is configured. It must be

package/out/zero-cache/src/workers/rehome.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"rehome.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/workers/rehome.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,IAAI,CAAC;AAGlC,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,0BAA0B,CAAC;AAIzD;;;;;GAKG;AACH,wBAAgB,2BAA2B,CAAC,EAC1C,cAAc,GACf,EAAE,IAAI,CAAC,UAAU,EAAE,gBAAgB,CAAC,GAAG,MAAM,GAAG,SAAS,CAMzD;AAED;;GAEG;AACH,wBAAgB,MAAM,CAAC,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,SAAS,QAenD"}
+{"version":3,"file":"rehome.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/workers/rehome.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,IAAI,CAAC;AAIlC,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,0BAA0B,CAAC;AAIzD;;;;;;;GAOG;AACH,eAAO,MAAM,qBAAqB,MAAM,CAAC;AACzC,eAAO,MAAM,qBAAqB,OAAO,CAAC;AAE1C;;;;;GAKG;AACH,wBAAgB,2BAA2B,CAAC,EAC1C,cAAc,GACf,EAAE,IAAI,CAAC,UAAU,EAAE,gBAAgB,CAAC,GAAG,MAAM,GAAG,SAAS,CAMzD;AAED;;GAEG;AACH,wBAAgB,MAAM,CAAC,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,SAAS,QAgBnD"}

package/out/zero-cache/src/workers/rehome.js

@@ -1,8 +1,9 @@
 import { Rehome } from "../../../zero-protocol/src/error-kind-enum.js";
 import { ZeroCache } from "../../../zero-protocol/src/error-origin-enum.js";
+import { randInt } from "../../../shared/src/rand.js";
 import { ProtocolErrorWithLevel } from "../types/error-with-level.js";
 import { sendError } from "./connection.js";
-//#region ../zero-cache/src/workers/rehome.ts
+var MAX_REHOME_BACKOFF_MS = 1e3;
 /**
 * Returns the `Set-Cookie` header line that expires the configured load
 * balancer affinity cookie, or `undefined` if none is configured. It must be
@@ -25,12 +26,13 @@ function rehome(lc, ws) {
 	const error = new ProtocolErrorWithLevel({
 		kind: Rehome,
 		message: "Server is draining",
+		maxBackoffMs: randInt(250, MAX_REHOME_BACKOFF_MS),
 		origin: ZeroCache
 	}, "info");
 	sendError(lc, ws, error.errorBody, error);
 	ws.close(1001, "Server is draining");
 }
 //#endregion
-export { expiredAffinityCookieHeader, rehome };
+export { MAX_REHOME_BACKOFF_MS, expiredAffinityCookieHeader, rehome };
 
 //# sourceMappingURL=rehome.js.map

package/out/zero-cache/src/workers/rehome.js.map

@@ -1 +1 @@
-{"version":3,"file":"rehome.js","names":["ErrorKind.Rehome","ErrorOrigin.ZeroCache"],"sources":["../../../../../zero-cache/src/workers/rehome.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {WebSocket} from 'ws';\nimport {ErrorKind} from '../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../zero-protocol/src/error-origin.ts';\nimport type {ZeroConfig} from '../config/zero-config.ts';\nimport {ProtocolErrorWithLevel} from '../types/error-with-level.ts';\nimport {sendError} from './connection.ts';\n\n/**\n * Returns the `Set-Cookie` header line that expires the configured load\n * balancer affinity cookie, or `undefined` if none is configured. It must be\n * part of the 101 response of a draining handshake so that the client's next\n * attempt is routed to a different server.\n */\nexport function expiredAffinityCookieHeader({\n  affinityCookie,\n}: Pick<ZeroConfig, 'affinityCookie'>): string | undefined {\n  if (!affinityCookie?.name) {\n    return undefined;\n  }\n  const {name, path, domain} = affinityCookie;\n  return `Set-Cookie: ${name}=; Max-Age=0; Path=${path}${domain ? `; Domain=${domain}` : ''}; Secure; HttpOnly; SameSite=None`;\n}\n\n/**\n * Sends a Rehome error to a connection on a draining server and closes it.\n */\nexport function rehome(lc: LogContext, ws: WebSocket) {\n  ws.on('error', error => {\n    lc.warn?.('error while rehoming draining connection', error);\n    ws.terminate();\n  });\n  const error = new ProtocolErrorWithLevel(\n    {\n      kind: ErrorKind.Rehome,\n      message: 'Server is draining',\n      origin: ErrorOrigin.ZeroCache,\n    },\n    'info',\n  );\n  sendError(lc, ws, error.errorBody, error);\n  ws.close(1001, 'Server is draining');\n}\n"],"mappings":";;;;;;;;;;;AAcA,SAAgB,4BAA4B,EAC1C,kBACyD;CACzD,IAAI,CAAC,gBAAgB,MACnB;CAEF,MAAM,EAAC,MAAM,MAAM,WAAU;CAC7B,OAAO,eAAe,KAAK,qBAAqB,OAAO,SAAS,YAAY,WAAW,GAAG;AAC5F;;;;AAKA,SAAgB,OAAO,IAAgB,IAAe;CACpD,GAAG,GAAG,UAAS,UAAS;EACtB,GAAG,OAAO,4CAA4C,KAAK;EAC3D,GAAG,UAAU;CACf,CAAC;CACD,MAAM,QAAQ,IAAI,uBAChB;EACE,MAAMA;EACN,SAAS;EACT,QAAQC;CACV,GACA,MACF;CACA,UAAU,IAAI,IAAI,MAAM,WAAW,KAAK;CACxC,GAAG,MAAM,MAAM,oBAAoB;AACrC"}
+{"version":3,"file":"rehome.js","names":["ErrorKind.Rehome","ErrorOrigin.ZeroCache"],"sources":["../../../../../zero-cache/src/workers/rehome.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {WebSocket} from 'ws';\nimport {randInt} from '../../../shared/src/rand.ts';\nimport {ErrorKind} from '../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../zero-protocol/src/error-origin.ts';\nimport type {ZeroConfig} from '../config/zero-config.ts';\nimport {ProtocolErrorWithLevel} from '../types/error-with-level.ts';\nimport {sendError} from './connection.ts';\n\n/**\n * The range of the `maxBackoffMs` sent with a draining Rehome. Without it, the\n * client waits its full reconnect interval (5 seconds), when all it needs to\n * do is try again and be routed to another server. The client does not add\n * jitter, so it is chosen per connection here, spreading out the clients that\n * arrive together and any that a load balancer sends back before it stops\n * routing to this server.\n */\nexport const MIN_REHOME_BACKOFF_MS = 250;\nexport const MAX_REHOME_BACKOFF_MS = 1000;\n\n/**\n * Returns the `Set-Cookie` header line that expires the configured load\n * balancer affinity cookie, or `undefined` if none is configured. It must be\n * part of the 101 response of a draining handshake so that the client's next\n * attempt is routed to a different server.\n */\nexport function expiredAffinityCookieHeader({\n  affinityCookie,\n}: Pick<ZeroConfig, 'affinityCookie'>): string | undefined {\n  if (!affinityCookie?.name) {\n    return undefined;\n  }\n  const {name, path, domain} = affinityCookie;\n  return `Set-Cookie: ${name}=; Max-Age=0; Path=${path}${domain ? `; Domain=${domain}` : ''}; Secure; HttpOnly; SameSite=None`;\n}\n\n/**\n * Sends a Rehome error to a connection on a draining server and closes it.\n */\nexport function rehome(lc: LogContext, ws: WebSocket) {\n  ws.on('error', error => {\n    lc.warn?.('error while rehoming draining connection', error);\n    ws.terminate();\n  });\n  const error = new ProtocolErrorWithLevel(\n    {\n      kind: ErrorKind.Rehome,\n      message: 'Server is draining',\n      maxBackoffMs: randInt(MIN_REHOME_BACKOFF_MS, MAX_REHOME_BACKOFF_MS),\n      origin: ErrorOrigin.ZeroCache,\n    },\n    'info',\n  );\n  sendError(lc, ws, error.errorBody, error);\n  ws.close(1001, 'Server is draining');\n}\n"],"mappings":";;;;;AAkBA,IAAa,wBAAwB;;;;;;;AAQrC,SAAgB,4BAA4B,EAC1C,kBACyD;CACzD,IAAI,CAAC,gBAAgB,MACnB;CAEF,MAAM,EAAC,MAAM,MAAM,WAAU;CAC7B,OAAO,eAAe,KAAK,qBAAqB,OAAO,SAAS,YAAY,WAAW,GAAG;AAC5F;;;;AAKA,SAAgB,OAAO,IAAgB,IAAe;CACpD,GAAG,GAAG,UAAS,UAAS;EACtB,GAAG,OAAO,4CAA4C,KAAK;EAC3D,GAAG,UAAU;CACf,CAAC;CACD,MAAM,QAAQ,IAAI,uBAChB;EACE,MAAMA;EACN,SAAS;EACT,cAAc,QAAA,KAA+B,qBAAqB;EAClE,QAAQC;CACV,GACA,MACF;CACA,UAAU,IAAI,IAAI,MAAM,WAAW,KAAK;CACxC,GAAG,MAAM,MAAM,oBAAoB;AACrC"}

package/out/zero-cache/src/workers/syncer-ws-message-handler.js

@@ -4,6 +4,7 @@ import { ZeroCache } from "../../../zero-protocol/src/error-origin-enum.js";
 import { startAsyncSpan, startSpan } from "../../../otel/src/span.js";
 import { version } from "../../../otel/src/version.js";
 import { ProtocolErrorWithLevel } from "../types/error-with-level.js";
+import { clientGroupLogID } from "../types/client-group-log-id.js";
 import "../services/view-syncer/connection-context-manager.js";
 import "../services/view-syncer/view-syncer.js";
 import { Lock } from "@rocicorp/lock";
@@ -36,7 +37,7 @@ var SyncerWsMessageHandler = class {
 		this.#mutagen = mutagen;
 		this.#connContextManager = connContextManager;
 		this.#mutationLock = new Lock();
-		this.#lc = lc.withContext("connection").withContext("clientID", clientID).withContext("clientGroupID", clientGroupID).withContext("wsID", wsID);
+		this.#lc = lc.withContext("connection").withContext("clientID", clientID).withContext("clientGroupID", clientGroupLogID(clientGroupID)).withContext("wsID", wsID);
 		this.#clientGroupID = clientGroupID;
 		this.#pusher = pusher;
 		this.#connectionSelector = {
@@ -61,7 +62,7 @@ var SyncerWsMessageHandler = class {
 					type: "fatal",
 					error: {
 						kind: InvalidPush,
-						message: `clientGroupID in mutation "${clientGroupID}" does not match clientGroupID of connection "${this.#clientGroupID}`,
+						message: "clientGroupID in mutation does not match clientGroupID of connection",
 						origin: ZeroCache
 					}
 				}];

package/out/zero-cache/src/workers/syncer-ws-message-handler.js.map

@@ -1 +1 @@
-{"version":3,"file":"syncer-ws-message-handler.js","names":["ErrorKind.InvalidPush","ErrorOrigin.ZeroCache","ErrorKind.InvalidMessage"],"sources":["../../../../../zero-cache/src/workers/syncer-ws-message-handler.ts"],"sourcesContent":["import {ROOT_CONTEXT, context, propagation, trace} from '@opentelemetry/api';\nimport {Lock} from '@rocicorp/lock';\nimport type {LogContext} from '@rocicorp/logger';\nimport {startAsyncSpan, startSpan} from '../../../otel/src/span.ts';\nimport {version} from '../../../otel/src/version.ts';\nimport {assert, unreachable} from '../../../shared/src/asserts.ts';\nimport {ErrorKind} from '../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../zero-protocol/src/error-origin.ts';\nimport type {ErrorBody} from '../../../zero-protocol/src/error.ts';\nimport type {UpstreamWithUnparsedAnalyzeQuery} from '../../../zero-protocol/src/up.ts';\nimport type {Mutagen} from '../services/mutagen/mutagen.ts';\nimport type {Pusher} from '../services/mutagen/pusher.ts';\nimport {\n  type ConnectionContextManager,\n  type ConnectionSelector,\n} from '../services/view-syncer/connection-context-manager.ts';\nimport {type ViewSyncer} from '../services/view-syncer/view-syncer.ts';\nimport {ProtocolErrorWithLevel} from '../types/error-with-level.ts';\nimport type {ConnectParams} from './connect-params.ts';\nimport type {HandlerResult, MessageHandler} from './connection.ts';\n\nconst tracer = trace.getTracer('syncer-ws-server', version);\n\n/**\n * Wraps a function in an OTEL context extracted from a W3C traceparent header.\n * This enables distributed tracing from the client through zero-cache to\n * the user's API server.\n */\nfunction withTraceparent<T>(traceparent: string | undefined, fn: () => T): T {\n  if (!traceparent) {\n    return fn();\n  }\n  const extracted = propagation.extract(ROOT_CONTEXT, {traceparent});\n  return context.with(extracted, fn);\n}\n\nexport class SyncerWsMessageHandler implements MessageHandler {\n  readonly #viewSyncer: ViewSyncer;\n  readonly #mutagen: Mutagen | undefined;\n  readonly #mutationLock: Lock;\n  readonly #lc: LogContext;\n  readonly #clientGroupID: string;\n  readonly #connectionSelector: ConnectionSelector;\n  readonly #connContextManager: ConnectionContextManager;\n  readonly #pusher: Pusher | undefined;\n  #initialized = false;\n\n  constructor(\n    lc: LogContext,\n    connectParams: ConnectParams,\n    connContextManager: ConnectionContextManager,\n    viewSyncer: ViewSyncer,\n    mutagen: Mutagen | undefined,\n    pusher: Pusher | undefined,\n  ) {\n    const {clientGroupID, clientID, wsID} = connectParams;\n    this.#viewSyncer = viewSyncer;\n    this.#mutagen = mutagen;\n    this.#connContextManager = connContextManager;\n    this.#mutationLock = new Lock();\n    this.#lc = lc\n      .withContext('connection')\n      .withContext('clientID', clientID)\n      .withContext('clientGroupID', clientGroupID)\n      .withContext('wsID', wsID);\n    this.#clientGroupID = clientGroupID;\n    this.#pusher = pusher;\n    this.#connectionSelector = {\n      clientID,\n      wsID,\n    };\n  }\n\n  async handleMessage(\n    msg: UpstreamWithUnparsedAnalyzeQuery,\n  ): Promise<HandlerResult[]> {\n    const lc = this.#lc;\n    const msgType = msg[0];\n    const viewSyncer = this.#viewSyncer;\n    switch (msgType) {\n      case 'ping':\n        lc.error?.('Ping is not supported at this layer by Zero');\n        break;\n      case 'pull':\n        lc.error?.('Pull is not supported by Zero');\n        break;\n      case 'push': {\n        return withTraceparent(msg[1].traceparent, () =>\n          startAsyncSpan<HandlerResult[]>(\n            tracer,\n            'connection.push',\n            async () => {\n              const {clientGroupID, mutations} = msg[1];\n              if (clientGroupID !== this.#clientGroupID) {\n                return [\n                  {\n                    type: 'fatal',\n                    error: {\n                      kind: ErrorKind.InvalidPush,\n                      message:\n                        `clientGroupID in mutation \"${clientGroupID}\" does not match ` +\n                        `clientGroupID of connection \"${this.#clientGroupID}`,\n                      origin: ErrorOrigin.ZeroCache,\n                    },\n                  } satisfies HandlerResult,\n                ];\n              }\n\n              if (mutations.length === 0) {\n                return [\n                  {\n                    type: 'ok',\n                  },\n                ];\n              }\n\n              // The client only ever sends 1 mutation per push.\n              // #pusher will throw if it sees a CRUD mutation.\n              // #mutagen will throw if it see a custom mutation.\n              if (mutations[0].type === 'custom') {\n                if (!this.#pusher) {\n                  return [\n                    {\n                      type: 'fatal',\n                      error: {\n                        kind: ErrorKind.InvalidPush,\n                        message:\n                          'A ZERO_MUTATE_URL must be set in order to process custom mutations.',\n                        origin: ErrorOrigin.ZeroCache,\n                      },\n                    } satisfies HandlerResult,\n                  ];\n                }\n                return [\n                  this.#pusher.enqueuePush(this.#connectionSelector, msg[1]),\n                ];\n              }\n\n              const mutagen = this.#mutagen;\n              if (!mutagen) {\n                return [\n                  {\n                    type: 'fatal',\n                    error: {\n                      kind: ErrorKind.InvalidPush,\n                      message: `Support for legacy CRUD mutations is disabled`,\n                      origin: ErrorOrigin.ZeroCache,\n                    },\n                  } satisfies HandlerResult,\n                ];\n              }\n\n              const auth = this.#connContextManager.mustGetConnectionContext(\n                this.#connectionSelector,\n              ).auth;\n              assert(\n                auth?.type !== 'opaque',\n                'Only JWT auth is supported for CRUD mutations',\n              );\n\n              // Hold a ref on the mutagen for the duration of the push so that\n              // the connection closing mid-push does not stop the service\n              // (closing its replica handle) between two of its mutations.\n              // (The connection holds a ref while open, and messages are not\n              // dispatched after it has closed, so the service is running\n              // here.)\n              mutagen.ref();\n              try {\n                // Hold a connection-level lock while processing mutations so that:\n                // 1. Mutations are processed in the order in which they are received and\n                // 2. A single view syncer connection cannot hog multiple upstream connections.\n                const ret = await this.#mutationLock.withLock(async () => {\n                  const errors: ErrorBody[] = [];\n                  for (const mutation of mutations) {\n                    const maybeError = await mutagen.processMutation(\n                      mutation,\n                      auth?.decoded,\n                      this.#pusher !== undefined,\n                    );\n                    if (maybeError !== undefined) {\n                      errors.push({\n                        kind: maybeError[0],\n                        message: maybeError[1],\n                        origin: ErrorOrigin.ZeroCache,\n                      });\n                    }\n                  }\n                  if (errors.length > 0) {\n                    return {type: 'transient', errors} satisfies HandlerResult;\n                  }\n                  return {type: 'ok'} satisfies HandlerResult;\n                });\n                return [ret];\n              } finally {\n                mutagen.unref();\n              }\n            },\n          ),\n        );\n      }\n      case 'changeDesiredQueries':\n        await withTraceparent(msg[1].traceparent, () =>\n          startAsyncSpan(tracer, 'connection.changeDesiredQueries', () =>\n            viewSyncer.changeDesiredQueries(this.#connectionSelector, msg),\n          ),\n        );\n        break;\n      case 'updateAuth':\n        await startAsyncSpan(tracer, 'connection.updateAuth', async () => {\n          const initialConnCtx =\n            this.#connContextManager.mustGetConnectionContext(\n              this.#connectionSelector,\n            );\n          const updatedConnCtx = await this.#connContextManager.updateAuth(\n            this.#connectionSelector,\n            msg[1],\n          );\n          const authRevisionChanged =\n            updatedConnCtx.revision !== initialConnCtx.revision;\n\n          await viewSyncer.updateAuth(\n            this.#connectionSelector,\n            msg,\n            authRevisionChanged,\n          );\n        });\n        break;\n      case 'deleteClients': {\n        const deletedClientIDs = await startAsyncSpan(\n          tracer,\n          'connection.deleteClients',\n          () => viewSyncer.deleteClients(this.#connectionSelector, msg),\n        );\n        if (this.#pusher && deletedClientIDs.length > 0) {\n          await this.#pusher.deleteClientMutations(\n            this.#connectionSelector,\n            deletedClientIDs,\n          );\n        }\n        break;\n      }\n      case 'initConnection': {\n        // A socket is initialized once, from the sec-protocol header or from\n        // its first initConnection message. A second one is a client bug, so\n        // reject it before it replaces this socket's view-syncer and pusher\n        // streams.\n        if (this.#initialized) {\n          throw new ProtocolErrorWithLevel(\n            {\n              kind: ErrorKind.InvalidMessage,\n              message: 'Connection was already initialized',\n              origin: ErrorOrigin.ZeroCache,\n            },\n            'warn',\n          );\n        }\n        this.#initialized = true;\n        this.#connContextManager.initConnection(\n          this.#connectionSelector,\n          msg[1],\n        );\n        return withTraceparent(msg[1].traceparent, () => {\n          const ret: HandlerResult[] = [\n            {\n              type: 'stream',\n              source: 'viewSyncer',\n              stream: startSpan(tracer, 'connection.initConnection', () =>\n                viewSyncer.initConnection(this.#connectionSelector, msg),\n              ),\n            },\n          ];\n\n          // Given we support both CRUD and Custom mutators,\n          // we do not initialize the `pusher` unless the user has opted\n          // into custom mutations. We detect that by checking\n          // if the pushURL has been set.\n          if (this.#pusher) {\n            ret.push({\n              type: 'stream',\n              source: 'pusher',\n              stream: this.#pusher.initConnection(this.#connectionSelector),\n            });\n          }\n\n          return ret;\n        });\n      }\n      case 'closeConnection':\n        // This message is deprecated and no longer used.\n        break;\n\n      case 'inspect':\n        await startAsyncSpan(tracer, 'connection.inspect', () =>\n          viewSyncer.inspect(this.#connectionSelector, msg),\n        );\n        break;\n\n      case 'ackMutationResponses':\n        if (this.#pusher) {\n          await this.#pusher.ackMutationResponses(\n            this.#connectionSelector,\n            msg[1],\n          );\n        }\n        break;\n\n      default:\n        unreachable(msgType);\n    }\n\n    return [{type: 'ok'}];\n  }\n}\n"],"mappings":";;;;;;;;;;;AAqBA,IAAM,SAAS,MAAM,UAAU,oBAAoB,OAAO;;;;;;AAO1D,SAAS,gBAAmB,aAAiC,IAAgB;CAC3E,IAAI,CAAC,aACH,OAAO,GAAG;CAEZ,MAAM,YAAY,YAAY,QAAQ,cAAc,EAAC,YAAW,CAAC;CACjE,OAAO,QAAQ,KAAK,WAAW,EAAE;AACnC;AAEA,IAAa,yBAAb,MAA8D;CAC5D;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA,eAAe;CAEf,YACE,IACA,eACA,oBACA,YACA,SACA,QACA;EACA,MAAM,EAAC,eAAe,UAAU,SAAQ;EACxC,KAAK,cAAc;EACnB,KAAK,WAAW;EAChB,KAAK,sBAAsB;EAC3B,KAAK,gBAAgB,IAAI,KAAK;EAC9B,KAAK,MAAM,GACR,YAAY,YAAY,CAAC,CACzB,YAAY,YAAY,QAAQ,CAAC,CACjC,YAAY,iBAAiB,aAAa,CAAC,CAC3C,YAAY,QAAQ,IAAI;EAC3B,KAAK,iBAAiB;EACtB,KAAK,UAAU;EACf,KAAK,sBAAsB;GACzB;GACA;EACF;CACF;CAEA,MAAM,cACJ,KAC0B;EAC1B,MAAM,KAAK,KAAK;EAChB,MAAM,UAAU,IAAI;EACpB,MAAM,aAAa,KAAK;EACxB,QAAQ,SAAR;GACE,KAAK;IACH,GAAG,QAAQ,6CAA6C;IACxD;GACF,KAAK;IACH,GAAG,QAAQ,+BAA+B;IAC1C;GACF,KAAK,QACH,OAAO,gBAAgB,IAAI,EAAE,CAAC,mBAC5B,eACE,QACA,mBACA,YAAY;IACV,MAAM,EAAC,eAAe,cAAa,IAAI;IACvC,IAAI,kBAAkB,KAAK,gBACzB,OAAO,CACL;KACE,MAAM;KACN,OAAO;MACL,MAAMA;MACN,SACE,8BAA8B,cAAc,gDACZ,KAAK;MACvC,QAAQC;KACV;IACF,CACF;IAGF,IAAI,UAAU,WAAW,GACvB,OAAO,CACL,EACE,MAAM,KACR,CACF;IAMF,IAAI,UAAU,EAAE,CAAC,SAAS,UAAU;KAClC,IAAI,CAAC,KAAK,SACR,OAAO,CACL;MACE,MAAM;MACN,OAAO;OACL,MAAMD;OACN,SACE;OACF,QAAQC;MACV;KACF,CACF;KAEF,OAAO,CACL,KAAK,QAAQ,YAAY,KAAK,qBAAqB,IAAI,EAAE,CAC3D;IACF;IAEA,MAAM,UAAU,KAAK;IACrB,IAAI,CAAC,SACH,OAAO,CACL;KACE,MAAM;KACN,OAAO;MACL,MAAMD;MACN,SAAS;MACT,QAAQC;KACV;IACF,CACF;IAGF,MAAM,OAAO,KAAK,oBAAoB,yBACpC,KAAK,mBACP,CAAC,CAAC;IACF,OACE,MAAM,SAAS,UACf,+CACF;IAQA,QAAQ,IAAI;IACZ,IAAI;KAyBF,OAAO,CAAC,MArBU,KAAK,cAAc,SAAS,YAAY;MACxD,MAAM,SAAsB,CAAC;MAC7B,KAAK,MAAM,YAAY,WAAW;OAChC,MAAM,aAAa,MAAM,QAAQ,gBAC/B,UACA,MAAM,SACN,KAAK,YAAY,KAAA,CACnB;OACA,IAAI,eAAe,KAAA,GACjB,OAAO,KAAK;QACV,MAAM,WAAW;QACjB,SAAS,WAAW;QACpB,QAAQA;OACV,CAAC;MAEL;MACA,IAAI,OAAO,SAAS,GAClB,OAAO;OAAC,MAAM;OAAa;MAAM;MAEnC,OAAO,EAAC,MAAM,KAAI;KACpB,CAAC,CACU;IACb,UAAU;KACR,QAAQ,MAAM;IAChB;GACF,CACF,CACF;GAEF,KAAK;IACH,MAAM,gBAAgB,IAAI,EAAE,CAAC,mBAC3B,eAAe,QAAQ,yCACrB,WAAW,qBAAqB,KAAK,qBAAqB,GAAG,CAC/D,CACF;IACA;GACF,KAAK;IACH,MAAM,eAAe,QAAQ,yBAAyB,YAAY;KAChE,MAAM,iBACJ,KAAK,oBAAoB,yBACvB,KAAK,mBACP;KAKF,MAAM,uBACJ,MAL2B,KAAK,oBAAoB,WACpD,KAAK,qBACL,IAAI,EACN,EAAA,CAEiB,aAAa,eAAe;KAE7C,MAAM,WAAW,WACf,KAAK,qBACL,KACA,mBACF;IACF,CAAC;IACD;GACF,KAAK,iBAAiB;IACpB,MAAM,mBAAmB,MAAM,eAC7B,QACA,kCACM,WAAW,cAAc,KAAK,qBAAqB,GAAG,CAC9D;IACA,IAAI,KAAK,WAAW,iBAAiB,SAAS,GAC5C,MAAM,KAAK,QAAQ,sBACjB,KAAK,qBACL,gBACF;IAEF;GACF;GACA,KAAK;IAKH,IAAI,KAAK,cACP,MAAM,IAAI,uBACR;KACE,MAAMC;KACN,SAAS;KACT,QAAQD;IACV,GACA,MACF;IAEF,KAAK,eAAe;IACpB,KAAK,oBAAoB,eACvB,KAAK,qBACL,IAAI,EACN;IACA,OAAO,gBAAgB,IAAI,EAAE,CAAC,mBAAmB;KAC/C,MAAM,MAAuB,CAC3B;MACE,MAAM;MACN,QAAQ;MACR,QAAQ,UAAU,QAAQ,mCACxB,WAAW,eAAe,KAAK,qBAAqB,GAAG,CACzD;KACF,CACF;KAMA,IAAI,KAAK,SACP,IAAI,KAAK;MACP,MAAM;MACN,QAAQ;MACR,QAAQ,KAAK,QAAQ,eAAe,KAAK,mBAAmB;KAC9D,CAAC;KAGH,OAAO;IACT,CAAC;GAEH,KAAK,mBAEH;GAEF,KAAK;IACH,MAAM,eAAe,QAAQ,4BAC3B,WAAW,QAAQ,KAAK,qBAAqB,GAAG,CAClD;IACA;GAEF,KAAK;IACH,IAAI,KAAK,SACP,MAAM,KAAK,QAAQ,qBACjB,KAAK,qBACL,IAAI,EACN;IAEF;GAEF,SACE,YAAY,OAAO;EACvB;EAEA,OAAO,CAAC,EAAC,MAAM,KAAI,CAAC;CACtB;AACF"}
+{"version":3,"file":"syncer-ws-message-handler.js","names":["ErrorKind.InvalidPush","ErrorOrigin.ZeroCache","ErrorKind.InvalidMessage"],"sources":["../../../../../zero-cache/src/workers/syncer-ws-message-handler.ts"],"sourcesContent":["import {ROOT_CONTEXT, context, propagation, trace} from '@opentelemetry/api';\nimport {Lock} from '@rocicorp/lock';\nimport type {LogContext} from '@rocicorp/logger';\nimport {startAsyncSpan, startSpan} from '../../../otel/src/span.ts';\nimport {version} from '../../../otel/src/version.ts';\nimport {assert, unreachable} from '../../../shared/src/asserts.ts';\nimport {ErrorKind} from '../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../zero-protocol/src/error-origin.ts';\nimport type {ErrorBody} from '../../../zero-protocol/src/error.ts';\nimport type {UpstreamWithUnparsedAnalyzeQuery} from '../../../zero-protocol/src/up.ts';\nimport type {Mutagen} from '../services/mutagen/mutagen.ts';\nimport type {Pusher} from '../services/mutagen/pusher.ts';\nimport {\n  type ConnectionContextManager,\n  type ConnectionSelector,\n} from '../services/view-syncer/connection-context-manager.ts';\nimport {type ViewSyncer} from '../services/view-syncer/view-syncer.ts';\nimport {clientGroupLogID} from '../types/client-group-log-id.ts';\nimport {ProtocolErrorWithLevel} from '../types/error-with-level.ts';\nimport type {ConnectParams} from './connect-params.ts';\nimport type {HandlerResult, MessageHandler} from './connection.ts';\n\nconst tracer = trace.getTracer('syncer-ws-server', version);\n\n/**\n * Wraps a function in an OTEL context extracted from a W3C traceparent header.\n * This enables distributed tracing from the client through zero-cache to\n * the user's API server.\n */\nfunction withTraceparent<T>(traceparent: string | undefined, fn: () => T): T {\n  if (!traceparent) {\n    return fn();\n  }\n  const extracted = propagation.extract(ROOT_CONTEXT, {traceparent});\n  return context.with(extracted, fn);\n}\n\nexport class SyncerWsMessageHandler implements MessageHandler {\n  readonly #viewSyncer: ViewSyncer;\n  readonly #mutagen: Mutagen | undefined;\n  readonly #mutationLock: Lock;\n  readonly #lc: LogContext;\n  readonly #clientGroupID: string;\n  readonly #connectionSelector: ConnectionSelector;\n  readonly #connContextManager: ConnectionContextManager;\n  readonly #pusher: Pusher | undefined;\n  #initialized = false;\n\n  constructor(\n    lc: LogContext,\n    connectParams: ConnectParams,\n    connContextManager: ConnectionContextManager,\n    viewSyncer: ViewSyncer,\n    mutagen: Mutagen | undefined,\n    pusher: Pusher | undefined,\n  ) {\n    const {clientGroupID, clientID, wsID} = connectParams;\n    this.#viewSyncer = viewSyncer;\n    this.#mutagen = mutagen;\n    this.#connContextManager = connContextManager;\n    this.#mutationLock = new Lock();\n    this.#lc = lc\n      .withContext('connection')\n      .withContext('clientID', clientID)\n      .withContext('clientGroupID', clientGroupLogID(clientGroupID))\n      .withContext('wsID', wsID);\n    this.#clientGroupID = clientGroupID;\n    this.#pusher = pusher;\n    this.#connectionSelector = {\n      clientID,\n      wsID,\n    };\n  }\n\n  async handleMessage(\n    msg: UpstreamWithUnparsedAnalyzeQuery,\n  ): Promise<HandlerResult[]> {\n    const lc = this.#lc;\n    const msgType = msg[0];\n    const viewSyncer = this.#viewSyncer;\n    switch (msgType) {\n      case 'ping':\n        lc.error?.('Ping is not supported at this layer by Zero');\n        break;\n      case 'pull':\n        lc.error?.('Pull is not supported by Zero');\n        break;\n      case 'push': {\n        return withTraceparent(msg[1].traceparent, () =>\n          startAsyncSpan<HandlerResult[]>(\n            tracer,\n            'connection.push',\n            async () => {\n              const {clientGroupID, mutations} = msg[1];\n              if (clientGroupID !== this.#clientGroupID) {\n                return [\n                  {\n                    type: 'fatal',\n                    error: {\n                      kind: ErrorKind.InvalidPush,\n                      // Neither ID is included: the message is also logged\n                      // as the reason the connection is closed.\n                      message:\n                        'clientGroupID in mutation does not match clientGroupID of connection',\n                      origin: ErrorOrigin.ZeroCache,\n                    },\n                  } satisfies HandlerResult,\n                ];\n              }\n\n              if (mutations.length === 0) {\n                return [\n                  {\n                    type: 'ok',\n                  },\n                ];\n              }\n\n              // The client only ever sends 1 mutation per push.\n              // #pusher will throw if it sees a CRUD mutation.\n              // #mutagen will throw if it see a custom mutation.\n              if (mutations[0].type === 'custom') {\n                if (!this.#pusher) {\n                  return [\n                    {\n                      type: 'fatal',\n                      error: {\n                        kind: ErrorKind.InvalidPush,\n                        message:\n                          'A ZERO_MUTATE_URL must be set in order to process custom mutations.',\n                        origin: ErrorOrigin.ZeroCache,\n                      },\n                    } satisfies HandlerResult,\n                  ];\n                }\n                return [\n                  this.#pusher.enqueuePush(this.#connectionSelector, msg[1]),\n                ];\n              }\n\n              const mutagen = this.#mutagen;\n              if (!mutagen) {\n                return [\n                  {\n                    type: 'fatal',\n                    error: {\n                      kind: ErrorKind.InvalidPush,\n                      message: `Support for legacy CRUD mutations is disabled`,\n                      origin: ErrorOrigin.ZeroCache,\n                    },\n                  } satisfies HandlerResult,\n                ];\n              }\n\n              const auth = this.#connContextManager.mustGetConnectionContext(\n                this.#connectionSelector,\n              ).auth;\n              assert(\n                auth?.type !== 'opaque',\n                'Only JWT auth is supported for CRUD mutations',\n              );\n\n              // Hold a ref on the mutagen for the duration of the push so that\n              // the connection closing mid-push does not stop the service\n              // (closing its replica handle) between two of its mutations.\n              // (The connection holds a ref while open, and messages are not\n              // dispatched after it has closed, so the service is running\n              // here.)\n              mutagen.ref();\n              try {\n                // Hold a connection-level lock while processing mutations so that:\n                // 1. Mutations are processed in the order in which they are received and\n                // 2. A single view syncer connection cannot hog multiple upstream connections.\n                const ret = await this.#mutationLock.withLock(async () => {\n                  const errors: ErrorBody[] = [];\n                  for (const mutation of mutations) {\n                    const maybeError = await mutagen.processMutation(\n                      mutation,\n                      auth?.decoded,\n                      this.#pusher !== undefined,\n                    );\n                    if (maybeError !== undefined) {\n                      errors.push({\n                        kind: maybeError[0],\n                        message: maybeError[1],\n                        origin: ErrorOrigin.ZeroCache,\n                      });\n                    }\n                  }\n                  if (errors.length > 0) {\n                    return {type: 'transient', errors} satisfies HandlerResult;\n                  }\n                  return {type: 'ok'} satisfies HandlerResult;\n                });\n                return [ret];\n              } finally {\n                mutagen.unref();\n              }\n            },\n          ),\n        );\n      }\n      case 'changeDesiredQueries':\n        await withTraceparent(msg[1].traceparent, () =>\n          startAsyncSpan(tracer, 'connection.changeDesiredQueries', () =>\n            viewSyncer.changeDesiredQueries(this.#connectionSelector, msg),\n          ),\n        );\n        break;\n      case 'updateAuth':\n        await startAsyncSpan(tracer, 'connection.updateAuth', async () => {\n          const initialConnCtx =\n            this.#connContextManager.mustGetConnectionContext(\n              this.#connectionSelector,\n            );\n          const updatedConnCtx = await this.#connContextManager.updateAuth(\n            this.#connectionSelector,\n            msg[1],\n          );\n          const authRevisionChanged =\n            updatedConnCtx.revision !== initialConnCtx.revision;\n\n          await viewSyncer.updateAuth(\n            this.#connectionSelector,\n            msg,\n            authRevisionChanged,\n          );\n        });\n        break;\n      case 'deleteClients': {\n        const deletedClientIDs = await startAsyncSpan(\n          tracer,\n          'connection.deleteClients',\n          () => viewSyncer.deleteClients(this.#connectionSelector, msg),\n        );\n        if (this.#pusher && deletedClientIDs.length > 0) {\n          await this.#pusher.deleteClientMutations(\n            this.#connectionSelector,\n            deletedClientIDs,\n          );\n        }\n        break;\n      }\n      case 'initConnection': {\n        // A socket is initialized once, from the sec-protocol header or from\n        // its first initConnection message. A second one is a client bug, so\n        // reject it before it replaces this socket's view-syncer and pusher\n        // streams.\n        if (this.#initialized) {\n          throw new ProtocolErrorWithLevel(\n            {\n              kind: ErrorKind.InvalidMessage,\n              message: 'Connection was already initialized',\n              origin: ErrorOrigin.ZeroCache,\n            },\n            'warn',\n          );\n        }\n        this.#initialized = true;\n        this.#connContextManager.initConnection(\n          this.#connectionSelector,\n          msg[1],\n        );\n        return withTraceparent(msg[1].traceparent, () => {\n          const ret: HandlerResult[] = [\n            {\n              type: 'stream',\n              source: 'viewSyncer',\n              stream: startSpan(tracer, 'connection.initConnection', () =>\n                viewSyncer.initConnection(this.#connectionSelector, msg),\n              ),\n            },\n          ];\n\n          // Given we support both CRUD and Custom mutators,\n          // we do not initialize the `pusher` unless the user has opted\n          // into custom mutations. We detect that by checking\n          // if the pushURL has been set.\n          if (this.#pusher) {\n            ret.push({\n              type: 'stream',\n              source: 'pusher',\n              stream: this.#pusher.initConnection(this.#connectionSelector),\n            });\n          }\n\n          return ret;\n        });\n      }\n      case 'closeConnection':\n        // This message is deprecated and no longer used.\n        break;\n\n      case 'inspect':\n        await startAsyncSpan(tracer, 'connection.inspect', () =>\n          viewSyncer.inspect(this.#connectionSelector, msg),\n        );\n        break;\n\n      case 'ackMutationResponses':\n        if (this.#pusher) {\n          await this.#pusher.ackMutationResponses(\n            this.#connectionSelector,\n            msg[1],\n          );\n        }\n        break;\n\n      default:\n        unreachable(msgType);\n    }\n\n    return [{type: 'ok'}];\n  }\n}\n"],"mappings":";;;;;;;;;;;;AAsBA,IAAM,SAAS,MAAM,UAAU,oBAAoB,OAAO;;;;;;AAO1D,SAAS,gBAAmB,aAAiC,IAAgB;CAC3E,IAAI,CAAC,aACH,OAAO,GAAG;CAEZ,MAAM,YAAY,YAAY,QAAQ,cAAc,EAAC,YAAW,CAAC;CACjE,OAAO,QAAQ,KAAK,WAAW,EAAE;AACnC;AAEA,IAAa,yBAAb,MAA8D;CAC5D;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA,eAAe;CAEf,YACE,IACA,eACA,oBACA,YACA,SACA,QACA;EACA,MAAM,EAAC,eAAe,UAAU,SAAQ;EACxC,KAAK,cAAc;EACnB,KAAK,WAAW;EAChB,KAAK,sBAAsB;EAC3B,KAAK,gBAAgB,IAAI,KAAK;EAC9B,KAAK,MAAM,GACR,YAAY,YAAY,CAAC,CACzB,YAAY,YAAY,QAAQ,CAAC,CACjC,YAAY,iBAAiB,iBAAiB,aAAa,CAAC,CAAC,CAC7D,YAAY,QAAQ,IAAI;EAC3B,KAAK,iBAAiB;EACtB,KAAK,UAAU;EACf,KAAK,sBAAsB;GACzB;GACA;EACF;CACF;CAEA,MAAM,cACJ,KAC0B;EAC1B,MAAM,KAAK,KAAK;EAChB,MAAM,UAAU,IAAI;EACpB,MAAM,aAAa,KAAK;EACxB,QAAQ,SAAR;GACE,KAAK;IACH,GAAG,QAAQ,6CAA6C;IACxD;GACF,KAAK;IACH,GAAG,QAAQ,+BAA+B;IAC1C;GACF,KAAK,QACH,OAAO,gBAAgB,IAAI,EAAE,CAAC,mBAC5B,eACE,QACA,mBACA,YAAY;IACV,MAAM,EAAC,eAAe,cAAa,IAAI;IACvC,IAAI,kBAAkB,KAAK,gBACzB,OAAO,CACL;KACE,MAAM;KACN,OAAO;MACL,MAAMA;MAGN,SACE;MACF,QAAQC;KACV;IACF,CACF;IAGF,IAAI,UAAU,WAAW,GACvB,OAAO,CACL,EACE,MAAM,KACR,CACF;IAMF,IAAI,UAAU,EAAE,CAAC,SAAS,UAAU;KAClC,IAAI,CAAC,KAAK,SACR,OAAO,CACL;MACE,MAAM;MACN,OAAO;OACL,MAAMD;OACN,SACE;OACF,QAAQC;MACV;KACF,CACF;KAEF,OAAO,CACL,KAAK,QAAQ,YAAY,KAAK,qBAAqB,IAAI,EAAE,CAC3D;IACF;IAEA,MAAM,UAAU,KAAK;IACrB,IAAI,CAAC,SACH,OAAO,CACL;KACE,MAAM;KACN,OAAO;MACL,MAAMD;MACN,SAAS;MACT,QAAQC;KACV;IACF,CACF;IAGF,MAAM,OAAO,KAAK,oBAAoB,yBACpC,KAAK,mBACP,CAAC,CAAC;IACF,OACE,MAAM,SAAS,UACf,+CACF;IAQA,QAAQ,IAAI;IACZ,IAAI;KAyBF,OAAO,CAAC,MArBU,KAAK,cAAc,SAAS,YAAY;MACxD,MAAM,SAAsB,CAAC;MAC7B,KAAK,MAAM,YAAY,WAAW;OAChC,MAAM,aAAa,MAAM,QAAQ,gBAC/B,UACA,MAAM,SACN,KAAK,YAAY,KAAA,CACnB;OACA,IAAI,eAAe,KAAA,GACjB,OAAO,KAAK;QACV,MAAM,WAAW;QACjB,SAAS,WAAW;QACpB,QAAQA;OACV,CAAC;MAEL;MACA,IAAI,OAAO,SAAS,GAClB,OAAO;OAAC,MAAM;OAAa;MAAM;MAEnC,OAAO,EAAC,MAAM,KAAI;KACpB,CAAC,CACU;IACb,UAAU;KACR,QAAQ,MAAM;IAChB;GACF,CACF,CACF;GAEF,KAAK;IACH,MAAM,gBAAgB,IAAI,EAAE,CAAC,mBAC3B,eAAe,QAAQ,yCACrB,WAAW,qBAAqB,KAAK,qBAAqB,GAAG,CAC/D,CACF;IACA;GACF,KAAK;IACH,MAAM,eAAe,QAAQ,yBAAyB,YAAY;KAChE,MAAM,iBACJ,KAAK,oBAAoB,yBACvB,KAAK,mBACP;KAKF,MAAM,uBACJ,MAL2B,KAAK,oBAAoB,WACpD,KAAK,qBACL,IAAI,EACN,EAAA,CAEiB,aAAa,eAAe;KAE7C,MAAM,WAAW,WACf,KAAK,qBACL,KACA,mBACF;IACF,CAAC;IACD;GACF,KAAK,iBAAiB;IACpB,MAAM,mBAAmB,MAAM,eAC7B,QACA,kCACM,WAAW,cAAc,KAAK,qBAAqB,GAAG,CAC9D;IACA,IAAI,KAAK,WAAW,iBAAiB,SAAS,GAC5C,MAAM,KAAK,QAAQ,sBACjB,KAAK,qBACL,gBACF;IAEF;GACF;GACA,KAAK;IAKH,IAAI,KAAK,cACP,MAAM,IAAI,uBACR;KACE,MAAMC;KACN,SAAS;KACT,QAAQD;IACV,GACA,MACF;IAEF,KAAK,eAAe;IACpB,KAAK,oBAAoB,eACvB,KAAK,qBACL,IAAI,EACN;IACA,OAAO,gBAAgB,IAAI,EAAE,CAAC,mBAAmB;KAC/C,MAAM,MAAuB,CAC3B;MACE,MAAM;MACN,QAAQ;MACR,QAAQ,UAAU,QAAQ,mCACxB,WAAW,eAAe,KAAK,qBAAqB,GAAG,CACzD;KACF,CACF;KAMA,IAAI,KAAK,SACP,IAAI,KAAK;MACP,MAAM;MACN,QAAQ;MACR,QAAQ,KAAK,QAAQ,eAAe,KAAK,mBAAmB;KAC9D,CAAC;KAGH,OAAO;IACT,CAAC;GAEH,KAAK,mBAEH;GAEF,KAAK;IACH,MAAM,eAAe,QAAQ,4BAC3B,WAAW,QAAQ,KAAK,qBAAqB,GAAG,CAClD;IACA;GAEF,KAAK;IACH,IAAI,KAAK,SACP,MAAM,KAAK,QAAQ,qBACjB,KAAK,qBACL,IAAI,EACN;IAEF;GAEF,SACE,YAAY,OAAO;EACvB;EAEA,OAAO,CAAC,EAAC,MAAM,KAAI,CAAC;CACtB;AACF"}

package/out/zero-cache/src/workers/syncer.js

@@ -6,6 +6,7 @@ import { ProtocolError, isProtocolError } from "../../../zero-protocol/src/error
 import "../config/zero-config.js";
 import { getOrCreateCounter, getOrCreateGauge, getOrCreateNativeHistogram, getOrCreateUpDownCounter } from "../observability/metrics.js";
 import { installWebSocketReceiver } from "../types/websocket-handoff.js";
+import { clientGroupLogID } from "../types/client-group-log-id.js";
 import { Connection, sendError } from "./connection.js";
 import { expiredAffinityCookieHeader, rehome } from "./rehome.js";
 import { createNotifierFrom, subscribeTo } from "./replicator.js";
@@ -341,27 +342,26 @@ var Syncer = class {
 		ws.once("close", () => {
 			this.#webSocketOpenConnections.add(-1, metricAttributes);
 		});
-		this.#lc.debug?.("creating connection", params.clientGroupID, params.clientID);
+		this.#lc.debug?.("creating connection", clientGroupLogID(params.clientGroupID), params.clientID);
 		recordConnectionAttempted();
 		const { clientID, clientGroupID, auth, userID } = params;
 		const hasProvidedAuth = auth !== void 0 && auth !== "";
 		const incomingUserID = userID ?? null;
-		if (hasProvidedAuth) {
-			const tokenOptions = tokenConfigOptions(this.#config.auth ?? {});
-			const hasPushOrMutate = this.#config?.push?.url !== void 0 || this.#config?.mutate?.url !== void 0;
-			const hasQueries = this.#config?.query?.url !== void 0 || this.#config?.getQueries?.url !== void 0;
-			if (!(tokenOptions.length === 1) && !(hasPushOrMutate && hasQueries)) {
-				recordConnectionFailure("configuration");
-				throw new Error("Exactly one of jwk, secret, or jwksUrl must be set in order to verify tokens but actually the following were set: " + JSON.stringify(tokenOptions) + ". You may also set both ZERO_MUTATE_URL and ZERO_QUERY_URL to enable custom mutations and queries without passing token verification options.");
-			}
+		const tokenOptions = tokenConfigOptions(this.#config.auth ?? {});
+		const hasPushOrMutate = this.#config?.push?.url !== void 0 || this.#config?.mutate?.url !== void 0;
+		const hasQueries = this.#config?.query?.url !== void 0 || this.#config?.getQueries?.url !== void 0;
+		const hasCustomEndpoints = hasPushOrMutate && hasQueries;
+		if (tokenOptions.length > 1 || hasProvidedAuth && tokenOptions.length === 0 && !hasCustomEndpoints) {
+			recordConnectionFailure("configuration");
+			throw new Error("Exactly one of jwk, secret, or jwksUrl must be set in order to verify tokens but actually the following were set: " + JSON.stringify(tokenOptions) + ". You may also set both ZERO_MUTATE_URL and ZERO_QUERY_URL to enable custom mutations and queries without passing token verification options.");
 		}
 		let initialAuth;
 		try {
-			initialAuth = await resolveAuth(this.#lc.withContext("clientGroupID", clientGroupID).withContext("clientID", clientID), void 0, incomingUserID, auth, this.#validateLegacyJWT);
+			initialAuth = await resolveAuth(this.#lc.withContext("clientGroupID", clientGroupLogID(clientGroupID)).withContext("clientID", clientID), void 0, incomingUserID, auth, this.#validateLegacyJWT);
 		} catch (e) {
 			if (isProtocolError(e)) {
 				this.#lc.warn?.("Rejecting sync connection during initial auth resolution", {
-					clientGroupID,
+					clientGroupID: clientGroupLogID(clientGroupID),
 					clientID,
 					incomingUserID,
 					hasProvidedAuth,
@@ -377,7 +377,7 @@ var Syncer = class {
 		}
 		if (ws.readyState !== WebSocket.OPEN) {
 			this.#lc.debug?.("websocket closed while resolving auth", {
-				clientGroupID,
+				clientGroupID: clientGroupLogID(clientGroupID),
 				clientID,
 				readyState: ws.readyState
 			});
@@ -405,7 +405,7 @@ var Syncer = class {
 		}
 		const existing = this.#connections.get(clientID);
 		if (existing) {
-			if (existing.wsID === params.wsID) this.#lc.warn?.(`duplicate connection for client ${clientID} with the same wsID ${params.wsID}, closing existing connection`, { clientGroupID });
+			if (existing.wsID === params.wsID) this.#lc.warn?.(`duplicate connection for client ${clientID} with the same wsID ${params.wsID}, closing existing connection`, { clientGroupID: clientGroupLogID(clientGroupID) });
 			this.#lc.debug?.(`client ${clientID} already connected, closing existing connection`);
 			existing.close(`replaced by ${params.wsID}`);
 		}
@@ -444,7 +444,7 @@ var Syncer = class {
 			recordConnectionSuccess();
 		} else recordConnectionFailure("protocol_version");
 		if (params.initConnectionMsg) {
-			this.#lc.debug?.("handling init connection message from sec header", params.clientGroupID, params.clientID);
+			this.#lc.debug?.("handling init connection message from sec header", clientGroupLogID(params.clientGroupID), params.clientID);
 			await connection.handleInitConnection(JSON.stringify(params.initConnectionMsg));
 		}
 	};
@@ -491,7 +491,7 @@ var Syncer = class {
 		while (this.#viewSyncers.size) {
 			await this.#drainCoordinator.forceDrainTimeout;
 			for (const vs of this.#viewSyncers.getServices()) {
-				this.#lc.debug?.(`draining view-syncer ${vs.id} (forced)`);
+				this.#lc.debug?.(`draining view-syncer ${clientGroupLogID(vs.id)} (forced)`);
 				vs.stop();
 				break;
 			}