@rocicorp/zero 1.12.0-canary.11 → 1.12.0-canary.13

package/out/zero-cache/src/workers/connection.js.map

@@ -1 +1 @@
-{"version":3,"file":"connection.js","names":["valita.parse","ErrorKind.VersionNotSupported","ErrorOrigin.ZeroCache","ErrorKind.InvalidMessage","WebSocket","ErrorKind.Internal"],"sources":["../../../../../zero-cache/src/workers/connection.ts"],"sourcesContent":["import {pipeline, Readable, Writable} from 'node:stream';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport type {CloseEvent, Data, ErrorEvent} from 'ws';\nimport WebSocket, {createWebSocketStream} from 'ws';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport * as valita from '../../../shared/src/valita.ts';\nimport type {ConnectedMessage} from '../../../zero-protocol/src/connect.ts';\nimport type {Downstream} from '../../../zero-protocol/src/down.ts';\nimport {ErrorKind} from '../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../zero-protocol/src/error-origin.ts';\nimport type {ErrorBody} from '../../../zero-protocol/src/error.ts';\nimport {\n  isProtocolError,\n  type ProtocolError,\n} from '../../../zero-protocol/src/error.ts';\nimport {\n  POKE_CHUNK_PROTOCOL_VERSION,\n  type PokeChunk,\n  type PokeEndMessage,\n  type PokePartMessage,\n} from '../../../zero-protocol/src/poke.ts';\nimport {\n  MIN_SERVER_SUPPORTED_SYNC_PROTOCOL,\n  PROTOCOL_VERSION,\n} from '../../../zero-protocol/src/protocol-version.ts';\nimport {\n  upstreamSchemaWithUnparsedAnalyzeQuery,\n  type UpstreamWithUnparsedAnalyzeQuery,\n} from '../../../zero-protocol/src/up.ts';\nimport {getOrCreateCounter} from '../observability/metrics.ts';\nimport type {ViewSyncerDownstream} from '../types/downstream.ts';\nimport {\n  ProtocolErrorWithLevel,\n  getLogLevel,\n  wrapWithProtocolError,\n} from '../types/error-with-level.ts';\nimport {PokeChunkEncoder} from '../types/poke-chunk.ts';\nimport type {Source} from '../types/streams.ts';\nimport type {ConnectParams} from './connect-params.ts';\n\nexport type HandlerResult =\n  | {\n      type: 'ok';\n    }\n  | {\n      type: 'fatal';\n      error: ErrorBody;\n    }\n  | {\n      type: 'transient';\n      errors: ErrorBody[];\n    }\n  | StreamResult;\n\nexport type StreamResult =\n  | {\n      type: 'stream';\n      source: 'viewSyncer';\n      stream: Source<ViewSyncerDownstream>;\n    }\n  | {\n      type: 'stream';\n      source: 'pusher';\n      stream: Source<Downstream>;\n    };\n\nexport interface MessageHandler {\n  handleMessage(\n    msg: UpstreamWithUnparsedAnalyzeQuery,\n  ): Promise<HandlerResult[]>;\n}\n\nfunction hasOwn(value: unknown, property: string): boolean {\n  return (\n    typeof value === 'object' &&\n    value !== null &&\n    Object.hasOwn(value, property)\n  );\n}\n\nfunction containsLegacyQuery(message: unknown): boolean {\n  if (!Array.isArray(message)) {\n    return false;\n  }\n\n  const body = message[1];\n  if (typeof body !== 'object' || body === null) {\n    return false;\n  }\n\n  if (\n    message[0] !== 'initConnection' &&\n    message[0] !== 'changeDesiredQueries'\n  ) {\n    return false;\n  }\n\n  const patch = body['desiredQueriesPatch'];\n  return (\n    Array.isArray(patch) && patch.some(operation => hasOwn(operation, 'ast'))\n  );\n}\n\nconst LEGACY_QUERIES_DISABLED_MESSAGE =\n  'Legacy queries are disabled by this Zero server. This client must use custom queries instead of sending query ASTs directly. To temporarily restore compatibility, set ZERO_ALLOW_LEGACY_QUERIES=true on zero-cache.';\n\n// Exported for testing purposes.\n//\n// Client messages are parsed leniently, so a newer client can add message\n// kinds and fields:\n// - A message of a kind this server doesn't know parses as an\n//   UnrecognizedTuple, for the connection to drop. Malformed messages still\n//   throw.\n// - Messages are parsed in strip mode, so fields this server doesn't know are\n//   dropped. Strip rather than passthrough: nothing this server doesn't know\n//   reaches the CVR or the API server, and valita's passthrough drops unknown\n//   fields anyway on any object with a transformed field below it (such as\n//   jsonSchema args). ASTs stay strict in the schema, because a server that\n//   ignored an AST field would run a different query.\nexport function parseUpstreamMessage(\n  value: unknown,\n  allowLegacyQueries: boolean,\n): UpstreamWithUnparsedAnalyzeQuery | valita.UnrecognizedTuple {\n  if (!allowLegacyQueries && containsLegacyQuery(value)) {\n    throw new Error(LEGACY_QUERIES_DISABLED_MESSAGE);\n  }\n  return valita.parse(value, upstreamSchemaWithUnparsedAnalyzeQuery, 'strip');\n}\n\n// Ensures that a downstream message is sent at least every interval, sending a\n// 'pong' if necessary. This is set to be slightly longer than the client-side\n// PING_INTERVAL of 5 seconds, so that in the common case, 'pong's are sent in\n// response to client-initiated 'ping's. However, if the inbound stream is\n// backed up because a command is taking a long time to process, the pings\n// will be stuck in the queue (i.e. back-pressured), in which case pongs will\n// be manually sent to notify the client of server liveness.\n//\n// This is equivalent to what is done for Postgres keepalives on the\n// replication stream (which can similarly be back-pressured):\n// https://github.com/rocicorp/mono/blob/f98cb369a2dbb15650328859c732db358f187ef0/packages/zero-cache/src/services/change-source/pg/logical-replication/stream.ts#L21\nconst DOWNSTREAM_MSG_INTERVAL_MS = 6_000;\nexport const WEBSOCKET_SEND_TIMEOUT_MS = 10_000;\nconst PROTOCOL_VERSION_ATTRIBUTE = 'protocol.version';\nconst EVENT_TYPE_ATTRIBUTE = 'event.type';\n\n/**\n * Represents a connection between the client and server.\n *\n * Handles incoming messages on the connection and dispatches\n * them to the correct service.\n *\n * Listens to the ViewSyncer and sends messages to the client.\n */\nexport class Connection {\n  readonly #ws: WebSocket;\n  readonly #wsID: string;\n  readonly #protocolVersion: number;\n  readonly #lc: LogContext;\n  readonly #onClose: () => void;\n  readonly #messageHandler: MessageHandler;\n  readonly #allowLegacyQueries: boolean;\n  readonly #downstreamSender: DownstreamSender;\n  readonly #downstreamMsgTimer: NodeJS.Timeout | undefined;\n  readonly #webSocketErrors = getOrCreateCounter(\n    'sync',\n    'websocket.errors',\n    'Client WebSocket error events.',\n  );\n\n  #viewSyncerOutboundStream: Source<ViewSyncerDownstream> | undefined;\n  #pusherOutboundStream: Source<Downstream> | undefined;\n  #closed = false;\n  #loggedUnknownMessage = false;\n\n  constructor(\n    lc: LogContext,\n    connectParams: ConnectParams,\n    ws: WebSocket,\n    allowLegacyQueries: boolean,\n    messageHandler: MessageHandler,\n    onClose: () => void,\n  ) {\n    const {clientGroupID, clientID, wsID, protocolVersion} = connectParams;\n    this.#messageHandler = messageHandler;\n    this.#allowLegacyQueries = allowLegacyQueries;\n\n    this.#ws = ws;\n    this.#wsID = wsID;\n    this.#protocolVersion = protocolVersion;\n\n    this.#lc = lc\n      .withContext('connection')\n      .withContext('clientID', clientID)\n      .withContext('clientGroupID', clientGroupID)\n      .withContext('wsID', wsID);\n    this.#downstreamSender = new DownstreamSender(\n      this.#lc,\n      ws,\n      protocolVersion,\n    );\n    this.#lc.debug?.('new connection');\n    this.#onClose = onClose;\n\n    this.#ws.addEventListener('close', this.#handleClose);\n    this.#ws.addEventListener('error', this.#handleError);\n\n    this.#proxyInbound();\n    this.#downstreamMsgTimer = setInterval(\n      this.#maybeSendPong,\n      DOWNSTREAM_MSG_INTERVAL_MS / 2,\n    );\n  }\n\n  /**\n   * Checks the protocol version and errors for unsupported protocols,\n   * sending the initial `connected` response on success.\n   *\n   * This is early in the connection lifecycle because {@link #handleMessage}\n   * will only parse messages with schema(s) of supported protocol versions.\n   */\n  init(): boolean {\n    if (\n      this.#protocolVersion > PROTOCOL_VERSION ||\n      this.#protocolVersion < MIN_SERVER_SUPPORTED_SYNC_PROTOCOL\n    ) {\n      this.#closeWithError({\n        kind: ErrorKind.VersionNotSupported,\n        message: `server is at sync protocol v${PROTOCOL_VERSION} and does not support v${\n          this.#protocolVersion\n        }. The ${\n          this.#protocolVersion > PROTOCOL_VERSION ? 'server' : 'client'\n        } must be updated to a newer release.`,\n        origin: ErrorOrigin.ZeroCache,\n      });\n    } else {\n      const connectedMessage: ConnectedMessage = [\n        'connected',\n        {wsid: this.#wsID, timestamp: Date.now()},\n      ];\n      this.send(connectedMessage, 'ignore-backpressure');\n      return true;\n    }\n    return false;\n  }\n\n  get wsID(): string {\n    return this.#wsID;\n  }\n\n  close(reason: string, ...args: unknown[]) {\n    if (this.#closed) {\n      return;\n    }\n    this.#closed = true;\n    this.#lc.info?.(`closing connection: ${reason}`, ...args);\n    this.#ws.removeEventListener('close', this.#handleClose);\n    this.#ws.removeEventListener('error', this.#handleError);\n    this.#viewSyncerOutboundStream?.cancel();\n    this.#viewSyncerOutboundStream = undefined;\n    this.#pusherOutboundStream?.cancel();\n    this.#pusherOutboundStream = undefined;\n    this.#onClose();\n    if (this.#ws.readyState !== this.#ws.CLOSED) {\n      this.#ws.close();\n    }\n    clearTimeout(this.#downstreamMsgTimer);\n\n    // spin down services if we have\n    // no more client connections for the client group?\n  }\n\n  handleInitConnection(initConnectionMsg: string) {\n    return this.#handleMessage({data: initConnectionMsg});\n  }\n\n  #handleMessage = async (event: {data: Data}) => {\n    const data = event.data.toString();\n    if (this.#closed) {\n      this.#lc.debug?.('Ignoring message received after closed', data);\n      return;\n    }\n\n    let msg;\n    try {\n      const value = JSON.parse(data);\n      msg = parseUpstreamMessage(value, this.#allowLegacyQueries);\n    } catch (e) {\n      const errorBody = {\n        kind: ErrorKind.InvalidMessage,\n        message: String(e),\n        origin: ErrorOrigin.ZeroCache,\n      } as const;\n      this.#closeWithError(\n        errorBody,\n        new ProtocolErrorWithLevel(errorBody, 'warn'),\n      );\n      return;\n    }\n\n    if (!upstreamSchemaWithUnparsedAnalyzeQuery.isKnown(msg)) {\n      // A message kind from a newer client.\n      if (!this.#loggedUnknownMessage) {\n        this.#loggedUnknownMessage = true;\n        this.#lc.info?.(\n          `Ignoring messages of unknown type, starting with ${JSON.stringify(msg[0])}`,\n        );\n      }\n      return;\n    }\n\n    try {\n      const msgType = msg[0];\n      if (msgType === 'ping') {\n        this.send(['pong', {}], 'ignore-backpressure');\n        return;\n      }\n\n      const result = await this.#messageHandler.handleMessage(msg);\n      for (const r of result) {\n        this.#handleMessageResult(r);\n      }\n    } catch (e) {\n      this.#closeWithThrown(e);\n    }\n  };\n\n  #handleMessageResult(result: HandlerResult): void {\n    switch (result.type) {\n      case 'fatal':\n        this.#closeWithError(result.error);\n        break;\n      case 'ok':\n        break;\n      case 'stream': {\n        switch (result.source) {\n          case 'viewSyncer':\n            assert(\n              this.#viewSyncerOutboundStream === undefined,\n              'Outbound stream already set for this connection!',\n            );\n            this.#viewSyncerOutboundStream = result.stream;\n            break;\n          case 'pusher':\n            assert(\n              this.#pusherOutboundStream === undefined,\n              'Outbound stream already set for this connection!',\n            );\n            this.#pusherOutboundStream = result.stream;\n            break;\n        }\n        if (result.source === 'viewSyncer') {\n          this.#proxyOutbound(result.stream, (downstream, callback) =>\n            this.send(downstream.message, callback, downstream.serialized),\n          );\n        } else {\n          this.#proxyOutbound(result.stream, (downstream, callback) =>\n            this.send(downstream, callback),\n          );\n        }\n        break;\n      }\n      case 'transient': {\n        for (const error of result.errors) {\n          this.sendError(error);\n        }\n      }\n    }\n  }\n\n  #handleClose = (e: CloseEvent) => {\n    const {code, reason, wasClean} = e;\n    if (!wasClean) {\n      this.#recordWebSocketError('unclean_close');\n    }\n    this.close('WebSocket close event', {code, reason, wasClean});\n  };\n\n  #handleError = (e: ErrorEvent) => {\n    this.#recordWebSocketError('error_event');\n    this.#lc.warn?.('WebSocket error event', e.message, e.error);\n  };\n\n  #recordWebSocketError(eventType: string) {\n    this.#webSocketErrors.add(1, {\n      [PROTOCOL_VERSION_ATTRIBUTE]: this.#protocolVersion,\n      [EVENT_TYPE_ATTRIBUTE]: eventType,\n    });\n  }\n\n  #proxyInbound() {\n    pipeline(\n      createWebSocketStream(this.#ws),\n      new Writable({\n        write: (data, _encoding, callback) => {\n          this.#handleMessage({data}).then(() => callback(), callback);\n        },\n      }),\n      // The done callback is not used, as #handleClose and #handleError,\n      // configured on the underlying WebSocket, provide more complete\n      // information.\n      () => {},\n    );\n  }\n\n  #proxyOutbound<T>(\n    outboundStream: Source<T>,\n    sendMessage: (\n      downstream: T,\n      callback: (err?: Error | null) => void,\n    ) => void,\n  ) {\n    // Note: createWebSocketStream() is avoided here in order to control\n    //       exception handling with #closeWithThrown(). If the Writable\n    //       from createWebSocketStream() were instead used, exceptions\n    //       from the outboundStream result in the Writable closing the\n    //       the websocket before the error message can be sent.\n    pipeline(\n      Readable.from(outboundStream),\n      new Writable({\n        objectMode: true,\n        write: (downstream: T, _encoding, callback) =>\n          sendMessage(downstream, callback),\n      }),\n      e =>\n        e\n          ? this.#closeWithThrown(e)\n          : this.close(`downstream closed by ViewSyncer`),\n    );\n  }\n\n  #closeWithThrown(e: unknown) {\n    const errorBody =\n      findProtocolError(e)?.errorBody ?? wrapWithProtocolError(e).errorBody;\n\n    this.#closeWithError(errorBody, e);\n  }\n\n  #closeWithError(errorBody: ErrorBody, thrown?: unknown) {\n    this.sendError(errorBody, thrown);\n    this.close(\n      `${errorBody.kind} (${errorBody.origin}): ${errorBody.message}`,\n      errorBody,\n    );\n  }\n\n  #lastDownstreamMsgTime = Date.now();\n\n  #maybeSendPong = () => {\n    if (Date.now() - this.#lastDownstreamMsgTime > DOWNSTREAM_MSG_INTERVAL_MS) {\n      this.#lc.debug?.('manually sending pong');\n      this.send(['pong', {}], 'ignore-backpressure');\n    }\n  };\n\n  send(\n    data: Downstream,\n    callback: ((err?: Error | null) => void) | 'ignore-backpressure',\n    serialized?: string | undefined,\n  ) {\n    this.#lastDownstreamMsgTime = Date.now();\n    this.#downstreamSender.send(data, callback, serialized);\n  }\n\n  sendError(errorBody: ErrorBody, thrown?: unknown) {\n    sendError(this.#lc, this.#ws, errorBody, thrown);\n  }\n}\n\nexport type WebSocketLike = Pick<WebSocket, 'readyState'> & {\n  send(data: string | PokeChunk, cb?: (err?: Error) => void): void;\n};\n\n// Exported for testing purposes.\nexport function send(\n  lc: LogContext,\n  ws: WebSocketLike,\n  data: Downstream,\n  callback: ((err?: Error | null) => void) | 'ignore-backpressure',\n  serialized?: string | undefined,\n) {\n  if (ws.readyState === WebSocket.OPEN) {\n    serialized ??= JSON.stringify(data);\n    if (callback === 'ignore-backpressure') {\n      ws.send(serialized);\n      return;\n    }\n\n    let completed = false;\n    const timer = setTimeout(() => {\n      complete(webSocketSendTimeoutError());\n    }, WEBSOCKET_SEND_TIMEOUT_MS);\n    timer.unref();\n\n    const complete = (error?: Error | null) => {\n      if (completed) {\n        return;\n      }\n      completed = true;\n      clearTimeout(timer);\n      callback(error);\n    };\n\n    ws.send(serialized, complete);\n  } else {\n    lc.debug?.(`Dropping outbound message on ws (state: ${ws.readyState})`, {\n      dropped: data,\n    });\n    if (callback !== 'ignore-backpressure') {\n      callback(\n        new ProtocolErrorWithLevel(\n          {\n            kind: ErrorKind.Internal,\n            message: 'WebSocket closed',\n            origin: ErrorOrigin.ZeroCache,\n          },\n          'info',\n        ),\n      );\n    }\n  }\n}\n\n/**\n * Sends downstream messages in the format supported by a connection's sync\n * protocol version. Keep version forks contained here so the view-syncer can\n * continue producing one canonical poke representation.\n *\n * Exported for compatibility testing.\n */\nexport class DownstreamSender {\n  readonly #lc: LogContext;\n  readonly #ws: WebSocketLike;\n  readonly #protocolVersion: number;\n  #pokeChunkEncoder: PokeChunkEncoder | undefined;\n\n  constructor(lc: LogContext, ws: WebSocketLike, protocolVersion: number) {\n    this.#lc = lc;\n    this.#ws = ws;\n    this.#protocolVersion = protocolVersion;\n  }\n\n  send(\n    data: Downstream,\n    callback: ((err?: Error | null) => void) | 'ignore-backpressure',\n    serialized?: string | undefined,\n  ): void {\n    if (this.#protocolVersion >= POKE_CHUNK_PROTOCOL_VERSION) {\n      switch (data[0]) {\n        case 'pokeStart':\n          assert(\n            this.#pokeChunkEncoder === undefined,\n            'cannot start a poke while another poke is in progress',\n          );\n          this.#pokeChunkEncoder = new PokeChunkEncoder();\n          break;\n        case 'pokePart':\n          this.#sendPokePart(data, callback, serialized);\n          return;\n        case 'pokeEnd':\n          this.#sendPokeEnd(data, callback);\n          return;\n      }\n    }\n    send(this.#lc, this.#ws, data, callback, serialized);\n  }\n\n  #sendPokePart(\n    pokePart: PokePartMessage,\n    callback: ((err?: Error | null) => void) | 'ignore-backpressure',\n    serialized?: string | undefined,\n  ): void {\n    const encoder = this.#pokeChunkEncoder;\n    assert(encoder, 'pokePart received without a pokeStart');\n\n    serialized ??= JSON.stringify(pokePart);\n    assert(\n      serialized.startsWith(POKE_PART_PREFIX) && serialized.endsWith(']'),\n      'invalid serialized pokePart',\n    );\n    void encoder\n      .addPatch(serialized.slice(POKE_PART_PREFIX.length, -1), chunk =>\n        sendBinary(this.#lc, this.#ws, chunk),\n      )\n      .then(\n        () => invokeCallback(callback),\n        error => invokeCallback(callback, toError(error)),\n      );\n  }\n\n  #sendPokeEnd(\n    pokeEnd: PokeEndMessage,\n    callback: ((err?: Error | null) => void) | 'ignore-backpressure',\n  ): void {\n    const encoder = this.#pokeChunkEncoder;\n    assert(encoder, 'pokeEnd received without a pokeStart');\n    this.#pokeChunkEncoder = undefined;\n\n    if (pokeEnd[1].cancel) {\n      encoder.cancel();\n      send(this.#lc, this.#ws, pokeEnd, callback);\n      return;\n    }\n\n    void encoder\n      .finish(chunk => sendBinary(this.#lc, this.#ws, chunk))\n      .then(() => sendAsync(this.#lc, this.#ws, pokeEnd))\n      .then(\n        () => invokeCallback(callback),\n        error => invokeCallback(callback, toError(error)),\n      );\n  }\n}\n\nconst POKE_PART_PREFIX = '[\"pokePart\",';\n\nfunction sendAsync(\n  lc: LogContext,\n  ws: WebSocketLike,\n  data: Downstream,\n): Promise<void> {\n  return new Promise((resolve, reject) => {\n    send(lc, ws, data, error => (error ? reject(error) : resolve()));\n  });\n}\n\nfunction sendBinary(\n  lc: LogContext,\n  ws: WebSocketLike,\n  chunk: PokeChunk,\n): Promise<void> {\n  return new Promise((resolve, reject) => {\n    if (ws.readyState !== WebSocket.OPEN) {\n      const error = webSocketClosedError();\n      lc.debug?.(\n        `Dropping outbound binary message on ws (state: ${ws.readyState})`,\n      );\n      reject(error);\n      return;\n    }\n    let completed = false;\n    const timer = setTimeout(\n      () => complete(webSocketSendTimeoutError()),\n      WEBSOCKET_SEND_TIMEOUT_MS,\n    );\n    timer.unref();\n\n    const complete = (error?: Error | null) => {\n      if (completed) {\n        return;\n      }\n      completed = true;\n      clearTimeout(timer);\n      error ? reject(error) : resolve();\n    };\n\n    ws.send(chunk, complete);\n  });\n}\n\nfunction invokeCallback(\n  callback: ((err?: Error | null) => void) | 'ignore-backpressure',\n  error?: Error,\n): void {\n  if (callback !== 'ignore-backpressure') {\n    callback(error);\n  }\n}\n\nfunction toError(error: unknown): Error {\n  return error instanceof Error ? error : new Error(String(error));\n}\n\nfunction webSocketClosedError(): ProtocolErrorWithLevel {\n  return new ProtocolErrorWithLevel(\n    {\n      kind: ErrorKind.Internal,\n      message: 'WebSocket closed',\n      origin: ErrorOrigin.ZeroCache,\n    },\n    'info',\n  );\n}\n\nfunction webSocketSendTimeoutError(): ProtocolErrorWithLevel {\n  return new ProtocolErrorWithLevel(\n    {\n      kind: ErrorKind.Internal,\n      message: `WebSocket send timed out after ${WEBSOCKET_SEND_TIMEOUT_MS} ms`,\n      origin: ErrorOrigin.ZeroCache,\n    },\n    'info',\n  );\n}\n\nexport function sendError(\n  lc: LogContext,\n  ws: WebSocket,\n  errorBody: ErrorBody,\n  thrown?: unknown,\n) {\n  lc = lc.withContext('errorKind', errorBody.kind);\n\n  let logLevel: LogLevel;\n\n  // If the thrown error is a ProtocolErrorWithLevel, its explicit logLevel takes precedence\n  if (thrown instanceof ProtocolErrorWithLevel) {\n    logLevel = thrown.logLevel;\n  }\n  // Errors with errno or transient socket codes are low-level, transient I/O issues\n  // (e.g., EPIPE, ECONNRESET) and should be warnings, not errors\n  else if (\n    hasErrno(thrown) ||\n    hasTransientSocketCode(thrown) ||\n    isTransientSocketMessage(errorBody.message)\n  ) {\n    logLevel = 'warn';\n  }\n  // Fallback: check errorBody.kind for errors that weren't thrown as ProtocolErrorWithLevel\n  else if (\n    errorBody.kind === ErrorKind.ClientNotFound ||\n    errorBody.kind === ErrorKind.TransformFailed\n  ) {\n    logLevel = 'warn';\n  } else {\n    logLevel = thrown ? getLogLevel(thrown) : 'info';\n  }\n\n  lc[logLevel]?.('Sending error on WebSocket', errorBody, thrown ?? '');\n  send(lc, ws, ['error', errorBody], 'ignore-backpressure');\n}\n\nfunction findProtocolError(error: unknown): ProtocolError | undefined {\n  if (isProtocolError(error)) {\n    return error;\n  }\n  if (error instanceof Error && error.cause) {\n    return findProtocolError(error.cause);\n  }\n  return undefined;\n}\n\nfunction hasErrno(error: unknown): boolean {\n  return Boolean(\n    error &&\n    typeof error === 'object' &&\n    'errno' in error &&\n    typeof (error as {errno: unknown}).errno !== 'undefined',\n  );\n}\n\n// System error codes that indicate transient socket conditions.\n// These are checked via the `code` property on errors.\nconst TRANSIENT_SOCKET_ERROR_CODES = new Set([\n  'EPIPE',\n  'ECONNRESET',\n  'ECANCELED',\n]);\n\n// Error messages that indicate transient socket conditions but don't have\n// standard error codes (e.g., WebSocket library errors).\nconst TRANSIENT_SOCKET_MESSAGE_PATTERNS = [\n  'socket was closed while data was being compressed',\n];\n\nfunction hasTransientSocketCode(error: unknown): boolean {\n  if (!error || typeof error !== 'object') {\n    return false;\n  }\n  const maybeCode =\n    'code' in error ? String((error as {code?: unknown}).code) : undefined;\n  return Boolean(\n    maybeCode && TRANSIENT_SOCKET_ERROR_CODES.has(maybeCode.toUpperCase()),\n  );\n}\n\nfunction isTransientSocketMessage(message: string | undefined): boolean {\n  if (!message) {\n    return false;\n  }\n  const lower = message.toLowerCase();\n  return TRANSIENT_SOCKET_MESSAGE_PATTERNS.some(pattern =>\n    lower.includes(pattern),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;AAwEA,SAAS,OAAO,OAAgB,UAA2B;CACzD,OACE,OAAO,UAAU,YACjB,UAAU,QACV,OAAO,OAAO,OAAO,QAAQ;AAEjC;AAEA,SAAS,oBAAoB,SAA2B;CACtD,IAAI,CAAC,MAAM,QAAQ,OAAO,GACxB,OAAO;CAGT,MAAM,OAAO,QAAQ;CACrB,IAAI,OAAO,SAAS,YAAY,SAAS,MACvC,OAAO;CAGT,IACE,QAAQ,OAAO,oBACf,QAAQ,OAAO,wBAEf,OAAO;CAGT,MAAM,QAAQ,KAAK;CACnB,OACE,MAAM,QAAQ,KAAK,KAAK,MAAM,MAAK,cAAa,OAAO,WAAW,KAAK,CAAC;AAE5E;AAEA,IAAM,kCACJ;AAeF,SAAgB,qBACd,OACA,oBAC6D;CAC7D,IAAI,CAAC,sBAAsB,oBAAoB,KAAK,GAClD,MAAM,IAAI,MAAM,+BAA+B;CAEjD,OAAOA,MAAa,OAAO,wCAAwC,OAAO;AAC5E;AAaA,IAAM,6BAA6B;AACnC,IAAa,4BAA4B;AACzC,IAAM,6BAA6B;AACnC,IAAM,uBAAuB;;;;;;;;;AAU7B,IAAa,aAAb,MAAwB;CACtB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA,mBAA4B,mBAC1B,QACA,oBACA,gCACF;CAEA;CACA;CACA,UAAU;CACV,wBAAwB;CAExB,YACE,IACA,eACA,IACA,oBACA,gBACA,SACA;EACA,MAAM,EAAC,eAAe,UAAU,MAAM,oBAAmB;EACzD,KAAK,kBAAkB;EACvB,KAAK,sBAAsB;EAE3B,KAAK,MAAM;EACX,KAAK,QAAQ;EACb,KAAK,mBAAmB;EAExB,KAAK,MAAM,GACR,YAAY,YAAY,CAAC,CACzB,YAAY,YAAY,QAAQ,CAAC,CACjC,YAAY,iBAAiB,aAAa,CAAC,CAC3C,YAAY,QAAQ,IAAI;EAC3B,KAAK,oBAAoB,IAAI,iBAC3B,KAAK,KACL,IACA,eACF;EACA,KAAK,IAAI,QAAQ,gBAAgB;EACjC,KAAK,WAAW;EAEhB,KAAK,IAAI,iBAAiB,SAAS,KAAK,YAAY;EACpD,KAAK,IAAI,iBAAiB,SAAS,KAAK,YAAY;EAEpD,KAAK,cAAc;EACnB,KAAK,sBAAsB,YACzB,KAAK,gBACL,6BAA6B,CAC/B;CACF;;;;;;;;CASA,OAAgB;EACd,IACE,KAAK,mBAAA,MACL,KAAK,mBAAA,IAEL,KAAK,gBAAgB;GACnB,MAAMC;GACN,SAAS,wDACP,KAAK,iBACN,QACC,KAAK,mBAAA,KAAsC,WAAW,SACvD;GACD,QAAQC;EACV,CAAC;OACI;GACL,MAAM,mBAAqC,CACzC,aACA;IAAC,MAAM,KAAK;IAAO,WAAW,KAAK,IAAI;GAAC,CAC1C;GACA,KAAK,KAAK,kBAAkB,qBAAqB;GACjD,OAAO;EACT;EACA,OAAO;CACT;CAEA,IAAI,OAAe;EACjB,OAAO,KAAK;CACd;CAEA,MAAM,QAAgB,GAAG,MAAiB;EACxC,IAAI,KAAK,SACP;EAEF,KAAK,UAAU;EACf,KAAK,IAAI,OAAO,uBAAuB,UAAU,GAAG,IAAI;EACxD,KAAK,IAAI,oBAAoB,SAAS,KAAK,YAAY;EACvD,KAAK,IAAI,oBAAoB,SAAS,KAAK,YAAY;EACvD,KAAK,2BAA2B,OAAO;EACvC,KAAK,4BAA4B,KAAA;EACjC,KAAK,uBAAuB,OAAO;EACnC,KAAK,wBAAwB,KAAA;EAC7B,KAAK,SAAS;EACd,IAAI,KAAK,IAAI,eAAe,KAAK,IAAI,QACnC,KAAK,IAAI,MAAM;EAEjB,aAAa,KAAK,mBAAmB;CAIvC;CAEA,qBAAqB,mBAA2B;EAC9C,OAAO,KAAK,eAAe,EAAC,MAAM,kBAAiB,CAAC;CACtD;CAEA,iBAAiB,OAAO,UAAwB;EAC9C,MAAM,OAAO,MAAM,KAAK,SAAS;EACjC,IAAI,KAAK,SAAS;GAChB,KAAK,IAAI,QAAQ,0CAA0C,IAAI;GAC/D;EACF;EAEA,IAAI;EACJ,IAAI;GAEF,MAAM,qBADQ,KAAK,MAAM,IACE,GAAO,KAAK,mBAAmB;EAC5D,SAAS,GAAG;GACV,MAAM,YAAY;IAChB,MAAMC;IACN,SAAS,OAAO,CAAC;IACjB,QAAQD;GACV;GACA,KAAK,gBACH,WACA,IAAI,uBAAuB,WAAW,MAAM,CAC9C;GACA;EACF;EAEA,IAAI,CAAC,uCAAuC,QAAQ,GAAG,GAAG;GAExD,IAAI,CAAC,KAAK,uBAAuB;IAC/B,KAAK,wBAAwB;IAC7B,KAAK,IAAI,OACP,oDAAoD,KAAK,UAAU,IAAI,EAAE,GAC3E;GACF;GACA;EACF;EAEA,IAAI;GAEF,IADgB,IAAI,OACJ,QAAQ;IACtB,KAAK,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,qBAAqB;IAC7C;GACF;GAEA,MAAM,SAAS,MAAM,KAAK,gBAAgB,cAAc,GAAG;GAC3D,KAAK,MAAM,KAAK,QACd,KAAK,qBAAqB,CAAC;EAE/B,SAAS,GAAG;GACV,KAAK,iBAAiB,CAAC;EACzB;CACF;CAEA,qBAAqB,QAA6B;EAChD,QAAQ,OAAO,MAAf;GACE,KAAK;IACH,KAAK,gBAAgB,OAAO,KAAK;IACjC;GACF,KAAK,MACH;GACF,KAAK;IACH,QAAQ,OAAO,QAAf;KACE,KAAK;MACH,OACE,KAAK,8BAA8B,KAAA,GACnC,kDACF;MACA,KAAK,4BAA4B,OAAO;MACxC;KACF,KAAK;MACH,OACE,KAAK,0BAA0B,KAAA,GAC/B,kDACF;MACA,KAAK,wBAAwB,OAAO;IAExC;IACA,IAAI,OAAO,WAAW,cACpB,KAAK,eAAe,OAAO,SAAS,YAAY,aAC9C,KAAK,KAAK,WAAW,SAAS,UAAU,WAAW,UAAU,CAC/D;SAEA,KAAK,eAAe,OAAO,SAAS,YAAY,aAC9C,KAAK,KAAK,YAAY,QAAQ,CAChC;IAEF;GAEF,KAAK,aACH,KAAK,MAAM,SAAS,OAAO,QACzB,KAAK,UAAU,KAAK;EAG1B;CACF;CAEA,gBAAgB,MAAkB;EAChC,MAAM,EAAC,MAAM,QAAQ,aAAY;EACjC,IAAI,CAAC,UACH,KAAK,sBAAsB,eAAe;EAE5C,KAAK,MAAM,yBAAyB;GAAC;GAAM;GAAQ;EAAQ,CAAC;CAC9D;CAEA,gBAAgB,MAAkB;EAChC,KAAK,sBAAsB,aAAa;EACxC,KAAK,IAAI,OAAO,yBAAyB,EAAE,SAAS,EAAE,KAAK;CAC7D;CAEA,sBAAsB,WAAmB;EACvC,KAAK,iBAAiB,IAAI,GAAG;IAC1B,6BAA6B,KAAK;IAClC,uBAAuB;EAC1B,CAAC;CACH;CAEA,gBAAgB;EACd,SACE,sBAAsB,KAAK,GAAG,GAC9B,IAAI,SAAS,EACX,QAAQ,MAAM,WAAW,aAAa;GACpC,KAAK,eAAe,EAAC,KAAI,CAAC,CAAC,CAAC,WAAW,SAAS,GAAG,QAAQ;EAC7D,EACF,CAAC,SAIK,CAAC,CACT;CACF;CAEA,eACE,gBACA,aAIA;EAMA,SACE,SAAS,KAAK,cAAc,GAC5B,IAAI,SAAS;GACX,YAAY;GACZ,QAAQ,YAAe,WAAW,aAChC,YAAY,YAAY,QAAQ;EACpC,CAAC,IACD,MACE,IACI,KAAK,iBAAiB,CAAC,IACvB,KAAK,MAAM,iCAAiC,CACpD;CACF;CAEA,iBAAiB,GAAY;EAC3B,MAAM,YACJ,kBAAkB,CAAC,CAAC,EAAE,aAAa,sBAAsB,CAAC,CAAC,CAAC;EAE9D,KAAK,gBAAgB,WAAW,CAAC;CACnC;CAEA,gBAAgB,WAAsB,QAAkB;EACtD,KAAK,UAAU,WAAW,MAAM;EAChC,KAAK,MACH,GAAG,UAAU,KAAK,IAAI,UAAU,OAAO,KAAK,UAAU,WACtD,SACF;CACF;CAEA,yBAAyB,KAAK,IAAI;CAElC,uBAAuB;EACrB,IAAI,KAAK,IAAI,IAAI,KAAK,yBAAyB,4BAA4B;GACzE,KAAK,IAAI,QAAQ,uBAAuB;GACxC,KAAK,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,qBAAqB;EAC/C;CACF;CAEA,KACE,MACA,UACA,YACA;EACA,KAAK,yBAAyB,KAAK,IAAI;EACvC,KAAK,kBAAkB,KAAK,MAAM,UAAU,UAAU;CACxD;CAEA,UAAU,WAAsB,QAAkB;EAChD,UAAU,KAAK,KAAK,KAAK,KAAK,WAAW,MAAM;CACjD;AACF;AAOA,SAAgB,KACd,IACA,IACA,MACA,UACA,YACA;CACA,IAAI,GAAG,eAAeE,YAAU,MAAM;EACpC,eAAe,KAAK,UAAU,IAAI;EAClC,IAAI,aAAa,uBAAuB;GACtC,GAAG,KAAK,UAAU;GAClB;EACF;EAEA,IAAI,YAAY;EAChB,MAAM,QAAQ,iBAAiB;GAC7B,SAAS,0BAA0B,CAAC;EACtC,GAAG,yBAAyB;EAC5B,MAAM,MAAM;EAEZ,MAAM,YAAY,UAAyB;GACzC,IAAI,WACF;GAEF,YAAY;GACZ,aAAa,KAAK;GAClB,SAAS,KAAK;EAChB;EAEA,GAAG,KAAK,YAAY,QAAQ;CAC9B,OAAO;EACL,GAAG,QAAQ,2CAA2C,GAAG,WAAW,IAAI,EACtE,SAAS,KACX,CAAC;EACD,IAAI,aAAa,uBACf,SACE,IAAI,uBACF;GACE,MAAMC;GACN,SAAS;GACT,QAAQH;EACV,GACA,MACF,CACF;CAEJ;AACF;;;;;;;;AASA,IAAa,mBAAb,MAA8B;CAC5B;CACA;CACA;CACA;CAEA,YAAY,IAAgB,IAAmB,iBAAyB;EACtE,KAAK,MAAM;EACX,KAAK,MAAM;EACX,KAAK,mBAAmB;CAC1B;CAEA,KACE,MACA,UACA,YACM;EACN,IAAI,KAAK,oBAAA,IACP,QAAQ,KAAK,IAAb;GACE,KAAK;IACH,OACE,KAAK,sBAAsB,KAAA,GAC3B,uDACF;IACA,KAAK,oBAAoB,IAAI,iBAAiB;IAC9C;GACF,KAAK;IACH,KAAK,cAAc,MAAM,UAAU,UAAU;IAC7C;GACF,KAAK;IACH,KAAK,aAAa,MAAM,QAAQ;IAChC;EACJ;EAEF,KAAK,KAAK,KAAK,KAAK,KAAK,MAAM,UAAU,UAAU;CACrD;CAEA,cACE,UACA,UACA,YACM;EACN,MAAM,UAAU,KAAK;EACrB,OAAO,SAAS,uCAAuC;EAEvD,eAAe,KAAK,UAAU,QAAQ;EACtC,OACE,WAAW,WAAW,gBAAgB,KAAK,WAAW,SAAS,GAAG,GAClE,6BACF;EACA,QACG,SAAS,WAAW,MAAM,iBAAiB,QAAQ,EAAE,IAAG,UACvD,WAAW,KAAK,KAAK,KAAK,KAAK,KAAK,CACtC,CAAC,CACA,WACO,eAAe,QAAQ,IAC7B,UAAS,eAAe,UAAU,QAAQ,KAAK,CAAC,CAClD;CACJ;CAEA,aACE,SACA,UACM;EACN,MAAM,UAAU,KAAK;EACrB,OAAO,SAAS,sCAAsC;EACtD,KAAK,oBAAoB,KAAA;EAEzB,IAAI,QAAQ,EAAE,CAAC,QAAQ;GACrB,QAAQ,OAAO;GACf,KAAK,KAAK,KAAK,KAAK,KAAK,SAAS,QAAQ;GAC1C;EACF;EAEA,QACG,QAAO,UAAS,WAAW,KAAK,KAAK,KAAK,KAAK,KAAK,CAAC,CAAC,CACtD,WAAW,UAAU,KAAK,KAAK,KAAK,KAAK,OAAO,CAAC,CAAC,CAClD,WACO,eAAe,QAAQ,IAC7B,UAAS,eAAe,UAAU,QAAQ,KAAK,CAAC,CAClD;CACJ;AACF;AAEA,IAAM,mBAAmB;AAEzB,SAAS,UACP,IACA,IACA,MACe;CACf,OAAO,IAAI,SAAS,SAAS,WAAW;EACtC,KAAK,IAAI,IAAI,OAAM,UAAU,QAAQ,OAAO,KAAK,IAAI,QAAQ,CAAE;CACjE,CAAC;AACH;AAEA,SAAS,WACP,IACA,IACA,OACe;CACf,OAAO,IAAI,SAAS,SAAS,WAAW;EACtC,IAAI,GAAG,eAAeE,YAAU,MAAM;GACpC,MAAM,QAAQ,qBAAqB;GACnC,GAAG,QACD,kDAAkD,GAAG,WAAW,EAClE;GACA,OAAO,KAAK;GACZ;EACF;EACA,IAAI,YAAY;EAChB,MAAM,QAAQ,iBACN,SAAS,0BAA0B,CAAC,GAC1C,yBACF;EACA,MAAM,MAAM;EAEZ,MAAM,YAAY,UAAyB;GACzC,IAAI,WACF;GAEF,YAAY;GACZ,aAAa,KAAK;GAClB,QAAQ,OAAO,KAAK,IAAI,QAAQ;EAClC;EAEA,GAAG,KAAK,OAAO,QAAQ;CACzB,CAAC;AACH;AAEA,SAAS,eACP,UACA,OACM;CACN,IAAI,aAAa,uBACf,SAAS,KAAK;AAElB;AAEA,SAAS,QAAQ,OAAuB;CACtC,OAAO,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;AACjE;AAEA,SAAS,uBAA+C;CACtD,OAAO,IAAI,uBACT;EACE,MAAMC;EACN,SAAS;EACT,QAAQH;CACV,GACA,MACF;AACF;AAEA,SAAS,4BAAoD;CAC3D,OAAO,IAAI,uBACT;EACE,MAAMG;EACN,SAAS,kCAAkC,0BAA0B;EACrE,QAAQH;CACV,GACA,MACF;AACF;AAEA,SAAgB,UACd,IACA,IACA,WACA,QACA;CACA,KAAK,GAAG,YAAY,aAAa,UAAU,IAAI;CAE/C,IAAI;CAGJ,IAAI,kBAAkB,wBACpB,WAAW,OAAO;MAIf,IACH,SAAS,MAAM,KACf,uBAAuB,MAAM,KAC7B,yBAAyB,UAAU,OAAO,GAE1C,WAAW;MAGR,IACH,UAAU,SAAS,oBACnB,UAAU,SAAS,mBAEnB,WAAW;MAEX,WAAW,SAAS,YAAY,MAAM,IAAI;CAG5C,GAAG,SAAS,GAAG,8BAA8B,WAAW,UAAU,EAAE;CACpE,KAAK,IAAI,IAAI,CAAC,SAAS,SAAS,GAAG,qBAAqB;AAC1D;AAEA,SAAS,kBAAkB,OAA2C;CACpE,IAAI,gBAAgB,KAAK,GACvB,OAAO;CAET,IAAI,iBAAiB,SAAS,MAAM,OAClC,OAAO,kBAAkB,MAAM,KAAK;AAGxC;AAEA,SAAS,SAAS,OAAyB;CACzC,OAAO,QACL,SACA,OAAO,UAAU,YACjB,WAAW,SACX,OAAQ,MAA2B,UAAU,WAC/C;AACF;AAIA,IAAM,+CAA+B,IAAI,IAAI;CAC3C;CACA;CACA;AACF,CAAC;AAID,IAAM,oCAAoC,CACxC,mDACF;AAEA,SAAS,uBAAuB,OAAyB;CACvD,IAAI,CAAC,SAAS,OAAO,UAAU,UAC7B,OAAO;CAET,MAAM,YACJ,UAAU,QAAQ,OAAQ,MAA2B,IAAI,IAAI,KAAA;CAC/D,OAAO,QACL,aAAa,6BAA6B,IAAI,UAAU,YAAY,CAAC,CACvE;AACF;AAEA,SAAS,yBAAyB,SAAsC;CACtE,IAAI,CAAC,SACH,OAAO;CAET,MAAM,QAAQ,QAAQ,YAAY;CAClC,OAAO,kCAAkC,MAAK,YAC5C,MAAM,SAAS,OAAO,CACxB;AACF"}
+{"version":3,"file":"connection.js","names":["valita.parse","ErrorKind.VersionNotSupported","ErrorOrigin.ZeroCache","ErrorKind.InvalidMessage","WebSocket","ErrorKind.Internal"],"sources":["../../../../../zero-cache/src/workers/connection.ts"],"sourcesContent":["import {pipeline, Readable, Writable} from 'node:stream';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport type {CloseEvent, Data, ErrorEvent} from 'ws';\nimport WebSocket, {createWebSocketStream} from 'ws';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport * as valita from '../../../shared/src/valita.ts';\nimport type {ConnectedMessage} from '../../../zero-protocol/src/connect.ts';\nimport type {Downstream} from '../../../zero-protocol/src/down.ts';\nimport {ErrorKind} from '../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../zero-protocol/src/error-origin.ts';\nimport type {ErrorBody} from '../../../zero-protocol/src/error.ts';\nimport {\n  isProtocolError,\n  type ProtocolError,\n} from '../../../zero-protocol/src/error.ts';\nimport {\n  POKE_CHUNK_PROTOCOL_VERSION,\n  type PokeChunk,\n  type PokeEndMessage,\n  type PokePartMessage,\n} from '../../../zero-protocol/src/poke.ts';\nimport {\n  MIN_SERVER_SUPPORTED_SYNC_PROTOCOL,\n  PROTOCOL_VERSION,\n} from '../../../zero-protocol/src/protocol-version.ts';\nimport {\n  upstreamSchemaWithUnparsedAnalyzeQuery,\n  type UpstreamWithUnparsedAnalyzeQuery,\n} from '../../../zero-protocol/src/up.ts';\nimport {getOrCreateCounter} from '../observability/metrics.ts';\nimport {clientGroupLogID} from '../types/client-group-log-id.ts';\nimport type {ViewSyncerDownstream} from '../types/downstream.ts';\nimport {\n  ProtocolErrorWithLevel,\n  getLogLevel,\n  wrapWithProtocolError,\n} from '../types/error-with-level.ts';\nimport {PokeChunkEncoder} from '../types/poke-chunk.ts';\nimport type {Source} from '../types/streams.ts';\nimport type {ConnectParams} from './connect-params.ts';\n\nexport type HandlerResult =\n  | {\n      type: 'ok';\n    }\n  | {\n      type: 'fatal';\n      error: ErrorBody;\n    }\n  | {\n      type: 'transient';\n      errors: ErrorBody[];\n    }\n  | StreamResult;\n\nexport type StreamResult =\n  | {\n      type: 'stream';\n      source: 'viewSyncer';\n      stream: Source<ViewSyncerDownstream>;\n    }\n  | {\n      type: 'stream';\n      source: 'pusher';\n      stream: Source<Downstream>;\n    };\n\nexport interface MessageHandler {\n  handleMessage(\n    msg: UpstreamWithUnparsedAnalyzeQuery,\n  ): Promise<HandlerResult[]>;\n}\n\nfunction hasOwn(value: unknown, property: string): boolean {\n  return (\n    typeof value === 'object' &&\n    value !== null &&\n    Object.hasOwn(value, property)\n  );\n}\n\nfunction containsLegacyQuery(message: unknown): boolean {\n  if (!Array.isArray(message)) {\n    return false;\n  }\n\n  const body = message[1];\n  if (typeof body !== 'object' || body === null) {\n    return false;\n  }\n\n  if (\n    message[0] !== 'initConnection' &&\n    message[0] !== 'changeDesiredQueries'\n  ) {\n    return false;\n  }\n\n  const patch = body['desiredQueriesPatch'];\n  return (\n    Array.isArray(patch) && patch.some(operation => hasOwn(operation, 'ast'))\n  );\n}\n\nconst LEGACY_QUERIES_DISABLED_MESSAGE =\n  'Legacy queries are disabled by this Zero server. This client must use custom queries instead of sending query ASTs directly. To temporarily restore compatibility, set ZERO_ALLOW_LEGACY_QUERIES=true on zero-cache.';\n\n// Exported for testing purposes.\n//\n// Client messages are parsed leniently, so a newer client can add message\n// kinds and fields:\n// - A message of a kind this server doesn't know parses as an\n//   UnrecognizedTuple, for the connection to drop. Malformed messages still\n//   throw.\n// - Messages are parsed in strip mode, so fields this server doesn't know are\n//   dropped. Strip rather than passthrough: nothing this server doesn't know\n//   reaches the CVR or the API server, and valita's passthrough drops unknown\n//   fields anyway on any object with a transformed field below it (such as\n//   jsonSchema args). ASTs stay strict in the schema, because a server that\n//   ignored an AST field would run a different query.\nexport function parseUpstreamMessage(\n  value: unknown,\n  allowLegacyQueries: boolean,\n): UpstreamWithUnparsedAnalyzeQuery | valita.UnrecognizedTuple {\n  if (!allowLegacyQueries && containsLegacyQuery(value)) {\n    throw new Error(LEGACY_QUERIES_DISABLED_MESSAGE);\n  }\n  return valita.parse(value, upstreamSchemaWithUnparsedAnalyzeQuery, 'strip');\n}\n\n// Ensures that a downstream message is sent at least every interval, sending a\n// 'pong' if necessary. This is set to be slightly longer than the client-side\n// PING_INTERVAL of 5 seconds, so that in the common case, 'pong's are sent in\n// response to client-initiated 'ping's. However, if the inbound stream is\n// backed up because a command is taking a long time to process, the pings\n// will be stuck in the queue (i.e. back-pressured), in which case pongs will\n// be manually sent to notify the client of server liveness.\n//\n// This is equivalent to what is done for Postgres keepalives on the\n// replication stream (which can similarly be back-pressured):\n// https://github.com/rocicorp/mono/blob/f98cb369a2dbb15650328859c732db358f187ef0/packages/zero-cache/src/services/change-source/pg/logical-replication/stream.ts#L21\nconst DOWNSTREAM_MSG_INTERVAL_MS = 6_000;\nexport const WEBSOCKET_SEND_TIMEOUT_MS = 10_000;\nconst PROTOCOL_VERSION_ATTRIBUTE = 'protocol.version';\nconst EVENT_TYPE_ATTRIBUTE = 'event.type';\n\n/**\n * Represents a connection between the client and server.\n *\n * Handles incoming messages on the connection and dispatches\n * them to the correct service.\n *\n * Listens to the ViewSyncer and sends messages to the client.\n */\nexport class Connection {\n  readonly #ws: WebSocket;\n  readonly #wsID: string;\n  readonly #protocolVersion: number;\n  readonly #lc: LogContext;\n  readonly #onClose: () => void;\n  readonly #messageHandler: MessageHandler;\n  readonly #allowLegacyQueries: boolean;\n  readonly #downstreamSender: DownstreamSender;\n  readonly #downstreamMsgTimer: NodeJS.Timeout | undefined;\n  readonly #webSocketErrors = getOrCreateCounter(\n    'sync',\n    'websocket.errors',\n    'Client WebSocket error events.',\n  );\n\n  #viewSyncerOutboundStream: Source<ViewSyncerDownstream> | undefined;\n  #pusherOutboundStream: Source<Downstream> | undefined;\n  #closed = false;\n  #loggedUnknownMessage = false;\n\n  constructor(\n    lc: LogContext,\n    connectParams: ConnectParams,\n    ws: WebSocket,\n    allowLegacyQueries: boolean,\n    messageHandler: MessageHandler,\n    onClose: () => void,\n  ) {\n    const {clientGroupID, clientID, wsID, protocolVersion} = connectParams;\n    this.#messageHandler = messageHandler;\n    this.#allowLegacyQueries = allowLegacyQueries;\n\n    this.#ws = ws;\n    this.#wsID = wsID;\n    this.#protocolVersion = protocolVersion;\n\n    this.#lc = lc\n      .withContext('connection')\n      .withContext('clientID', clientID)\n      .withContext('clientGroupID', clientGroupLogID(clientGroupID))\n      .withContext('wsID', wsID);\n    this.#downstreamSender = new DownstreamSender(\n      this.#lc,\n      ws,\n      protocolVersion,\n    );\n    this.#lc.debug?.('new connection');\n    this.#onClose = onClose;\n\n    this.#ws.addEventListener('close', this.#handleClose);\n    this.#ws.addEventListener('error', this.#handleError);\n\n    this.#proxyInbound();\n    this.#downstreamMsgTimer = setInterval(\n      this.#maybeSendPong,\n      DOWNSTREAM_MSG_INTERVAL_MS / 2,\n    );\n  }\n\n  /**\n   * Checks the protocol version and errors for unsupported protocols,\n   * sending the initial `connected` response on success.\n   *\n   * This is early in the connection lifecycle because {@link #handleMessage}\n   * will only parse messages with schema(s) of supported protocol versions.\n   */\n  init(): boolean {\n    if (\n      this.#protocolVersion > PROTOCOL_VERSION ||\n      this.#protocolVersion < MIN_SERVER_SUPPORTED_SYNC_PROTOCOL\n    ) {\n      this.#closeWithError({\n        kind: ErrorKind.VersionNotSupported,\n        message: `server is at sync protocol v${PROTOCOL_VERSION} and does not support v${\n          this.#protocolVersion\n        }. The ${\n          this.#protocolVersion > PROTOCOL_VERSION ? 'server' : 'client'\n        } must be updated to a newer release.`,\n        origin: ErrorOrigin.ZeroCache,\n      });\n    } else {\n      const connectedMessage: ConnectedMessage = [\n        'connected',\n        {wsid: this.#wsID, timestamp: Date.now()},\n      ];\n      this.send(connectedMessage, 'ignore-backpressure');\n      return true;\n    }\n    return false;\n  }\n\n  get wsID(): string {\n    return this.#wsID;\n  }\n\n  close(reason: string, ...args: unknown[]) {\n    if (this.#closed) {\n      return;\n    }\n    this.#closed = true;\n    this.#lc.info?.(`closing connection: ${reason}`, ...args);\n    this.#ws.removeEventListener('close', this.#handleClose);\n    this.#ws.removeEventListener('error', this.#handleError);\n    this.#viewSyncerOutboundStream?.cancel();\n    this.#viewSyncerOutboundStream = undefined;\n    this.#pusherOutboundStream?.cancel();\n    this.#pusherOutboundStream = undefined;\n    this.#onClose();\n    if (this.#ws.readyState !== this.#ws.CLOSED) {\n      this.#ws.close();\n    }\n    clearTimeout(this.#downstreamMsgTimer);\n\n    // spin down services if we have\n    // no more client connections for the client group?\n  }\n\n  handleInitConnection(initConnectionMsg: string) {\n    return this.#handleMessage({data: initConnectionMsg});\n  }\n\n  #handleMessage = async (event: {data: Data}) => {\n    const data = event.data.toString();\n    if (this.#closed) {\n      this.#lc.debug?.('Ignoring message received after closed', data);\n      return;\n    }\n\n    let msg;\n    try {\n      const value = JSON.parse(data);\n      msg = parseUpstreamMessage(value, this.#allowLegacyQueries);\n    } catch (e) {\n      const errorBody = {\n        kind: ErrorKind.InvalidMessage,\n        message: String(e),\n        origin: ErrorOrigin.ZeroCache,\n      } as const;\n      this.#closeWithError(\n        errorBody,\n        new ProtocolErrorWithLevel(errorBody, 'warn'),\n      );\n      return;\n    }\n\n    if (!upstreamSchemaWithUnparsedAnalyzeQuery.isKnown(msg)) {\n      // A message kind from a newer client.\n      if (!this.#loggedUnknownMessage) {\n        this.#loggedUnknownMessage = true;\n        this.#lc.info?.(\n          `Ignoring messages of unknown type, starting with ${JSON.stringify(msg[0])}`,\n        );\n      }\n      return;\n    }\n\n    try {\n      const msgType = msg[0];\n      if (msgType === 'ping') {\n        this.send(['pong', {}], 'ignore-backpressure');\n        return;\n      }\n\n      const result = await this.#messageHandler.handleMessage(msg);\n      for (const r of result) {\n        this.#handleMessageResult(r);\n      }\n    } catch (e) {\n      this.#closeWithThrown(e);\n    }\n  };\n\n  #handleMessageResult(result: HandlerResult): void {\n    switch (result.type) {\n      case 'fatal':\n        this.#closeWithError(result.error);\n        break;\n      case 'ok':\n        break;\n      case 'stream': {\n        switch (result.source) {\n          case 'viewSyncer':\n            assert(\n              this.#viewSyncerOutboundStream === undefined,\n              'Outbound stream already set for this connection!',\n            );\n            this.#viewSyncerOutboundStream = result.stream;\n            break;\n          case 'pusher':\n            assert(\n              this.#pusherOutboundStream === undefined,\n              'Outbound stream already set for this connection!',\n            );\n            this.#pusherOutboundStream = result.stream;\n            break;\n        }\n        if (result.source === 'viewSyncer') {\n          this.#proxyOutbound(result.stream, (downstream, callback) =>\n            this.send(downstream.message, callback, downstream.serialized),\n          );\n        } else {\n          this.#proxyOutbound(result.stream, (downstream, callback) =>\n            this.send(downstream, callback),\n          );\n        }\n        break;\n      }\n      case 'transient': {\n        for (const error of result.errors) {\n          this.sendError(error);\n        }\n      }\n    }\n  }\n\n  #handleClose = (e: CloseEvent) => {\n    const {code, reason, wasClean} = e;\n    if (!wasClean) {\n      this.#recordWebSocketError('unclean_close');\n    }\n    this.close('WebSocket close event', {code, reason, wasClean});\n  };\n\n  #handleError = (e: ErrorEvent) => {\n    this.#recordWebSocketError('error_event');\n    this.#lc.warn?.('WebSocket error event', e.message, e.error);\n  };\n\n  #recordWebSocketError(eventType: string) {\n    this.#webSocketErrors.add(1, {\n      [PROTOCOL_VERSION_ATTRIBUTE]: this.#protocolVersion,\n      [EVENT_TYPE_ATTRIBUTE]: eventType,\n    });\n  }\n\n  #proxyInbound() {\n    pipeline(\n      createWebSocketStream(this.#ws),\n      new Writable({\n        write: (data, _encoding, callback) => {\n          this.#handleMessage({data}).then(() => callback(), callback);\n        },\n      }),\n      // The done callback is not used, as #handleClose and #handleError,\n      // configured on the underlying WebSocket, provide more complete\n      // information.\n      () => {},\n    );\n  }\n\n  #proxyOutbound<T>(\n    outboundStream: Source<T>,\n    sendMessage: (\n      downstream: T,\n      callback: (err?: Error | null) => void,\n    ) => void,\n  ) {\n    // Note: createWebSocketStream() is avoided here in order to control\n    //       exception handling with #closeWithThrown(). If the Writable\n    //       from createWebSocketStream() were instead used, exceptions\n    //       from the outboundStream result in the Writable closing the\n    //       the websocket before the error message can be sent.\n    pipeline(\n      Readable.from(outboundStream),\n      new Writable({\n        objectMode: true,\n        write: (downstream: T, _encoding, callback) =>\n          sendMessage(downstream, callback),\n      }),\n      e =>\n        e\n          ? this.#closeWithThrown(e)\n          : this.close(`downstream closed by ViewSyncer`),\n    );\n  }\n\n  #closeWithThrown(e: unknown) {\n    const errorBody =\n      findProtocolError(e)?.errorBody ?? wrapWithProtocolError(e).errorBody;\n\n    this.#closeWithError(errorBody, e);\n  }\n\n  #closeWithError(errorBody: ErrorBody, thrown?: unknown) {\n    this.sendError(errorBody, thrown);\n    this.close(\n      `${errorBody.kind} (${errorBody.origin}): ${errorBody.message}`,\n      errorBody,\n    );\n  }\n\n  #lastDownstreamMsgTime = Date.now();\n\n  #maybeSendPong = () => {\n    if (Date.now() - this.#lastDownstreamMsgTime > DOWNSTREAM_MSG_INTERVAL_MS) {\n      this.#lc.debug?.('manually sending pong');\n      this.send(['pong', {}], 'ignore-backpressure');\n    }\n  };\n\n  send(\n    data: Downstream,\n    callback: ((err?: Error | null) => void) | 'ignore-backpressure',\n    serialized?: string | undefined,\n  ) {\n    this.#lastDownstreamMsgTime = Date.now();\n    this.#downstreamSender.send(data, callback, serialized);\n  }\n\n  sendError(errorBody: ErrorBody, thrown?: unknown) {\n    sendError(this.#lc, this.#ws, errorBody, thrown);\n  }\n}\n\nexport type WebSocketLike = Pick<WebSocket, 'readyState'> & {\n  send(data: string | PokeChunk, cb?: (err?: Error) => void): void;\n};\n\n// Exported for testing purposes.\nexport function send(\n  lc: LogContext,\n  ws: WebSocketLike,\n  data: Downstream,\n  callback: ((err?: Error | null) => void) | 'ignore-backpressure',\n  serialized?: string | undefined,\n) {\n  if (ws.readyState === WebSocket.OPEN) {\n    serialized ??= JSON.stringify(data);\n    if (callback === 'ignore-backpressure') {\n      ws.send(serialized);\n      return;\n    }\n\n    let completed = false;\n    const timer = setTimeout(() => {\n      complete(webSocketSendTimeoutError());\n    }, WEBSOCKET_SEND_TIMEOUT_MS);\n    timer.unref();\n\n    const complete = (error?: Error | null) => {\n      if (completed) {\n        return;\n      }\n      completed = true;\n      clearTimeout(timer);\n      callback(error);\n    };\n\n    ws.send(serialized, complete);\n  } else {\n    lc.debug?.(`Dropping outbound message on ws (state: ${ws.readyState})`, {\n      dropped: data,\n    });\n    if (callback !== 'ignore-backpressure') {\n      callback(\n        new ProtocolErrorWithLevel(\n          {\n            kind: ErrorKind.Internal,\n            message: 'WebSocket closed',\n            origin: ErrorOrigin.ZeroCache,\n          },\n          'info',\n        ),\n      );\n    }\n  }\n}\n\n/**\n * Sends downstream messages in the format supported by a connection's sync\n * protocol version. Keep version forks contained here so the view-syncer can\n * continue producing one canonical poke representation.\n *\n * Exported for compatibility testing.\n */\nexport class DownstreamSender {\n  readonly #lc: LogContext;\n  readonly #ws: WebSocketLike;\n  readonly #protocolVersion: number;\n  #pokeChunkEncoder: PokeChunkEncoder | undefined;\n  /** The last finished poke's buffer, so the next poke allocates nothing. */\n  #idlePokeChunkBuffer: Uint8Array | undefined;\n\n  constructor(lc: LogContext, ws: WebSocketLike, protocolVersion: number) {\n    this.#lc = lc;\n    this.#ws = ws;\n    this.#protocolVersion = protocolVersion;\n  }\n\n  send(\n    data: Downstream,\n    callback: ((err?: Error | null) => void) | 'ignore-backpressure',\n    serialized?: string | undefined,\n  ): void {\n    if (this.#protocolVersion >= POKE_CHUNK_PROTOCOL_VERSION) {\n      switch (data[0]) {\n        case 'pokeStart':\n          assert(\n            this.#pokeChunkEncoder === undefined,\n            'cannot start a poke while another poke is in progress',\n          );\n          this.#pokeChunkEncoder = new PokeChunkEncoder(\n            this.#idlePokeChunkBuffer,\n          );\n          this.#idlePokeChunkBuffer = undefined;\n          break;\n        case 'pokePart':\n          this.#sendPokePart(data, callback, serialized);\n          return;\n        case 'pokeEnd':\n          this.#sendPokeEnd(data, callback);\n          return;\n      }\n    }\n    send(this.#lc, this.#ws, data, callback, serialized);\n  }\n\n  #sendPokePart(\n    pokePart: PokePartMessage,\n    callback: ((err?: Error | null) => void) | 'ignore-backpressure',\n    serialized?: string | undefined,\n  ): void {\n    const encoder = this.#pokeChunkEncoder;\n    assert(encoder, 'pokePart received without a pokeStart');\n\n    serialized ??= JSON.stringify(pokePart);\n    assert(\n      serialized.startsWith(POKE_PART_PREFIX) && serialized.endsWith(']'),\n      'invalid serialized pokePart',\n    );\n    void encoder\n      .addPatch(serialized.slice(POKE_PART_PREFIX.length, -1), chunk =>\n        sendBinary(this.#lc, this.#ws, chunk),\n      )\n      .then(\n        () => invokeCallback(callback),\n        error => invokeCallback(callback, toError(error)),\n      );\n  }\n\n  #sendPokeEnd(\n    pokeEnd: PokeEndMessage,\n    callback: ((err?: Error | null) => void) | 'ignore-backpressure',\n  ): void {\n    const encoder = this.#pokeChunkEncoder;\n    assert(encoder, 'pokeEnd received without a pokeStart');\n    this.#pokeChunkEncoder = undefined;\n\n    if (pokeEnd[1].cancel) {\n      encoder.cancel();\n      send(this.#lc, this.#ws, pokeEnd, callback);\n      return;\n    }\n\n    void encoder\n      .finish(chunk => sendBinary(this.#lc, this.#ws, chunk))\n      .then(buffer => {\n        this.#idlePokeChunkBuffer = buffer;\n        return sendAsync(this.#lc, this.#ws, pokeEnd);\n      })\n      .then(\n        () => invokeCallback(callback),\n        error => invokeCallback(callback, toError(error)),\n      );\n  }\n}\n\nconst POKE_PART_PREFIX = '[\"pokePart\",';\n\nfunction sendAsync(\n  lc: LogContext,\n  ws: WebSocketLike,\n  data: Downstream,\n): Promise<void> {\n  return new Promise((resolve, reject) => {\n    send(lc, ws, data, error => (error ? reject(error) : resolve()));\n  });\n}\n\nfunction sendBinary(\n  lc: LogContext,\n  ws: WebSocketLike,\n  chunk: PokeChunk,\n): Promise<void> {\n  return new Promise((resolve, reject) => {\n    if (ws.readyState !== WebSocket.OPEN) {\n      const error = webSocketClosedError();\n      lc.debug?.(\n        `Dropping outbound binary message on ws (state: ${ws.readyState})`,\n      );\n      reject(error);\n      return;\n    }\n    let completed = false;\n    const timer = setTimeout(\n      () => complete(webSocketSendTimeoutError()),\n      WEBSOCKET_SEND_TIMEOUT_MS,\n    );\n    timer.unref();\n\n    const complete = (error?: Error | null) => {\n      if (completed) {\n        return;\n      }\n      completed = true;\n      clearTimeout(timer);\n      error ? reject(error) : resolve();\n    };\n\n    ws.send(chunk, complete);\n  });\n}\n\nfunction invokeCallback(\n  callback: ((err?: Error | null) => void) | 'ignore-backpressure',\n  error?: Error,\n): void {\n  if (callback !== 'ignore-backpressure') {\n    callback(error);\n  }\n}\n\nfunction toError(error: unknown): Error {\n  return error instanceof Error ? error : new Error(String(error));\n}\n\nfunction webSocketClosedError(): ProtocolErrorWithLevel {\n  return new ProtocolErrorWithLevel(\n    {\n      kind: ErrorKind.Internal,\n      message: 'WebSocket closed',\n      origin: ErrorOrigin.ZeroCache,\n    },\n    'info',\n  );\n}\n\nfunction webSocketSendTimeoutError(): ProtocolErrorWithLevel {\n  return new ProtocolErrorWithLevel(\n    {\n      kind: ErrorKind.Internal,\n      message: `WebSocket send timed out after ${WEBSOCKET_SEND_TIMEOUT_MS} ms`,\n      origin: ErrorOrigin.ZeroCache,\n    },\n    'info',\n  );\n}\n\nexport function sendError(\n  lc: LogContext,\n  ws: WebSocket,\n  errorBody: ErrorBody,\n  thrown?: unknown,\n) {\n  lc = lc.withContext('errorKind', errorBody.kind);\n\n  let logLevel: LogLevel;\n\n  // If the thrown error is a ProtocolErrorWithLevel, its explicit logLevel takes precedence\n  if (thrown instanceof ProtocolErrorWithLevel) {\n    logLevel = thrown.logLevel;\n  }\n  // Errors with errno or transient socket codes are low-level, transient I/O issues\n  // (e.g., EPIPE, ECONNRESET) and should be warnings, not errors\n  else if (\n    hasErrno(thrown) ||\n    hasTransientSocketCode(thrown) ||\n    isTransientSocketMessage(errorBody.message)\n  ) {\n    logLevel = 'warn';\n  }\n  // Fallback: check errorBody.kind for errors that weren't thrown as ProtocolErrorWithLevel\n  else if (\n    errorBody.kind === ErrorKind.ClientNotFound ||\n    errorBody.kind === ErrorKind.TransformFailed\n  ) {\n    logLevel = 'warn';\n  } else {\n    logLevel = thrown ? getLogLevel(thrown) : 'info';\n  }\n\n  lc[logLevel]?.('Sending error on WebSocket', errorBody, thrown ?? '');\n  send(lc, ws, ['error', errorBody], 'ignore-backpressure');\n}\n\nfunction findProtocolError(error: unknown): ProtocolError | undefined {\n  if (isProtocolError(error)) {\n    return error;\n  }\n  if (error instanceof Error && error.cause) {\n    return findProtocolError(error.cause);\n  }\n  return undefined;\n}\n\nfunction hasErrno(error: unknown): boolean {\n  return Boolean(\n    error &&\n    typeof error === 'object' &&\n    'errno' in error &&\n    typeof (error as {errno: unknown}).errno !== 'undefined',\n  );\n}\n\n// System error codes that indicate transient socket conditions.\n// These are checked via the `code` property on errors.\nconst TRANSIENT_SOCKET_ERROR_CODES = new Set([\n  'EPIPE',\n  'ECONNRESET',\n  'ECANCELED',\n]);\n\n// Error messages that indicate transient socket conditions but don't have\n// standard error codes (e.g., WebSocket library errors).\nconst TRANSIENT_SOCKET_MESSAGE_PATTERNS = [\n  'socket was closed while data was being compressed',\n];\n\nfunction hasTransientSocketCode(error: unknown): boolean {\n  if (!error || typeof error !== 'object') {\n    return false;\n  }\n  const maybeCode =\n    'code' in error ? String((error as {code?: unknown}).code) : undefined;\n  return Boolean(\n    maybeCode && TRANSIENT_SOCKET_ERROR_CODES.has(maybeCode.toUpperCase()),\n  );\n}\n\nfunction isTransientSocketMessage(message: string | undefined): boolean {\n  if (!message) {\n    return false;\n  }\n  const lower = message.toLowerCase();\n  return TRANSIENT_SOCKET_MESSAGE_PATTERNS.some(pattern =>\n    lower.includes(pattern),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;AAyEA,SAAS,OAAO,OAAgB,UAA2B;CACzD,OACE,OAAO,UAAU,YACjB,UAAU,QACV,OAAO,OAAO,OAAO,QAAQ;AAEjC;AAEA,SAAS,oBAAoB,SAA2B;CACtD,IAAI,CAAC,MAAM,QAAQ,OAAO,GACxB,OAAO;CAGT,MAAM,OAAO,QAAQ;CACrB,IAAI,OAAO,SAAS,YAAY,SAAS,MACvC,OAAO;CAGT,IACE,QAAQ,OAAO,oBACf,QAAQ,OAAO,wBAEf,OAAO;CAGT,MAAM,QAAQ,KAAK;CACnB,OACE,MAAM,QAAQ,KAAK,KAAK,MAAM,MAAK,cAAa,OAAO,WAAW,KAAK,CAAC;AAE5E;AAEA,IAAM,kCACJ;AAeF,SAAgB,qBACd,OACA,oBAC6D;CAC7D,IAAI,CAAC,sBAAsB,oBAAoB,KAAK,GAClD,MAAM,IAAI,MAAM,+BAA+B;CAEjD,OAAOA,MAAa,OAAO,wCAAwC,OAAO;AAC5E;AAaA,IAAM,6BAA6B;AACnC,IAAa,4BAA4B;AACzC,IAAM,6BAA6B;AACnC,IAAM,uBAAuB;;;;;;;;;AAU7B,IAAa,aAAb,MAAwB;CACtB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA,mBAA4B,mBAC1B,QACA,oBACA,gCACF;CAEA;CACA;CACA,UAAU;CACV,wBAAwB;CAExB,YACE,IACA,eACA,IACA,oBACA,gBACA,SACA;EACA,MAAM,EAAC,eAAe,UAAU,MAAM,oBAAmB;EACzD,KAAK,kBAAkB;EACvB,KAAK,sBAAsB;EAE3B,KAAK,MAAM;EACX,KAAK,QAAQ;EACb,KAAK,mBAAmB;EAExB,KAAK,MAAM,GACR,YAAY,YAAY,CAAC,CACzB,YAAY,YAAY,QAAQ,CAAC,CACjC,YAAY,iBAAiB,iBAAiB,aAAa,CAAC,CAAC,CAC7D,YAAY,QAAQ,IAAI;EAC3B,KAAK,oBAAoB,IAAI,iBAC3B,KAAK,KACL,IACA,eACF;EACA,KAAK,IAAI,QAAQ,gBAAgB;EACjC,KAAK,WAAW;EAEhB,KAAK,IAAI,iBAAiB,SAAS,KAAK,YAAY;EACpD,KAAK,IAAI,iBAAiB,SAAS,KAAK,YAAY;EAEpD,KAAK,cAAc;EACnB,KAAK,sBAAsB,YACzB,KAAK,gBACL,6BAA6B,CAC/B;CACF;;;;;;;;CASA,OAAgB;EACd,IACE,KAAK,mBAAA,MACL,KAAK,mBAAA,IAEL,KAAK,gBAAgB;GACnB,MAAMC;GACN,SAAS,wDACP,KAAK,iBACN,QACC,KAAK,mBAAA,KAAsC,WAAW,SACvD;GACD,QAAQC;EACV,CAAC;OACI;GACL,MAAM,mBAAqC,CACzC,aACA;IAAC,MAAM,KAAK;IAAO,WAAW,KAAK,IAAI;GAAC,CAC1C;GACA,KAAK,KAAK,kBAAkB,qBAAqB;GACjD,OAAO;EACT;EACA,OAAO;CACT;CAEA,IAAI,OAAe;EACjB,OAAO,KAAK;CACd;CAEA,MAAM,QAAgB,GAAG,MAAiB;EACxC,IAAI,KAAK,SACP;EAEF,KAAK,UAAU;EACf,KAAK,IAAI,OAAO,uBAAuB,UAAU,GAAG,IAAI;EACxD,KAAK,IAAI,oBAAoB,SAAS,KAAK,YAAY;EACvD,KAAK,IAAI,oBAAoB,SAAS,KAAK,YAAY;EACvD,KAAK,2BAA2B,OAAO;EACvC,KAAK,4BAA4B,KAAA;EACjC,KAAK,uBAAuB,OAAO;EACnC,KAAK,wBAAwB,KAAA;EAC7B,KAAK,SAAS;EACd,IAAI,KAAK,IAAI,eAAe,KAAK,IAAI,QACnC,KAAK,IAAI,MAAM;EAEjB,aAAa,KAAK,mBAAmB;CAIvC;CAEA,qBAAqB,mBAA2B;EAC9C,OAAO,KAAK,eAAe,EAAC,MAAM,kBAAiB,CAAC;CACtD;CAEA,iBAAiB,OAAO,UAAwB;EAC9C,MAAM,OAAO,MAAM,KAAK,SAAS;EACjC,IAAI,KAAK,SAAS;GAChB,KAAK,IAAI,QAAQ,0CAA0C,IAAI;GAC/D;EACF;EAEA,IAAI;EACJ,IAAI;GAEF,MAAM,qBADQ,KAAK,MAAM,IACE,GAAO,KAAK,mBAAmB;EAC5D,SAAS,GAAG;GACV,MAAM,YAAY;IAChB,MAAMC;IACN,SAAS,OAAO,CAAC;IACjB,QAAQD;GACV;GACA,KAAK,gBACH,WACA,IAAI,uBAAuB,WAAW,MAAM,CAC9C;GACA;EACF;EAEA,IAAI,CAAC,uCAAuC,QAAQ,GAAG,GAAG;GAExD,IAAI,CAAC,KAAK,uBAAuB;IAC/B,KAAK,wBAAwB;IAC7B,KAAK,IAAI,OACP,oDAAoD,KAAK,UAAU,IAAI,EAAE,GAC3E;GACF;GACA;EACF;EAEA,IAAI;GAEF,IADgB,IAAI,OACJ,QAAQ;IACtB,KAAK,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,qBAAqB;IAC7C;GACF;GAEA,MAAM,SAAS,MAAM,KAAK,gBAAgB,cAAc,GAAG;GAC3D,KAAK,MAAM,KAAK,QACd,KAAK,qBAAqB,CAAC;EAE/B,SAAS,GAAG;GACV,KAAK,iBAAiB,CAAC;EACzB;CACF;CAEA,qBAAqB,QAA6B;EAChD,QAAQ,OAAO,MAAf;GACE,KAAK;IACH,KAAK,gBAAgB,OAAO,KAAK;IACjC;GACF,KAAK,MACH;GACF,KAAK;IACH,QAAQ,OAAO,QAAf;KACE,KAAK;MACH,OACE,KAAK,8BAA8B,KAAA,GACnC,kDACF;MACA,KAAK,4BAA4B,OAAO;MACxC;KACF,KAAK;MACH,OACE,KAAK,0BAA0B,KAAA,GAC/B,kDACF;MACA,KAAK,wBAAwB,OAAO;IAExC;IACA,IAAI,OAAO,WAAW,cACpB,KAAK,eAAe,OAAO,SAAS,YAAY,aAC9C,KAAK,KAAK,WAAW,SAAS,UAAU,WAAW,UAAU,CAC/D;SAEA,KAAK,eAAe,OAAO,SAAS,YAAY,aAC9C,KAAK,KAAK,YAAY,QAAQ,CAChC;IAEF;GAEF,KAAK,aACH,KAAK,MAAM,SAAS,OAAO,QACzB,KAAK,UAAU,KAAK;EAG1B;CACF;CAEA,gBAAgB,MAAkB;EAChC,MAAM,EAAC,MAAM,QAAQ,aAAY;EACjC,IAAI,CAAC,UACH,KAAK,sBAAsB,eAAe;EAE5C,KAAK,MAAM,yBAAyB;GAAC;GAAM;GAAQ;EAAQ,CAAC;CAC9D;CAEA,gBAAgB,MAAkB;EAChC,KAAK,sBAAsB,aAAa;EACxC,KAAK,IAAI,OAAO,yBAAyB,EAAE,SAAS,EAAE,KAAK;CAC7D;CAEA,sBAAsB,WAAmB;EACvC,KAAK,iBAAiB,IAAI,GAAG;IAC1B,6BAA6B,KAAK;IAClC,uBAAuB;EAC1B,CAAC;CACH;CAEA,gBAAgB;EACd,SACE,sBAAsB,KAAK,GAAG,GAC9B,IAAI,SAAS,EACX,QAAQ,MAAM,WAAW,aAAa;GACpC,KAAK,eAAe,EAAC,KAAI,CAAC,CAAC,CAAC,WAAW,SAAS,GAAG,QAAQ;EAC7D,EACF,CAAC,SAIK,CAAC,CACT;CACF;CAEA,eACE,gBACA,aAIA;EAMA,SACE,SAAS,KAAK,cAAc,GAC5B,IAAI,SAAS;GACX,YAAY;GACZ,QAAQ,YAAe,WAAW,aAChC,YAAY,YAAY,QAAQ;EACpC,CAAC,IACD,MACE,IACI,KAAK,iBAAiB,CAAC,IACvB,KAAK,MAAM,iCAAiC,CACpD;CACF;CAEA,iBAAiB,GAAY;EAC3B,MAAM,YACJ,kBAAkB,CAAC,CAAC,EAAE,aAAa,sBAAsB,CAAC,CAAC,CAAC;EAE9D,KAAK,gBAAgB,WAAW,CAAC;CACnC;CAEA,gBAAgB,WAAsB,QAAkB;EACtD,KAAK,UAAU,WAAW,MAAM;EAChC,KAAK,MACH,GAAG,UAAU,KAAK,IAAI,UAAU,OAAO,KAAK,UAAU,WACtD,SACF;CACF;CAEA,yBAAyB,KAAK,IAAI;CAElC,uBAAuB;EACrB,IAAI,KAAK,IAAI,IAAI,KAAK,yBAAyB,4BAA4B;GACzE,KAAK,IAAI,QAAQ,uBAAuB;GACxC,KAAK,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,qBAAqB;EAC/C;CACF;CAEA,KACE,MACA,UACA,YACA;EACA,KAAK,yBAAyB,KAAK,IAAI;EACvC,KAAK,kBAAkB,KAAK,MAAM,UAAU,UAAU;CACxD;CAEA,UAAU,WAAsB,QAAkB;EAChD,UAAU,KAAK,KAAK,KAAK,KAAK,WAAW,MAAM;CACjD;AACF;AAOA,SAAgB,KACd,IACA,IACA,MACA,UACA,YACA;CACA,IAAI,GAAG,eAAeE,YAAU,MAAM;EACpC,eAAe,KAAK,UAAU,IAAI;EAClC,IAAI,aAAa,uBAAuB;GACtC,GAAG,KAAK,UAAU;GAClB;EACF;EAEA,IAAI,YAAY;EAChB,MAAM,QAAQ,iBAAiB;GAC7B,SAAS,0BAA0B,CAAC;EACtC,GAAG,yBAAyB;EAC5B,MAAM,MAAM;EAEZ,MAAM,YAAY,UAAyB;GACzC,IAAI,WACF;GAEF,YAAY;GACZ,aAAa,KAAK;GAClB,SAAS,KAAK;EAChB;EAEA,GAAG,KAAK,YAAY,QAAQ;CAC9B,OAAO;EACL,GAAG,QAAQ,2CAA2C,GAAG,WAAW,IAAI,EACtE,SAAS,KACX,CAAC;EACD,IAAI,aAAa,uBACf,SACE,IAAI,uBACF;GACE,MAAMC;GACN,SAAS;GACT,QAAQH;EACV,GACA,MACF,CACF;CAEJ;AACF;;;;;;;;AASA,IAAa,mBAAb,MAA8B;CAC5B;CACA;CACA;CACA;;CAEA;CAEA,YAAY,IAAgB,IAAmB,iBAAyB;EACtE,KAAK,MAAM;EACX,KAAK,MAAM;EACX,KAAK,mBAAmB;CAC1B;CAEA,KACE,MACA,UACA,YACM;EACN,IAAI,KAAK,oBAAA,IACP,QAAQ,KAAK,IAAb;GACE,KAAK;IACH,OACE,KAAK,sBAAsB,KAAA,GAC3B,uDACF;IACA,KAAK,oBAAoB,IAAI,iBAC3B,KAAK,oBACP;IACA,KAAK,uBAAuB,KAAA;IAC5B;GACF,KAAK;IACH,KAAK,cAAc,MAAM,UAAU,UAAU;IAC7C;GACF,KAAK;IACH,KAAK,aAAa,MAAM,QAAQ;IAChC;EACJ;EAEF,KAAK,KAAK,KAAK,KAAK,KAAK,MAAM,UAAU,UAAU;CACrD;CAEA,cACE,UACA,UACA,YACM;EACN,MAAM,UAAU,KAAK;EACrB,OAAO,SAAS,uCAAuC;EAEvD,eAAe,KAAK,UAAU,QAAQ;EACtC,OACE,WAAW,WAAW,gBAAgB,KAAK,WAAW,SAAS,GAAG,GAClE,6BACF;EACA,QACG,SAAS,WAAW,MAAM,iBAAiB,QAAQ,EAAE,IAAG,UACvD,WAAW,KAAK,KAAK,KAAK,KAAK,KAAK,CACtC,CAAC,CACA,WACO,eAAe,QAAQ,IAC7B,UAAS,eAAe,UAAU,QAAQ,KAAK,CAAC,CAClD;CACJ;CAEA,aACE,SACA,UACM;EACN,MAAM,UAAU,KAAK;EACrB,OAAO,SAAS,sCAAsC;EACtD,KAAK,oBAAoB,KAAA;EAEzB,IAAI,QAAQ,EAAE,CAAC,QAAQ;GACrB,QAAQ,OAAO;GACf,KAAK,KAAK,KAAK,KAAK,KAAK,SAAS,QAAQ;GAC1C;EACF;EAEA,QACG,QAAO,UAAS,WAAW,KAAK,KAAK,KAAK,KAAK,KAAK,CAAC,CAAC,CACtD,MAAK,WAAU;GACd,KAAK,uBAAuB;GAC5B,OAAO,UAAU,KAAK,KAAK,KAAK,KAAK,OAAO;EAC9C,CAAC,CAAC,CACD,WACO,eAAe,QAAQ,IAC7B,UAAS,eAAe,UAAU,QAAQ,KAAK,CAAC,CAClD;CACJ;AACF;AAEA,IAAM,mBAAmB;AAEzB,SAAS,UACP,IACA,IACA,MACe;CACf,OAAO,IAAI,SAAS,SAAS,WAAW;EACtC,KAAK,IAAI,IAAI,OAAM,UAAU,QAAQ,OAAO,KAAK,IAAI,QAAQ,CAAE;CACjE,CAAC;AACH;AAEA,SAAS,WACP,IACA,IACA,OACe;CACf,OAAO,IAAI,SAAS,SAAS,WAAW;EACtC,IAAI,GAAG,eAAeE,YAAU,MAAM;GACpC,MAAM,QAAQ,qBAAqB;GACnC,GAAG,QACD,kDAAkD,GAAG,WAAW,EAClE;GACA,OAAO,KAAK;GACZ;EACF;EACA,IAAI,YAAY;EAChB,MAAM,QAAQ,iBACN,SAAS,0BAA0B,CAAC,GAC1C,yBACF;EACA,MAAM,MAAM;EAEZ,MAAM,YAAY,UAAyB;GACzC,IAAI,WACF;GAEF,YAAY;GACZ,aAAa,KAAK;GAClB,QAAQ,OAAO,KAAK,IAAI,QAAQ;EAClC;EAEA,GAAG,KAAK,OAAO,QAAQ;CACzB,CAAC;AACH;AAEA,SAAS,eACP,UACA,OACM;CACN,IAAI,aAAa,uBACf,SAAS,KAAK;AAElB;AAEA,SAAS,QAAQ,OAAuB;CACtC,OAAO,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;AACjE;AAEA,SAAS,uBAA+C;CACtD,OAAO,IAAI,uBACT;EACE,MAAMC;EACN,SAAS;EACT,QAAQH;CACV,GACA,MACF;AACF;AAEA,SAAS,4BAAoD;CAC3D,OAAO,IAAI,uBACT;EACE,MAAMG;EACN,SAAS,kCAAkC,0BAA0B;EACrE,QAAQH;CACV,GACA,MACF;AACF;AAEA,SAAgB,UACd,IACA,IACA,WACA,QACA;CACA,KAAK,GAAG,YAAY,aAAa,UAAU,IAAI;CAE/C,IAAI;CAGJ,IAAI,kBAAkB,wBACpB,WAAW,OAAO;MAIf,IACH,SAAS,MAAM,KACf,uBAAuB,MAAM,KAC7B,yBAAyB,UAAU,OAAO,GAE1C,WAAW;MAGR,IACH,UAAU,SAAS,oBACnB,UAAU,SAAS,mBAEnB,WAAW;MAEX,WAAW,SAAS,YAAY,MAAM,IAAI;CAG5C,GAAG,SAAS,GAAG,8BAA8B,WAAW,UAAU,EAAE;CACpE,KAAK,IAAI,IAAI,CAAC,SAAS,SAAS,GAAG,qBAAqB;AAC1D;AAEA,SAAS,kBAAkB,OAA2C;CACpE,IAAI,gBAAgB,KAAK,GACvB,OAAO;CAET,IAAI,iBAAiB,SAAS,MAAM,OAClC,OAAO,kBAAkB,MAAM,KAAK;AAGxC;AAEA,SAAS,SAAS,OAAyB;CACzC,OAAO,QACL,SACA,OAAO,UAAU,YACjB,WAAW,SACX,OAAQ,MAA2B,UAAU,WAC/C;AACF;AAIA,IAAM,+CAA+B,IAAI,IAAI;CAC3C;CACA;CACA;AACF,CAAC;AAID,IAAM,oCAAoC,CACxC,mDACF;AAEA,SAAS,uBAAuB,OAAyB;CACvD,IAAI,CAAC,SAAS,OAAO,UAAU,UAC7B,OAAO;CAET,MAAM,YACJ,UAAU,QAAQ,OAAQ,MAA2B,IAAI,IAAI,KAAA;CAC/D,OAAO,QACL,aAAa,6BAA6B,IAAI,UAAU,YAAY,CAAC,CACvE;AACF;AAEA,SAAS,yBAAyB,SAAsC;CACtE,IAAI,CAAC,SACH,OAAO;CAET,MAAM,QAAQ,QAAQ,YAAY;CAClC,OAAO,kCAAkC,MAAK,YAC5C,MAAM,SAAS,OAAO,CACxB;AACF"}

package/out/zero-cache/src/workers/rehome.d.ts

@@ -1,6 +1,16 @@
 import type { LogContext } from '@rocicorp/logger';
 import type { WebSocket } from 'ws';
 import type { ZeroConfig } from '../config/zero-config.ts';
+/**
+ * The range of the `maxBackoffMs` sent with a draining Rehome. Without it, the
+ * client waits its full reconnect interval (5 seconds), when all it needs to
+ * do is try again and be routed to another server. The client does not add
+ * jitter, so it is chosen per connection here, spreading out the clients that
+ * arrive together and any that a load balancer sends back before it stops
+ * routing to this server.
+ */
+export declare const MIN_REHOME_BACKOFF_MS = 250;
+export declare const MAX_REHOME_BACKOFF_MS = 1000;
 /**
  * Returns the `Set-Cookie` header line that expires the configured load
  * balancer affinity cookie, or `undefined` if none is configured. It must be

package/out/zero-cache/src/workers/rehome.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"rehome.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/workers/rehome.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,IAAI,CAAC;AAGlC,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,0BAA0B,CAAC;AAIzD;;;;;GAKG;AACH,wBAAgB,2BAA2B,CAAC,EAC1C,cAAc,GACf,EAAE,IAAI,CAAC,UAAU,EAAE,gBAAgB,CAAC,GAAG,MAAM,GAAG,SAAS,CAMzD;AAED;;GAEG;AACH,wBAAgB,MAAM,CAAC,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,SAAS,QAenD"}
+{"version":3,"file":"rehome.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/workers/rehome.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,IAAI,CAAC;AAIlC,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,0BAA0B,CAAC;AAIzD;;;;;;;GAOG;AACH,eAAO,MAAM,qBAAqB,MAAM,CAAC;AACzC,eAAO,MAAM,qBAAqB,OAAO,CAAC;AAE1C;;;;;GAKG;AACH,wBAAgB,2BAA2B,CAAC,EAC1C,cAAc,GACf,EAAE,IAAI,CAAC,UAAU,EAAE,gBAAgB,CAAC,GAAG,MAAM,GAAG,SAAS,CAMzD;AAED;;GAEG;AACH,wBAAgB,MAAM,CAAC,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,SAAS,QAgBnD"}

package/out/zero-cache/src/workers/rehome.js

@@ -1,8 +1,9 @@
 import { Rehome } from "../../../zero-protocol/src/error-kind-enum.js";
 import { ZeroCache } from "../../../zero-protocol/src/error-origin-enum.js";
+import { randInt } from "../../../shared/src/rand.js";
 import { ProtocolErrorWithLevel } from "../types/error-with-level.js";
 import { sendError } from "./connection.js";
-//#region ../zero-cache/src/workers/rehome.ts
+var MAX_REHOME_BACKOFF_MS = 1e3;
 /**
 * Returns the `Set-Cookie` header line that expires the configured load
 * balancer affinity cookie, or `undefined` if none is configured. It must be
@@ -25,12 +26,13 @@ function rehome(lc, ws) {
 	const error = new ProtocolErrorWithLevel({
 		kind: Rehome,
 		message: "Server is draining",
+		maxBackoffMs: randInt(250, MAX_REHOME_BACKOFF_MS),
 		origin: ZeroCache
 	}, "info");
 	sendError(lc, ws, error.errorBody, error);
 	ws.close(1001, "Server is draining");
 }
 //#endregion
-export { expiredAffinityCookieHeader, rehome };
+export { MAX_REHOME_BACKOFF_MS, expiredAffinityCookieHeader, rehome };
 
 //# sourceMappingURL=rehome.js.map

package/out/zero-cache/src/workers/rehome.js.map

@@ -1 +1 @@
-{"version":3,"file":"rehome.js","names":["ErrorKind.Rehome","ErrorOrigin.ZeroCache"],"sources":["../../../../../zero-cache/src/workers/rehome.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {WebSocket} from 'ws';\nimport {ErrorKind} from '../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../zero-protocol/src/error-origin.ts';\nimport type {ZeroConfig} from '../config/zero-config.ts';\nimport {ProtocolErrorWithLevel} from '../types/error-with-level.ts';\nimport {sendError} from './connection.ts';\n\n/**\n * Returns the `Set-Cookie` header line that expires the configured load\n * balancer affinity cookie, or `undefined` if none is configured. It must be\n * part of the 101 response of a draining handshake so that the client's next\n * attempt is routed to a different server.\n */\nexport function expiredAffinityCookieHeader({\n  affinityCookie,\n}: Pick<ZeroConfig, 'affinityCookie'>): string | undefined {\n  if (!affinityCookie?.name) {\n    return undefined;\n  }\n  const {name, path, domain} = affinityCookie;\n  return `Set-Cookie: ${name}=; Max-Age=0; Path=${path}${domain ? `; Domain=${domain}` : ''}; Secure; HttpOnly; SameSite=None`;\n}\n\n/**\n * Sends a Rehome error to a connection on a draining server and closes it.\n */\nexport function rehome(lc: LogContext, ws: WebSocket) {\n  ws.on('error', error => {\n    lc.warn?.('error while rehoming draining connection', error);\n    ws.terminate();\n  });\n  const error = new ProtocolErrorWithLevel(\n    {\n      kind: ErrorKind.Rehome,\n      message: 'Server is draining',\n      origin: ErrorOrigin.ZeroCache,\n    },\n    'info',\n  );\n  sendError(lc, ws, error.errorBody, error);\n  ws.close(1001, 'Server is draining');\n}\n"],"mappings":";;;;;;;;;;;AAcA,SAAgB,4BAA4B,EAC1C,kBACyD;CACzD,IAAI,CAAC,gBAAgB,MACnB;CAEF,MAAM,EAAC,MAAM,MAAM,WAAU;CAC7B,OAAO,eAAe,KAAK,qBAAqB,OAAO,SAAS,YAAY,WAAW,GAAG;AAC5F;;;;AAKA,SAAgB,OAAO,IAAgB,IAAe;CACpD,GAAG,GAAG,UAAS,UAAS;EACtB,GAAG,OAAO,4CAA4C,KAAK;EAC3D,GAAG,UAAU;CACf,CAAC;CACD,MAAM,QAAQ,IAAI,uBAChB;EACE,MAAMA;EACN,SAAS;EACT,QAAQC;CACV,GACA,MACF;CACA,UAAU,IAAI,IAAI,MAAM,WAAW,KAAK;CACxC,GAAG,MAAM,MAAM,oBAAoB;AACrC"}
+{"version":3,"file":"rehome.js","names":["ErrorKind.Rehome","ErrorOrigin.ZeroCache"],"sources":["../../../../../zero-cache/src/workers/rehome.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {WebSocket} from 'ws';\nimport {randInt} from '../../../shared/src/rand.ts';\nimport {ErrorKind} from '../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../zero-protocol/src/error-origin.ts';\nimport type {ZeroConfig} from '../config/zero-config.ts';\nimport {ProtocolErrorWithLevel} from '../types/error-with-level.ts';\nimport {sendError} from './connection.ts';\n\n/**\n * The range of the `maxBackoffMs` sent with a draining Rehome. Without it, the\n * client waits its full reconnect interval (5 seconds), when all it needs to\n * do is try again and be routed to another server. The client does not add\n * jitter, so it is chosen per connection here, spreading out the clients that\n * arrive together and any that a load balancer sends back before it stops\n * routing to this server.\n */\nexport const MIN_REHOME_BACKOFF_MS = 250;\nexport const MAX_REHOME_BACKOFF_MS = 1000;\n\n/**\n * Returns the `Set-Cookie` header line that expires the configured load\n * balancer affinity cookie, or `undefined` if none is configured. It must be\n * part of the 101 response of a draining handshake so that the client's next\n * attempt is routed to a different server.\n */\nexport function expiredAffinityCookieHeader({\n  affinityCookie,\n}: Pick<ZeroConfig, 'affinityCookie'>): string | undefined {\n  if (!affinityCookie?.name) {\n    return undefined;\n  }\n  const {name, path, domain} = affinityCookie;\n  return `Set-Cookie: ${name}=; Max-Age=0; Path=${path}${domain ? `; Domain=${domain}` : ''}; Secure; HttpOnly; SameSite=None`;\n}\n\n/**\n * Sends a Rehome error to a connection on a draining server and closes it.\n */\nexport function rehome(lc: LogContext, ws: WebSocket) {\n  ws.on('error', error => {\n    lc.warn?.('error while rehoming draining connection', error);\n    ws.terminate();\n  });\n  const error = new ProtocolErrorWithLevel(\n    {\n      kind: ErrorKind.Rehome,\n      message: 'Server is draining',\n      maxBackoffMs: randInt(MIN_REHOME_BACKOFF_MS, MAX_REHOME_BACKOFF_MS),\n      origin: ErrorOrigin.ZeroCache,\n    },\n    'info',\n  );\n  sendError(lc, ws, error.errorBody, error);\n  ws.close(1001, 'Server is draining');\n}\n"],"mappings":";;;;;AAkBA,IAAa,wBAAwB;;;;;;;AAQrC,SAAgB,4BAA4B,EAC1C,kBACyD;CACzD,IAAI,CAAC,gBAAgB,MACnB;CAEF,MAAM,EAAC,MAAM,MAAM,WAAU;CAC7B,OAAO,eAAe,KAAK,qBAAqB,OAAO,SAAS,YAAY,WAAW,GAAG;AAC5F;;;;AAKA,SAAgB,OAAO,IAAgB,IAAe;CACpD,GAAG,GAAG,UAAS,UAAS;EACtB,GAAG,OAAO,4CAA4C,KAAK;EAC3D,GAAG,UAAU;CACf,CAAC;CACD,MAAM,QAAQ,IAAI,uBAChB;EACE,MAAMA;EACN,SAAS;EACT,cAAc,QAAA,KAA+B,qBAAqB;EAClE,QAAQC;CACV,GACA,MACF;CACA,UAAU,IAAI,IAAI,MAAM,WAAW,KAAK;CACxC,GAAG,MAAM,MAAM,oBAAoB;AACrC"}

package/out/zero-cache/src/workers/replicator.js

@@ -90,7 +90,7 @@ async function prepare(lc, { file, vacuumIntervalHours }, walMode, mode) {
 			}
 		}
 		await setJournalMode(lc, replica, walMode);
-		const pragmas = getPragmaConfig(mode);
+		const pragmas = getPragmaConfig();
 		applyPragmas(replica, pragmas);
 		replica.pragma("optimize = 0x10002");
 		lc.info?.(`optimized ${file}`);
@@ -119,15 +119,20 @@ async function setJournalMode(lc, replica, mode) {
 	throw err;
 }
 /**
-* Returns the PragmaConfig for a given replica file mode.
+* Returns the PragmaConfig for the replica.
 * This is used by both the main thread (setupReplica) and
 * the write worker thread to apply the same pragma settings.
+*
+* The writer never checkpoints, since an automatic checkpoint runs (and
+* syncs the disk) within the COMMIT, stalling replication on a slow disk.
+* Instead, litestream checkpoints the backup replica, and the
+* `ServingCheckpointer` checkpoints the serving replica.
 */
-function getPragmaConfig(mode) {
+function getPragmaConfig() {
 	return {
 		busyTimeout: 3e4,
 		analysisLimit: 1e3,
-		walAutocheckpoint: mode === "backup" ? 0 : void 0
+		walAutocheckpoint: 0
 	};
 }
 function setupReplica(lc, mode, replicaOptions) {

package/out/zero-cache/src/workers/replicator.js.map

@@ -1 +1 @@
-{"version":3,"file":"replicator.js","names":["v.literalUnion"],"sources":["../../../../../zero-cache/src/workers/replicator.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {SqliteError} from '@rocicorp/zero-sqlite3';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport type {ReplicaOptions} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {upgradeReplica} from '../services/change-source/common/replica-schema.ts';\nimport {deleteChangeLogDB} from '../services/replicator/change-log-db.ts';\nimport {Notifier} from '../services/replicator/notifier.ts';\nimport type {\n  ReplicaState,\n  ReplicaStateNotifier,\n  Replicator,\n} from '../services/replicator/replicator.ts';\nimport {\n  getAscendingEvents,\n  recordEvent,\n} from '../services/replicator/schema/replication-state.ts';\nimport {\n  applyPragmas,\n  type PragmaConfig,\n} from '../services/replicator/write-worker-client.ts';\nimport type {Worker} from '../types/processes.ts';\n\nexport const replicaFileModeSchema = v.literalUnion(\n  'serving',\n  'serving-copy',\n  'backup',\n);\n\nexport type ReplicaFileMode = v.Infer<typeof replicaFileModeSchema>;\n\nexport type WalMode = 'wal' | 'wal2';\n\nexport function replicaFileName(replicaFile: string, mode: ReplicaFileMode) {\n  return mode === 'serving-copy' ? `${replicaFile}-serving-copy` : replicaFile;\n}\n\n/**\n * Whether a replicator worker deletes the change-log file beside its replica at\n * startup.\n *\n * The change log belongs to the change-streamer, so this only ever cleans up a\n * file that nothing is writing. The predicate is deliberately the config flag\n * and nothing else: keying it on anything derived per replica -- a file mode, or\n * the retired `logsChangeStream` -- would make every replicator start unlink the\n * change-streamer's live log, because no replicator writes it any more. Paths\n * coincide whenever `fileMode !== 'serving-copy'`, so that hazard is real in the\n * shipped no-`backupURL` configuration.\n *\n * Deleting is safe when nothing writes the log: it is excluded from the\n * litestream backup, only the writer purges it, and re-enabling the writer\n * reseeds at the resume watermark.\n */\nexport function replicatorDeletesStaleChangeLog(\n  sqliteChangeLogMode: string,\n): boolean {\n  return sqliteChangeLogMode === 'off';\n}\n\n/**\n * The `mode=off` cleanup itself: deletes the change-log file beside `dbPath`\n * when {@link replicatorDeletesStaleChangeLog} says nothing writes it, and\n * reports whether it did. The decision and the delete live in one function so\n * that the guard test covers the call the replicator actually makes, not just\n * the predicate.\n */\nexport function deleteStaleChangeLog(\n  sqliteChangeLogMode: string,\n  dbPath: string,\n): boolean {\n  if (!replicatorDeletesStaleChangeLog(sqliteChangeLogMode)) {\n    return false;\n  }\n  deleteChangeLogDB(dbPath);\n  return true;\n}\n\nconst MILLIS_PER_HOUR = 1000 * 60 * 60;\nconst MB = 1024 * 1024;\n\nasync function prepare(\n  lc: LogContext,\n  {file, vacuumIntervalHours}: ReplicaOptions,\n  walMode: WalMode,\n  mode: ReplicaFileMode,\n): Promise<{file: string; walMode: WalMode; pageSize: number}> {\n  // Perform any upgrades to the replica in case the backup is an\n  // earlier version.\n  await upgradeReplica(lc, `${mode}-replica`, file);\n\n  const replica = new Database(lc, file);\n  let pageSize: number;\n  try {\n    // Switching between WAL modes (e.g. from a restored 'wal' replica to\n    // 'wal2') goes through 'delete', which folds the WAL(2) files into the\n    // main db. Leaving WAL mode requires the WAL not to be persisted (see\n    // Database.persistWal()). A replica already in the target mode keeps its\n    // WAL, from which a litestream backup may continue.\n    const [{journal_mode: journalMode}] = replica.pragma<{\n      journal_mode: string;\n    }>('journal_mode');\n    if (journalMode !== walMode) {\n      replica.persistWal(false);\n      try {\n        await setJournalMode(lc, replica, 'delete');\n      } finally {\n        replica.persistWal(true);\n      }\n    }\n\n    [{page_size: pageSize}] = replica.pragma<{page_size: number}>('page_size');\n    const [{page_count: pageCount}] = replica.pragma<{page_count: number}>(\n      'page_count',\n    );\n    const [{freelist_count: freelistCount}] = replica.pragma<{\n      freelist_count: number;\n    }>('freelist_count');\n\n    const dbSize = ((pageCount * pageSize) / MB).toFixed(2);\n    const freelistSize = ((freelistCount * pageSize) / MB).toFixed(2);\n\n    // TODO: Consider adding a freelist size or ratio based vacuum trigger.\n    lc.info?.(`Size of db ${file}: ${dbSize} MB (${freelistSize} MB freeable)`);\n\n    // Check for the VACUUM threshold.\n    const events = getAscendingEvents(replica);\n    lc.debug?.(`Runtime events for db ${file}`, {events});\n    if (vacuumIntervalHours !== undefined) {\n      const millisSinceLastEvent =\n        Date.now() - (events.at(-1)?.timestamp.getTime() ?? 0);\n      if (millisSinceLastEvent / MILLIS_PER_HOUR > vacuumIntervalHours) {\n        lc.info?.(`Performing maintenance cleanup on ${file}`);\n        const t0 = performance.now();\n        replica.unsafeMode(true);\n        // Leaving WAL mode requires the WAL not to be persisted (see\n        // Database.persistWal()). The VACUUM rewrites the database outside\n        // of the WAL, so a litestream backup cannot continue from it anyway.\n        replica.persistWal(false);\n        try {\n          replica.pragma('journal_mode = OFF');\n          replica.exec('VACUUM');\n          recordEvent(replica, 'vacuum');\n          const t1 = performance.now();\n          lc.info?.(`VACUUM completed (${t1 - t0} ms)`);\n        } catch (e) {\n          // Another connection holds the replica open. The VACUUM is retried\n          // at the next startup.\n          if (!(e instanceof SqliteError && e.code === 'SQLITE_BUSY')) {\n            throw e;\n          }\n          lc.warn?.(`Skipping VACUUM of ${file}: the database is locked`, e);\n        } finally {\n          replica.persistWal(true);\n          replica.unsafeMode(false);\n        }\n      }\n    }\n\n    await setJournalMode(lc, replica, walMode);\n\n    const pragmas = getPragmaConfig(mode);\n    applyPragmas(replica, pragmas);\n\n    replica.pragma('optimize = 0x10002');\n    lc.info?.(`optimized ${file}`);\n  } finally {\n    replica.close();\n  }\n  return {file, walMode, pageSize};\n}\n\n// Setting the journal_mode requires an exclusive lock on the replica.\n// Add resilience against random replica reads (for stats, etc.) by\n// retrying if the database is locked. Note that the busy_timeout doesn't\n// work here.\nasync function setJournalMode(\n  lc: LogContext,\n  replica: Database,\n  mode: 'delete' | 'wal' | 'wal2',\n) {\n  lc.info?.(`setting ${replica.name} to ${mode} mode`);\n  let err: unknown;\n  for (let i = 0; i < 5; i++) {\n    try {\n      replica.pragma(`journal_mode = ${mode}`);\n      return;\n    } catch (e) {\n      lc.warn?.(`error setting journal_mode to ${mode} (attempt ${i + 1})`, e);\n      err = e;\n    }\n    await sleep(500);\n  }\n  throw err;\n}\n\n/**\n * Returns the PragmaConfig for a given replica file mode.\n * This is used by both the main thread (setupReplica) and\n * the write worker thread to apply the same pragma settings.\n */\nexport function getPragmaConfig(mode: ReplicaFileMode): PragmaConfig {\n  return {\n    busyTimeout: 30000,\n    analysisLimit: 1000,\n    walAutocheckpoint: mode === 'backup' ? 0 : undefined,\n  };\n}\n\nexport function setupReplica(\n  lc: LogContext,\n  mode: ReplicaFileMode,\n  replicaOptions: ReplicaOptions,\n) {\n  lc.info?.(`setting up ${mode} replica`);\n\n  switch (mode) {\n    case 'backup':\n      return prepare(lc, replicaOptions, 'wal', mode);\n\n    case 'serving-copy': {\n      // In 'serving-copy' mode, the original file is being used for 'backup'\n      // mode, so we make a copy for servicing sync requests.\n      const {file} = replicaOptions;\n      const copyLocation = replicaFileName(file, mode);\n      deleteLiteDB(copyLocation);\n\n      const start = Date.now();\n      lc.info?.(`copying ${file} to ${copyLocation}`);\n      const replica = new Database(lc, file);\n      replica.prepare(`VACUUM INTO ?`).run(copyLocation);\n      replica.close();\n      lc.info?.(`finished copy (${Date.now() - start} ms)`);\n\n      return prepare(lc, {...replicaOptions, file: copyLocation}, 'wal2', mode);\n    }\n\n    case 'serving':\n      return prepare(lc, replicaOptions, 'wal2', mode);\n\n    default:\n      throw new Error(`Invalid ReplicaMode ${mode}`);\n  }\n}\n\nexport function setUpMessageHandlers(\n  lc: LogContext,\n  replicator: Replicator,\n  parent: Worker,\n) {\n  handleSubscriptionsFrom(lc, parent, replicator);\n}\n\ntype Notification = ['notify', ReplicaState];\n\nexport function handleSubscriptionsFrom(\n  lc: LogContext,\n  subscriber: Worker,\n  notifier: ReplicaStateNotifier,\n) {\n  subscriber.onMessageType('subscribe', async () => {\n    const subscription = notifier.subscribe();\n\n    subscriber.on('close', () => {\n      lc.debug?.(`closing replication subscription from ${subscriber.pid}`);\n      subscription.cancel();\n    });\n\n    for await (const msg of subscription) {\n      try {\n        subscriber.send<Notification>(['notify', msg]);\n      } catch (e) {\n        const log =\n          e instanceof Error &&\n          'code' in e &&\n          // This can happen in a race condition if the subscribing process\n          // is closed before the 'close' message is processed.\n          e.code === 'ERR_IPC_CHANNEL_CLOSED'\n            ? 'warn'\n            : 'error';\n\n        lc[log]?.(\n          `error sending replicator notification to ${subscriber.pid}: ${String(e)}`,\n          e,\n        );\n      }\n    }\n  });\n}\n\n/**\n * Creates a Notifier to relay notifications the notifier of another Worker.\n * This does not send the initial subscription message. Use {@link subscribeTo}\n * to initiate the subscription.\n */\nexport function createNotifierFrom(\n  _lc: LogContext,\n  source: Worker,\n  onNotify?: (state: ReplicaState) => void,\n): Notifier {\n  const notifier = new Notifier();\n  source.onMessageType<Notification>('notify', msg => {\n    onNotify?.(msg);\n    void notifier.notifySubscribers(msg);\n  });\n  return notifier;\n}\n\nexport function subscribeTo(_lc: LogContext, source: Worker) {\n  source.send(['subscribe', {}]);\n}\n"],"mappings":";;;;;;;;;;;AAyBA,IAAa,wBAAwBA,aACnC,WACA,gBACA,QACF;AAMA,SAAgB,gBAAgB,aAAqB,MAAuB;CAC1E,OAAO,SAAS,iBAAiB,GAAG,YAAY,iBAAiB;AACnE;;;;;;;;;;;;;;;;;AAkBA,SAAgB,gCACd,qBACS;CACT,OAAO,wBAAwB;AACjC;;;;;;;;AASA,SAAgB,qBACd,qBACA,QACS;CACT,IAAI,CAAC,gCAAgC,mBAAmB,GACtD,OAAO;CAET,kBAAkB,MAAM;CACxB,OAAO;AACT;AAEA,IAAM,kBAAkB;AACxB,IAAM,KAAK;AAEX,eAAe,QACb,IACA,EAAC,MAAM,uBACP,SACA,MAC6D;CAG7D,MAAM,eAAe,IAAI,GAAG,KAAK,WAAW,IAAI;CAEhD,MAAM,UAAU,IAAI,SAAS,IAAI,IAAI;CACrC,IAAI;CACJ,IAAI;EAMF,MAAM,CAAC,EAAC,cAAc,iBAAgB,QAAQ,OAE3C,cAAc;EACjB,IAAI,gBAAgB,SAAS;GAC3B,QAAQ,WAAW,KAAK;GACxB,IAAI;IACF,MAAM,eAAe,IAAI,SAAS,QAAQ;GAC5C,UAAU;IACR,QAAQ,WAAW,IAAI;GACzB;EACF;EAEA,CAAC,CAAC,WAAW,aAAa,QAAQ,OAA4B,WAAW;EACzE,MAAM,CAAC,EAAC,YAAY,eAAc,QAAQ,OACxC,YACF;EACA,MAAM,CAAC,EAAC,gBAAgB,mBAAkB,QAAQ,OAE/C,gBAAgB;EAEnB,MAAM,UAAW,YAAY,WAAY,GAAA,CAAI,QAAQ,CAAC;EACtD,MAAM,gBAAiB,gBAAgB,WAAY,GAAA,CAAI,QAAQ,CAAC;EAGhE,GAAG,OAAO,cAAc,KAAK,IAAI,OAAO,OAAO,aAAa,cAAc;EAG1E,MAAM,SAAS,mBAAmB,OAAO;EACzC,GAAG,QAAQ,yBAAyB,QAAQ,EAAC,OAAM,CAAC;EACpD,IAAI,wBAAwB,KAAA,GAExB;QAAA,KAAK,IAAI,KAAK,OAAO,GAAG,EAAE,CAAC,EAAE,UAAU,QAAQ,KAAK,MAC3B,kBAAkB,qBAAqB;IAChE,GAAG,OAAO,qCAAqC,MAAM;IACrD,MAAM,KAAK,YAAY,IAAI;IAC3B,QAAQ,WAAW,IAAI;IAIvB,QAAQ,WAAW,KAAK;IACxB,IAAI;KACF,QAAQ,OAAO,oBAAoB;KACnC,QAAQ,KAAK,QAAQ;KACrB,YAAY,SAAS,QAAQ;KAC7B,MAAM,KAAK,YAAY,IAAI;KAC3B,GAAG,OAAO,qBAAqB,KAAK,GAAG,KAAK;IAC9C,SAAS,GAAG;KAGV,IAAI,EAAE,aAAa,eAAe,EAAE,SAAS,gBAC3C,MAAM;KAER,GAAG,OAAO,sBAAsB,KAAK,2BAA2B,CAAC;IACnE,UAAU;KACR,QAAQ,WAAW,IAAI;KACvB,QAAQ,WAAW,KAAK;IAC1B;GACF;;EAGF,MAAM,eAAe,IAAI,SAAS,OAAO;EAEzC,MAAM,UAAU,gBAAgB,IAAI;EACpC,aAAa,SAAS,OAAO;EAE7B,QAAQ,OAAO,oBAAoB;EACnC,GAAG,OAAO,aAAa,MAAM;CAC/B,UAAU;EACR,QAAQ,MAAM;CAChB;CACA,OAAO;EAAC;EAAM;EAAS;CAAQ;AACjC;AAMA,eAAe,eACb,IACA,SACA,MACA;CACA,GAAG,OAAO,WAAW,QAAQ,KAAK,MAAM,KAAK,MAAM;CACnD,IAAI;CACJ,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK;EAC1B,IAAI;GACF,QAAQ,OAAO,kBAAkB,MAAM;GACvC;EACF,SAAS,GAAG;GACV,GAAG,OAAO,iCAAiC,KAAK,YAAY,IAAI,EAAE,IAAI,CAAC;GACvE,MAAM;EACR;EACA,MAAM,MAAM,GAAG;CACjB;CACA,MAAM;AACR;;;;;;AAOA,SAAgB,gBAAgB,MAAqC;CACnE,OAAO;EACL,aAAa;EACb,eAAe;EACf,mBAAmB,SAAS,WAAW,IAAI,KAAA;CAC7C;AACF;AAEA,SAAgB,aACd,IACA,MACA,gBACA;CACA,GAAG,OAAO,cAAc,KAAK,SAAS;CAEtC,QAAQ,MAAR;EACE,KAAK,UACH,OAAO,QAAQ,IAAI,gBAAgB,OAAO,IAAI;EAEhD,KAAK,gBAAgB;GAGnB,MAAM,EAAC,SAAQ;GACf,MAAM,eAAe,gBAAgB,MAAM,IAAI;GAC/C,aAAa,YAAY;GAEzB,MAAM,QAAQ,KAAK,IAAI;GACvB,GAAG,OAAO,WAAW,KAAK,MAAM,cAAc;GAC9C,MAAM,UAAU,IAAI,SAAS,IAAI,IAAI;GACrC,QAAQ,QAAQ,eAAe,CAAC,CAAC,IAAI,YAAY;GACjD,QAAQ,MAAM;GACd,GAAG,OAAO,kBAAkB,KAAK,IAAI,IAAI,MAAM,KAAK;GAEpD,OAAO,QAAQ,IAAI;IAAC,GAAG;IAAgB,MAAM;GAAY,GAAG,QAAQ,IAAI;EAC1E;EAEA,KAAK,WACH,OAAO,QAAQ,IAAI,gBAAgB,QAAQ,IAAI;EAEjD,SACE,MAAM,IAAI,MAAM,uBAAuB,MAAM;CACjD;AACF;AAEA,SAAgB,qBACd,IACA,YACA,QACA;CACA,wBAAwB,IAAI,QAAQ,UAAU;AAChD;AAIA,SAAgB,wBACd,IACA,YACA,UACA;CACA,WAAW,cAAc,aAAa,YAAY;EAChD,MAAM,eAAe,SAAS,UAAU;EAExC,WAAW,GAAG,eAAe;GAC3B,GAAG,QAAQ,yCAAyC,WAAW,KAAK;GACpE,aAAa,OAAO;EACtB,CAAC;EAED,WAAW,MAAM,OAAO,cACtB,IAAI;GACF,WAAW,KAAmB,CAAC,UAAU,GAAG,CAAC;EAC/C,SAAS,GAAG;GAUV,GARE,aAAa,SACb,UAAU,KAGV,EAAE,SAAS,2BACP,SACA,QAEC,GACL,4CAA4C,WAAW,IAAI,IAAI,OAAO,CAAC,KACvE,CACF;EACF;CAEJ,CAAC;AACH;;;;;;AAOA,SAAgB,mBACd,KACA,QACA,UACU;CACV,MAAM,WAAW,IAAI,SAAS;CAC9B,OAAO,cAA4B,WAAU,QAAO;EAClD,WAAW,GAAG;EACd,SAAc,kBAAkB,GAAG;CACrC,CAAC;CACD,OAAO;AACT;AAEA,SAAgB,YAAY,KAAiB,QAAgB;CAC3D,OAAO,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC;AAC/B"}
+{"version":3,"file":"replicator.js","names":["v.literalUnion"],"sources":["../../../../../zero-cache/src/workers/replicator.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {SqliteError} from '@rocicorp/zero-sqlite3';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport type {ReplicaOptions} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {upgradeReplica} from '../services/change-source/common/replica-schema.ts';\nimport {deleteChangeLogDB} from '../services/replicator/change-log-db.ts';\nimport {Notifier} from '../services/replicator/notifier.ts';\nimport type {\n  ReplicaState,\n  ReplicaStateNotifier,\n  Replicator,\n} from '../services/replicator/replicator.ts';\nimport {\n  getAscendingEvents,\n  recordEvent,\n} from '../services/replicator/schema/replication-state.ts';\nimport {\n  applyPragmas,\n  type PragmaConfig,\n} from '../services/replicator/write-worker-client.ts';\nimport type {Worker} from '../types/processes.ts';\n\nexport const replicaFileModeSchema = v.literalUnion(\n  'serving',\n  'serving-copy',\n  'backup',\n);\n\nexport type ReplicaFileMode = v.Infer<typeof replicaFileModeSchema>;\n\nexport type WalMode = 'wal' | 'wal2';\n\nexport function replicaFileName(replicaFile: string, mode: ReplicaFileMode) {\n  return mode === 'serving-copy' ? `${replicaFile}-serving-copy` : replicaFile;\n}\n\n/**\n * Whether a replicator worker deletes the change-log file beside its replica at\n * startup.\n *\n * The change log belongs to the change-streamer, so this only ever cleans up a\n * file that nothing is writing. The predicate is deliberately the config flag\n * and nothing else: keying it on anything derived per replica -- a file mode, or\n * the retired `logsChangeStream` -- would make every replicator start unlink the\n * change-streamer's live log, because no replicator writes it any more. Paths\n * coincide whenever `fileMode !== 'serving-copy'`, so that hazard is real in the\n * shipped no-`backupURL` configuration.\n *\n * Deleting is safe when nothing writes the log: it is excluded from the\n * litestream backup, only the writer purges it, and re-enabling the writer\n * reseeds at the resume watermark.\n */\nexport function replicatorDeletesStaleChangeLog(\n  sqliteChangeLogMode: string,\n): boolean {\n  return sqliteChangeLogMode === 'off';\n}\n\n/**\n * The `mode=off` cleanup itself: deletes the change-log file beside `dbPath`\n * when {@link replicatorDeletesStaleChangeLog} says nothing writes it, and\n * reports whether it did. The decision and the delete live in one function so\n * that the guard test covers the call the replicator actually makes, not just\n * the predicate.\n */\nexport function deleteStaleChangeLog(\n  sqliteChangeLogMode: string,\n  dbPath: string,\n): boolean {\n  if (!replicatorDeletesStaleChangeLog(sqliteChangeLogMode)) {\n    return false;\n  }\n  deleteChangeLogDB(dbPath);\n  return true;\n}\n\nconst MILLIS_PER_HOUR = 1000 * 60 * 60;\nconst MB = 1024 * 1024;\n\nasync function prepare(\n  lc: LogContext,\n  {file, vacuumIntervalHours}: ReplicaOptions,\n  walMode: WalMode,\n  mode: ReplicaFileMode,\n): Promise<{file: string; walMode: WalMode; pageSize: number}> {\n  // Perform any upgrades to the replica in case the backup is an\n  // earlier version.\n  await upgradeReplica(lc, `${mode}-replica`, file);\n\n  const replica = new Database(lc, file);\n  let pageSize: number;\n  try {\n    // Switching between WAL modes (e.g. from a restored 'wal' replica to\n    // 'wal2') goes through 'delete', which folds the WAL(2) files into the\n    // main db. Leaving WAL mode requires the WAL not to be persisted (see\n    // Database.persistWal()). A replica already in the target mode keeps its\n    // WAL, from which a litestream backup may continue.\n    const [{journal_mode: journalMode}] = replica.pragma<{\n      journal_mode: string;\n    }>('journal_mode');\n    if (journalMode !== walMode) {\n      replica.persistWal(false);\n      try {\n        await setJournalMode(lc, replica, 'delete');\n      } finally {\n        replica.persistWal(true);\n      }\n    }\n\n    [{page_size: pageSize}] = replica.pragma<{page_size: number}>('page_size');\n    const [{page_count: pageCount}] = replica.pragma<{page_count: number}>(\n      'page_count',\n    );\n    const [{freelist_count: freelistCount}] = replica.pragma<{\n      freelist_count: number;\n    }>('freelist_count');\n\n    const dbSize = ((pageCount * pageSize) / MB).toFixed(2);\n    const freelistSize = ((freelistCount * pageSize) / MB).toFixed(2);\n\n    // TODO: Consider adding a freelist size or ratio based vacuum trigger.\n    lc.info?.(`Size of db ${file}: ${dbSize} MB (${freelistSize} MB freeable)`);\n\n    // Check for the VACUUM threshold.\n    const events = getAscendingEvents(replica);\n    lc.debug?.(`Runtime events for db ${file}`, {events});\n    if (vacuumIntervalHours !== undefined) {\n      const millisSinceLastEvent =\n        Date.now() - (events.at(-1)?.timestamp.getTime() ?? 0);\n      if (millisSinceLastEvent / MILLIS_PER_HOUR > vacuumIntervalHours) {\n        lc.info?.(`Performing maintenance cleanup on ${file}`);\n        const t0 = performance.now();\n        replica.unsafeMode(true);\n        // Leaving WAL mode requires the WAL not to be persisted (see\n        // Database.persistWal()). The VACUUM rewrites the database outside\n        // of the WAL, so a litestream backup cannot continue from it anyway.\n        replica.persistWal(false);\n        try {\n          replica.pragma('journal_mode = OFF');\n          replica.exec('VACUUM');\n          recordEvent(replica, 'vacuum');\n          const t1 = performance.now();\n          lc.info?.(`VACUUM completed (${t1 - t0} ms)`);\n        } catch (e) {\n          // Another connection holds the replica open. The VACUUM is retried\n          // at the next startup.\n          if (!(e instanceof SqliteError && e.code === 'SQLITE_BUSY')) {\n            throw e;\n          }\n          lc.warn?.(`Skipping VACUUM of ${file}: the database is locked`, e);\n        } finally {\n          replica.persistWal(true);\n          replica.unsafeMode(false);\n        }\n      }\n    }\n\n    await setJournalMode(lc, replica, walMode);\n\n    const pragmas = getPragmaConfig();\n    applyPragmas(replica, pragmas);\n\n    replica.pragma('optimize = 0x10002');\n    lc.info?.(`optimized ${file}`);\n  } finally {\n    replica.close();\n  }\n  return {file, walMode, pageSize};\n}\n\n// Setting the journal_mode requires an exclusive lock on the replica.\n// Add resilience against random replica reads (for stats, etc.) by\n// retrying if the database is locked. Note that the busy_timeout doesn't\n// work here.\nasync function setJournalMode(\n  lc: LogContext,\n  replica: Database,\n  mode: 'delete' | 'wal' | 'wal2',\n) {\n  lc.info?.(`setting ${replica.name} to ${mode} mode`);\n  let err: unknown;\n  for (let i = 0; i < 5; i++) {\n    try {\n      replica.pragma(`journal_mode = ${mode}`);\n      return;\n    } catch (e) {\n      lc.warn?.(`error setting journal_mode to ${mode} (attempt ${i + 1})`, e);\n      err = e;\n    }\n    await sleep(500);\n  }\n  throw err;\n}\n\n/**\n * Returns the PragmaConfig for the replica.\n * This is used by both the main thread (setupReplica) and\n * the write worker thread to apply the same pragma settings.\n *\n * The writer never checkpoints, since an automatic checkpoint runs (and\n * syncs the disk) within the COMMIT, stalling replication on a slow disk.\n * Instead, litestream checkpoints the backup replica, and the\n * `ServingCheckpointer` checkpoints the serving replica.\n */\nexport function getPragmaConfig(): PragmaConfig {\n  return {\n    busyTimeout: 30000,\n    analysisLimit: 1000,\n    walAutocheckpoint: 0,\n  };\n}\n\nexport function setupReplica(\n  lc: LogContext,\n  mode: ReplicaFileMode,\n  replicaOptions: ReplicaOptions,\n) {\n  lc.info?.(`setting up ${mode} replica`);\n\n  switch (mode) {\n    case 'backup':\n      return prepare(lc, replicaOptions, 'wal', mode);\n\n    case 'serving-copy': {\n      // In 'serving-copy' mode, the original file is being used for 'backup'\n      // mode, so we make a copy for servicing sync requests.\n      const {file} = replicaOptions;\n      const copyLocation = replicaFileName(file, mode);\n      deleteLiteDB(copyLocation);\n\n      const start = Date.now();\n      lc.info?.(`copying ${file} to ${copyLocation}`);\n      const replica = new Database(lc, file);\n      replica.prepare(`VACUUM INTO ?`).run(copyLocation);\n      replica.close();\n      lc.info?.(`finished copy (${Date.now() - start} ms)`);\n\n      return prepare(lc, {...replicaOptions, file: copyLocation}, 'wal2', mode);\n    }\n\n    case 'serving':\n      return prepare(lc, replicaOptions, 'wal2', mode);\n\n    default:\n      throw new Error(`Invalid ReplicaMode ${mode}`);\n  }\n}\n\nexport function setUpMessageHandlers(\n  lc: LogContext,\n  replicator: Replicator,\n  parent: Worker,\n) {\n  handleSubscriptionsFrom(lc, parent, replicator);\n}\n\ntype Notification = ['notify', ReplicaState];\n\nexport function handleSubscriptionsFrom(\n  lc: LogContext,\n  subscriber: Worker,\n  notifier: ReplicaStateNotifier,\n) {\n  subscriber.onMessageType('subscribe', async () => {\n    const subscription = notifier.subscribe();\n\n    subscriber.on('close', () => {\n      lc.debug?.(`closing replication subscription from ${subscriber.pid}`);\n      subscription.cancel();\n    });\n\n    for await (const msg of subscription) {\n      try {\n        subscriber.send<Notification>(['notify', msg]);\n      } catch (e) {\n        const log =\n          e instanceof Error &&\n          'code' in e &&\n          // This can happen in a race condition if the subscribing process\n          // is closed before the 'close' message is processed.\n          e.code === 'ERR_IPC_CHANNEL_CLOSED'\n            ? 'warn'\n            : 'error';\n\n        lc[log]?.(\n          `error sending replicator notification to ${subscriber.pid}: ${String(e)}`,\n          e,\n        );\n      }\n    }\n  });\n}\n\n/**\n * Creates a Notifier to relay notifications the notifier of another Worker.\n * This does not send the initial subscription message. Use {@link subscribeTo}\n * to initiate the subscription.\n */\nexport function createNotifierFrom(\n  _lc: LogContext,\n  source: Worker,\n  onNotify?: (state: ReplicaState) => void,\n): Notifier {\n  const notifier = new Notifier();\n  source.onMessageType<Notification>('notify', msg => {\n    onNotify?.(msg);\n    void notifier.notifySubscribers(msg);\n  });\n  return notifier;\n}\n\nexport function subscribeTo(_lc: LogContext, source: Worker) {\n  source.send(['subscribe', {}]);\n}\n"],"mappings":";;;;;;;;;;;AAyBA,IAAa,wBAAwBA,aACnC,WACA,gBACA,QACF;AAMA,SAAgB,gBAAgB,aAAqB,MAAuB;CAC1E,OAAO,SAAS,iBAAiB,GAAG,YAAY,iBAAiB;AACnE;;;;;;;;;;;;;;;;;AAkBA,SAAgB,gCACd,qBACS;CACT,OAAO,wBAAwB;AACjC;;;;;;;;AASA,SAAgB,qBACd,qBACA,QACS;CACT,IAAI,CAAC,gCAAgC,mBAAmB,GACtD,OAAO;CAET,kBAAkB,MAAM;CACxB,OAAO;AACT;AAEA,IAAM,kBAAkB;AACxB,IAAM,KAAK;AAEX,eAAe,QACb,IACA,EAAC,MAAM,uBACP,SACA,MAC6D;CAG7D,MAAM,eAAe,IAAI,GAAG,KAAK,WAAW,IAAI;CAEhD,MAAM,UAAU,IAAI,SAAS,IAAI,IAAI;CACrC,IAAI;CACJ,IAAI;EAMF,MAAM,CAAC,EAAC,cAAc,iBAAgB,QAAQ,OAE3C,cAAc;EACjB,IAAI,gBAAgB,SAAS;GAC3B,QAAQ,WAAW,KAAK;GACxB,IAAI;IACF,MAAM,eAAe,IAAI,SAAS,QAAQ;GAC5C,UAAU;IACR,QAAQ,WAAW,IAAI;GACzB;EACF;EAEA,CAAC,CAAC,WAAW,aAAa,QAAQ,OAA4B,WAAW;EACzE,MAAM,CAAC,EAAC,YAAY,eAAc,QAAQ,OACxC,YACF;EACA,MAAM,CAAC,EAAC,gBAAgB,mBAAkB,QAAQ,OAE/C,gBAAgB;EAEnB,MAAM,UAAW,YAAY,WAAY,GAAA,CAAI,QAAQ,CAAC;EACtD,MAAM,gBAAiB,gBAAgB,WAAY,GAAA,CAAI,QAAQ,CAAC;EAGhE,GAAG,OAAO,cAAc,KAAK,IAAI,OAAO,OAAO,aAAa,cAAc;EAG1E,MAAM,SAAS,mBAAmB,OAAO;EACzC,GAAG,QAAQ,yBAAyB,QAAQ,EAAC,OAAM,CAAC;EACpD,IAAI,wBAAwB,KAAA,GAExB;QAAA,KAAK,IAAI,KAAK,OAAO,GAAG,EAAE,CAAC,EAAE,UAAU,QAAQ,KAAK,MAC3B,kBAAkB,qBAAqB;IAChE,GAAG,OAAO,qCAAqC,MAAM;IACrD,MAAM,KAAK,YAAY,IAAI;IAC3B,QAAQ,WAAW,IAAI;IAIvB,QAAQ,WAAW,KAAK;IACxB,IAAI;KACF,QAAQ,OAAO,oBAAoB;KACnC,QAAQ,KAAK,QAAQ;KACrB,YAAY,SAAS,QAAQ;KAC7B,MAAM,KAAK,YAAY,IAAI;KAC3B,GAAG,OAAO,qBAAqB,KAAK,GAAG,KAAK;IAC9C,SAAS,GAAG;KAGV,IAAI,EAAE,aAAa,eAAe,EAAE,SAAS,gBAC3C,MAAM;KAER,GAAG,OAAO,sBAAsB,KAAK,2BAA2B,CAAC;IACnE,UAAU;KACR,QAAQ,WAAW,IAAI;KACvB,QAAQ,WAAW,KAAK;IAC1B;GACF;;EAGF,MAAM,eAAe,IAAI,SAAS,OAAO;EAEzC,MAAM,UAAU,gBAAgB;EAChC,aAAa,SAAS,OAAO;EAE7B,QAAQ,OAAO,oBAAoB;EACnC,GAAG,OAAO,aAAa,MAAM;CAC/B,UAAU;EACR,QAAQ,MAAM;CAChB;CACA,OAAO;EAAC;EAAM;EAAS;CAAQ;AACjC;AAMA,eAAe,eACb,IACA,SACA,MACA;CACA,GAAG,OAAO,WAAW,QAAQ,KAAK,MAAM,KAAK,MAAM;CACnD,IAAI;CACJ,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK;EAC1B,IAAI;GACF,QAAQ,OAAO,kBAAkB,MAAM;GACvC;EACF,SAAS,GAAG;GACV,GAAG,OAAO,iCAAiC,KAAK,YAAY,IAAI,EAAE,IAAI,CAAC;GACvE,MAAM;EACR;EACA,MAAM,MAAM,GAAG;CACjB;CACA,MAAM;AACR;;;;;;;;;;;AAYA,SAAgB,kBAAgC;CAC9C,OAAO;EACL,aAAa;EACb,eAAe;EACf,mBAAmB;CACrB;AACF;AAEA,SAAgB,aACd,IACA,MACA,gBACA;CACA,GAAG,OAAO,cAAc,KAAK,SAAS;CAEtC,QAAQ,MAAR;EACE,KAAK,UACH,OAAO,QAAQ,IAAI,gBAAgB,OAAO,IAAI;EAEhD,KAAK,gBAAgB;GAGnB,MAAM,EAAC,SAAQ;GACf,MAAM,eAAe,gBAAgB,MAAM,IAAI;GAC/C,aAAa,YAAY;GAEzB,MAAM,QAAQ,KAAK,IAAI;GACvB,GAAG,OAAO,WAAW,KAAK,MAAM,cAAc;GAC9C,MAAM,UAAU,IAAI,SAAS,IAAI,IAAI;GACrC,QAAQ,QAAQ,eAAe,CAAC,CAAC,IAAI,YAAY;GACjD,QAAQ,MAAM;GACd,GAAG,OAAO,kBAAkB,KAAK,IAAI,IAAI,MAAM,KAAK;GAEpD,OAAO,QAAQ,IAAI;IAAC,GAAG;IAAgB,MAAM;GAAY,GAAG,QAAQ,IAAI;EAC1E;EAEA,KAAK,WACH,OAAO,QAAQ,IAAI,gBAAgB,QAAQ,IAAI;EAEjD,SACE,MAAM,IAAI,MAAM,uBAAuB,MAAM;CACjD;AACF;AAEA,SAAgB,qBACd,IACA,YACA,QACA;CACA,wBAAwB,IAAI,QAAQ,UAAU;AAChD;AAIA,SAAgB,wBACd,IACA,YACA,UACA;CACA,WAAW,cAAc,aAAa,YAAY;EAChD,MAAM,eAAe,SAAS,UAAU;EAExC,WAAW,GAAG,eAAe;GAC3B,GAAG,QAAQ,yCAAyC,WAAW,KAAK;GACpE,aAAa,OAAO;EACtB,CAAC;EAED,WAAW,MAAM,OAAO,cACtB,IAAI;GACF,WAAW,KAAmB,CAAC,UAAU,GAAG,CAAC;EAC/C,SAAS,GAAG;GAUV,GARE,aAAa,SACb,UAAU,KAGV,EAAE,SAAS,2BACP,SACA,QAEC,GACL,4CAA4C,WAAW,IAAI,IAAI,OAAO,CAAC,KACvE,CACF;EACF;CAEJ,CAAC;AACH;;;;;;AAOA,SAAgB,mBACd,KACA,QACA,UACU;CACV,MAAM,WAAW,IAAI,SAAS;CAC9B,OAAO,cAA4B,WAAU,QAAO;EAClD,WAAW,GAAG;EACd,SAAc,kBAAkB,GAAG;CACrC,CAAC;CACD,OAAO;AACT;AAEA,SAAgB,YAAY,KAAiB,QAAgB;CAC3D,OAAO,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC;AAC/B"}

package/out/zero-cache/src/workers/syncer-ws-message-handler.js

@@ -4,6 +4,7 @@ import { ZeroCache } from "../../../zero-protocol/src/error-origin-enum.js";
 import { startAsyncSpan, startSpan } from "../../../otel/src/span.js";
 import { version } from "../../../otel/src/version.js";
 import { ProtocolErrorWithLevel } from "../types/error-with-level.js";
+import { clientGroupLogID } from "../types/client-group-log-id.js";
 import "../services/view-syncer/connection-context-manager.js";
 import "../services/view-syncer/view-syncer.js";
 import { Lock } from "@rocicorp/lock";
@@ -36,7 +37,7 @@ var SyncerWsMessageHandler = class {
 		this.#mutagen = mutagen;
 		this.#connContextManager = connContextManager;
 		this.#mutationLock = new Lock();
-		this.#lc = lc.withContext("connection").withContext("clientID", clientID).withContext("clientGroupID", clientGroupID).withContext("wsID", wsID);
+		this.#lc = lc.withContext("connection").withContext("clientID", clientID).withContext("clientGroupID", clientGroupLogID(clientGroupID)).withContext("wsID", wsID);
 		this.#clientGroupID = clientGroupID;
 		this.#pusher = pusher;
 		this.#connectionSelector = {
@@ -61,7 +62,7 @@ var SyncerWsMessageHandler = class {
 					type: "fatal",
 					error: {
 						kind: InvalidPush,
-						message: `clientGroupID in mutation "${clientGroupID}" does not match clientGroupID of connection "${this.#clientGroupID}`,
+						message: "clientGroupID in mutation does not match clientGroupID of connection",
 						origin: ZeroCache
 					}
 				}];

package/out/zero-cache/src/workers/syncer-ws-message-handler.js.map

@@ -1 +1 @@
-{"version":3,"file":"syncer-ws-message-handler.js","names":["ErrorKind.InvalidPush","ErrorOrigin.ZeroCache","ErrorKind.InvalidMessage"],"sources":["../../../../../zero-cache/src/workers/syncer-ws-message-handler.ts"],"sourcesContent":["import {ROOT_CONTEXT, context, propagation, trace} from '@opentelemetry/api';\nimport {Lock} from '@rocicorp/lock';\nimport type {LogContext} from '@rocicorp/logger';\nimport {startAsyncSpan, startSpan} from '../../../otel/src/span.ts';\nimport {version} from '../../../otel/src/version.ts';\nimport {assert, unreachable} from '../../../shared/src/asserts.ts';\nimport {ErrorKind} from '../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../zero-protocol/src/error-origin.ts';\nimport type {ErrorBody} from '../../../zero-protocol/src/error.ts';\nimport type {UpstreamWithUnparsedAnalyzeQuery} from '../../../zero-protocol/src/up.ts';\nimport type {Mutagen} from '../services/mutagen/mutagen.ts';\nimport type {Pusher} from '../services/mutagen/pusher.ts';\nimport {\n  type ConnectionContextManager,\n  type ConnectionSelector,\n} from '../services/view-syncer/connection-context-manager.ts';\nimport {type ViewSyncer} from '../services/view-syncer/view-syncer.ts';\nimport {ProtocolErrorWithLevel} from '../types/error-with-level.ts';\nimport type {ConnectParams} from './connect-params.ts';\nimport type {HandlerResult, MessageHandler} from './connection.ts';\n\nconst tracer = trace.getTracer('syncer-ws-server', version);\n\n/**\n * Wraps a function in an OTEL context extracted from a W3C traceparent header.\n * This enables distributed tracing from the client through zero-cache to\n * the user's API server.\n */\nfunction withTraceparent<T>(traceparent: string | undefined, fn: () => T): T {\n  if (!traceparent) {\n    return fn();\n  }\n  const extracted = propagation.extract(ROOT_CONTEXT, {traceparent});\n  return context.with(extracted, fn);\n}\n\nexport class SyncerWsMessageHandler implements MessageHandler {\n  readonly #viewSyncer: ViewSyncer;\n  readonly #mutagen: Mutagen | undefined;\n  readonly #mutationLock: Lock;\n  readonly #lc: LogContext;\n  readonly #clientGroupID: string;\n  readonly #connectionSelector: ConnectionSelector;\n  readonly #connContextManager: ConnectionContextManager;\n  readonly #pusher: Pusher | undefined;\n  #initialized = false;\n\n  constructor(\n    lc: LogContext,\n    connectParams: ConnectParams,\n    connContextManager: ConnectionContextManager,\n    viewSyncer: ViewSyncer,\n    mutagen: Mutagen | undefined,\n    pusher: Pusher | undefined,\n  ) {\n    const {clientGroupID, clientID, wsID} = connectParams;\n    this.#viewSyncer = viewSyncer;\n    this.#mutagen = mutagen;\n    this.#connContextManager = connContextManager;\n    this.#mutationLock = new Lock();\n    this.#lc = lc\n      .withContext('connection')\n      .withContext('clientID', clientID)\n      .withContext('clientGroupID', clientGroupID)\n      .withContext('wsID', wsID);\n    this.#clientGroupID = clientGroupID;\n    this.#pusher = pusher;\n    this.#connectionSelector = {\n      clientID,\n      wsID,\n    };\n  }\n\n  async handleMessage(\n    msg: UpstreamWithUnparsedAnalyzeQuery,\n  ): Promise<HandlerResult[]> {\n    const lc = this.#lc;\n    const msgType = msg[0];\n    const viewSyncer = this.#viewSyncer;\n    switch (msgType) {\n      case 'ping':\n        lc.error?.('Ping is not supported at this layer by Zero');\n        break;\n      case 'pull':\n        lc.error?.('Pull is not supported by Zero');\n        break;\n      case 'push': {\n        return withTraceparent(msg[1].traceparent, () =>\n          startAsyncSpan<HandlerResult[]>(\n            tracer,\n            'connection.push',\n            async () => {\n              const {clientGroupID, mutations} = msg[1];\n              if (clientGroupID !== this.#clientGroupID) {\n                return [\n                  {\n                    type: 'fatal',\n                    error: {\n                      kind: ErrorKind.InvalidPush,\n                      message:\n                        `clientGroupID in mutation \"${clientGroupID}\" does not match ` +\n                        `clientGroupID of connection \"${this.#clientGroupID}`,\n                      origin: ErrorOrigin.ZeroCache,\n                    },\n                  } satisfies HandlerResult,\n                ];\n              }\n\n              if (mutations.length === 0) {\n                return [\n                  {\n                    type: 'ok',\n                  },\n                ];\n              }\n\n              // The client only ever sends 1 mutation per push.\n              // #pusher will throw if it sees a CRUD mutation.\n              // #mutagen will throw if it see a custom mutation.\n              if (mutations[0].type === 'custom') {\n                if (!this.#pusher) {\n                  return [\n                    {\n                      type: 'fatal',\n                      error: {\n                        kind: ErrorKind.InvalidPush,\n                        message:\n                          'A ZERO_MUTATE_URL must be set in order to process custom mutations.',\n                        origin: ErrorOrigin.ZeroCache,\n                      },\n                    } satisfies HandlerResult,\n                  ];\n                }\n                return [\n                  this.#pusher.enqueuePush(this.#connectionSelector, msg[1]),\n                ];\n              }\n\n              const mutagen = this.#mutagen;\n              if (!mutagen) {\n                return [\n                  {\n                    type: 'fatal',\n                    error: {\n                      kind: ErrorKind.InvalidPush,\n                      message: `Support for legacy CRUD mutations is disabled`,\n                      origin: ErrorOrigin.ZeroCache,\n                    },\n                  } satisfies HandlerResult,\n                ];\n              }\n\n              const auth = this.#connContextManager.mustGetConnectionContext(\n                this.#connectionSelector,\n              ).auth;\n              assert(\n                auth?.type !== 'opaque',\n                'Only JWT auth is supported for CRUD mutations',\n              );\n\n              // Hold a ref on the mutagen for the duration of the push so that\n              // the connection closing mid-push does not stop the service\n              // (closing its replica handle) between two of its mutations.\n              // (The connection holds a ref while open, and messages are not\n              // dispatched after it has closed, so the service is running\n              // here.)\n              mutagen.ref();\n              try {\n                // Hold a connection-level lock while processing mutations so that:\n                // 1. Mutations are processed in the order in which they are received and\n                // 2. A single view syncer connection cannot hog multiple upstream connections.\n                const ret = await this.#mutationLock.withLock(async () => {\n                  const errors: ErrorBody[] = [];\n                  for (const mutation of mutations) {\n                    const maybeError = await mutagen.processMutation(\n                      mutation,\n                      auth?.decoded,\n                      this.#pusher !== undefined,\n                    );\n                    if (maybeError !== undefined) {\n                      errors.push({\n                        kind: maybeError[0],\n                        message: maybeError[1],\n                        origin: ErrorOrigin.ZeroCache,\n                      });\n                    }\n                  }\n                  if (errors.length > 0) {\n                    return {type: 'transient', errors} satisfies HandlerResult;\n                  }\n                  return {type: 'ok'} satisfies HandlerResult;\n                });\n                return [ret];\n              } finally {\n                mutagen.unref();\n              }\n            },\n          ),\n        );\n      }\n      case 'changeDesiredQueries':\n        await withTraceparent(msg[1].traceparent, () =>\n          startAsyncSpan(tracer, 'connection.changeDesiredQueries', () =>\n            viewSyncer.changeDesiredQueries(this.#connectionSelector, msg),\n          ),\n        );\n        break;\n      case 'updateAuth':\n        await startAsyncSpan(tracer, 'connection.updateAuth', async () => {\n          const initialConnCtx =\n            this.#connContextManager.mustGetConnectionContext(\n              this.#connectionSelector,\n            );\n          const updatedConnCtx = await this.#connContextManager.updateAuth(\n            this.#connectionSelector,\n            msg[1],\n          );\n          const authRevisionChanged =\n            updatedConnCtx.revision !== initialConnCtx.revision;\n\n          await viewSyncer.updateAuth(\n            this.#connectionSelector,\n            msg,\n            authRevisionChanged,\n          );\n        });\n        break;\n      case 'deleteClients': {\n        const deletedClientIDs = await startAsyncSpan(\n          tracer,\n          'connection.deleteClients',\n          () => viewSyncer.deleteClients(this.#connectionSelector, msg),\n        );\n        if (this.#pusher && deletedClientIDs.length > 0) {\n          await this.#pusher.deleteClientMutations(\n            this.#connectionSelector,\n            deletedClientIDs,\n          );\n        }\n        break;\n      }\n      case 'initConnection': {\n        // A socket is initialized once, from the sec-protocol header or from\n        // its first initConnection message. A second one is a client bug, so\n        // reject it before it replaces this socket's view-syncer and pusher\n        // streams.\n        if (this.#initialized) {\n          throw new ProtocolErrorWithLevel(\n            {\n              kind: ErrorKind.InvalidMessage,\n              message: 'Connection was already initialized',\n              origin: ErrorOrigin.ZeroCache,\n            },\n            'warn',\n          );\n        }\n        this.#initialized = true;\n        this.#connContextManager.initConnection(\n          this.#connectionSelector,\n          msg[1],\n        );\n        return withTraceparent(msg[1].traceparent, () => {\n          const ret: HandlerResult[] = [\n            {\n              type: 'stream',\n              source: 'viewSyncer',\n              stream: startSpan(tracer, 'connection.initConnection', () =>\n                viewSyncer.initConnection(this.#connectionSelector, msg),\n              ),\n            },\n          ];\n\n          // Given we support both CRUD and Custom mutators,\n          // we do not initialize the `pusher` unless the user has opted\n          // into custom mutations. We detect that by checking\n          // if the pushURL has been set.\n          if (this.#pusher) {\n            ret.push({\n              type: 'stream',\n              source: 'pusher',\n              stream: this.#pusher.initConnection(this.#connectionSelector),\n            });\n          }\n\n          return ret;\n        });\n      }\n      case 'closeConnection':\n        // This message is deprecated and no longer used.\n        break;\n\n      case 'inspect':\n        await startAsyncSpan(tracer, 'connection.inspect', () =>\n          viewSyncer.inspect(this.#connectionSelector, msg),\n        );\n        break;\n\n      case 'ackMutationResponses':\n        if (this.#pusher) {\n          await this.#pusher.ackMutationResponses(\n            this.#connectionSelector,\n            msg[1],\n          );\n        }\n        break;\n\n      default:\n        unreachable(msgType);\n    }\n\n    return [{type: 'ok'}];\n  }\n}\n"],"mappings":";;;;;;;;;;;AAqBA,IAAM,SAAS,MAAM,UAAU,oBAAoB,OAAO;;;;;;AAO1D,SAAS,gBAAmB,aAAiC,IAAgB;CAC3E,IAAI,CAAC,aACH,OAAO,GAAG;CAEZ,MAAM,YAAY,YAAY,QAAQ,cAAc,EAAC,YAAW,CAAC;CACjE,OAAO,QAAQ,KAAK,WAAW,EAAE;AACnC;AAEA,IAAa,yBAAb,MAA8D;CAC5D;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA,eAAe;CAEf,YACE,IACA,eACA,oBACA,YACA,SACA,QACA;EACA,MAAM,EAAC,eAAe,UAAU,SAAQ;EACxC,KAAK,cAAc;EACnB,KAAK,WAAW;EAChB,KAAK,sBAAsB;EAC3B,KAAK,gBAAgB,IAAI,KAAK;EAC9B,KAAK,MAAM,GACR,YAAY,YAAY,CAAC,CACzB,YAAY,YAAY,QAAQ,CAAC,CACjC,YAAY,iBAAiB,aAAa,CAAC,CAC3C,YAAY,QAAQ,IAAI;EAC3B,KAAK,iBAAiB;EACtB,KAAK,UAAU;EACf,KAAK,sBAAsB;GACzB;GACA;EACF;CACF;CAEA,MAAM,cACJ,KAC0B;EAC1B,MAAM,KAAK,KAAK;EAChB,MAAM,UAAU,IAAI;EACpB,MAAM,aAAa,KAAK;EACxB,QAAQ,SAAR;GACE,KAAK;IACH,GAAG,QAAQ,6CAA6C;IACxD;GACF,KAAK;IACH,GAAG,QAAQ,+BAA+B;IAC1C;GACF,KAAK,QACH,OAAO,gBAAgB,IAAI,EAAE,CAAC,mBAC5B,eACE,QACA,mBACA,YAAY;IACV,MAAM,EAAC,eAAe,cAAa,IAAI;IACvC,IAAI,kBAAkB,KAAK,gBACzB,OAAO,CACL;KACE,MAAM;KACN,OAAO;MACL,MAAMA;MACN,SACE,8BAA8B,cAAc,gDACZ,KAAK;MACvC,QAAQC;KACV;IACF,CACF;IAGF,IAAI,UAAU,WAAW,GACvB,OAAO,CACL,EACE,MAAM,KACR,CACF;IAMF,IAAI,UAAU,EAAE,CAAC,SAAS,UAAU;KAClC,IAAI,CAAC,KAAK,SACR,OAAO,CACL;MACE,MAAM;MACN,OAAO;OACL,MAAMD;OACN,SACE;OACF,QAAQC;MACV;KACF,CACF;KAEF,OAAO,CACL,KAAK,QAAQ,YAAY,KAAK,qBAAqB,IAAI,EAAE,CAC3D;IACF;IAEA,MAAM,UAAU,KAAK;IACrB,IAAI,CAAC,SACH,OAAO,CACL;KACE,MAAM;KACN,OAAO;MACL,MAAMD;MACN,SAAS;MACT,QAAQC;KACV;IACF,CACF;IAGF,MAAM,OAAO,KAAK,oBAAoB,yBACpC,KAAK,mBACP,CAAC,CAAC;IACF,OACE,MAAM,SAAS,UACf,+CACF;IAQA,QAAQ,IAAI;IACZ,IAAI;KAyBF,OAAO,CAAC,MArBU,KAAK,cAAc,SAAS,YAAY;MACxD,MAAM,SAAsB,CAAC;MAC7B,KAAK,MAAM,YAAY,WAAW;OAChC,MAAM,aAAa,MAAM,QAAQ,gBAC/B,UACA,MAAM,SACN,KAAK,YAAY,KAAA,CACnB;OACA,IAAI,eAAe,KAAA,GACjB,OAAO,KAAK;QACV,MAAM,WAAW;QACjB,SAAS,WAAW;QACpB,QAAQA;OACV,CAAC;MAEL;MACA,IAAI,OAAO,SAAS,GAClB,OAAO;OAAC,MAAM;OAAa;MAAM;MAEnC,OAAO,EAAC,MAAM,KAAI;KACpB,CAAC,CACU;IACb,UAAU;KACR,QAAQ,MAAM;IAChB;GACF,CACF,CACF;GAEF,KAAK;IACH,MAAM,gBAAgB,IAAI,EAAE,CAAC,mBAC3B,eAAe,QAAQ,yCACrB,WAAW,qBAAqB,KAAK,qBAAqB,GAAG,CAC/D,CACF;IACA;GACF,KAAK;IACH,MAAM,eAAe,QAAQ,yBAAyB,YAAY;KAChE,MAAM,iBACJ,KAAK,oBAAoB,yBACvB,KAAK,mBACP;KAKF,MAAM,uBACJ,MAL2B,KAAK,oBAAoB,WACpD,KAAK,qBACL,IAAI,EACN,EAAA,CAEiB,aAAa,eAAe;KAE7C,MAAM,WAAW,WACf,KAAK,qBACL,KACA,mBACF;IACF,CAAC;IACD;GACF,KAAK,iBAAiB;IACpB,MAAM,mBAAmB,MAAM,eAC7B,QACA,kCACM,WAAW,cAAc,KAAK,qBAAqB,GAAG,CAC9D;IACA,IAAI,KAAK,WAAW,iBAAiB,SAAS,GAC5C,MAAM,KAAK,QAAQ,sBACjB,KAAK,qBACL,gBACF;IAEF;GACF;GACA,KAAK;IAKH,IAAI,KAAK,cACP,MAAM,IAAI,uBACR;KACE,MAAMC;KACN,SAAS;KACT,QAAQD;IACV,GACA,MACF;IAEF,KAAK,eAAe;IACpB,KAAK,oBAAoB,eACvB,KAAK,qBACL,IAAI,EACN;IACA,OAAO,gBAAgB,IAAI,EAAE,CAAC,mBAAmB;KAC/C,MAAM,MAAuB,CAC3B;MACE,MAAM;MACN,QAAQ;MACR,QAAQ,UAAU,QAAQ,mCACxB,WAAW,eAAe,KAAK,qBAAqB,GAAG,CACzD;KACF,CACF;KAMA,IAAI,KAAK,SACP,IAAI,KAAK;MACP,MAAM;MACN,QAAQ;MACR,QAAQ,KAAK,QAAQ,eAAe,KAAK,mBAAmB;KAC9D,CAAC;KAGH,OAAO;IACT,CAAC;GAEH,KAAK,mBAEH;GAEF,KAAK;IACH,MAAM,eAAe,QAAQ,4BAC3B,WAAW,QAAQ,KAAK,qBAAqB,GAAG,CAClD;IACA;GAEF,KAAK;IACH,IAAI,KAAK,SACP,MAAM,KAAK,QAAQ,qBACjB,KAAK,qBACL,IAAI,EACN;IAEF;GAEF,SACE,YAAY,OAAO;EACvB;EAEA,OAAO,CAAC,EAAC,MAAM,KAAI,CAAC;CACtB;AACF"}
+{"version":3,"file":"syncer-ws-message-handler.js","names":["ErrorKind.InvalidPush","ErrorOrigin.ZeroCache","ErrorKind.InvalidMessage"],"sources":["../../../../../zero-cache/src/workers/syncer-ws-message-handler.ts"],"sourcesContent":["import {ROOT_CONTEXT, context, propagation, trace} from '@opentelemetry/api';\nimport {Lock} from '@rocicorp/lock';\nimport type {LogContext} from '@rocicorp/logger';\nimport {startAsyncSpan, startSpan} from '../../../otel/src/span.ts';\nimport {version} from '../../../otel/src/version.ts';\nimport {assert, unreachable} from '../../../shared/src/asserts.ts';\nimport {ErrorKind} from '../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../zero-protocol/src/error-origin.ts';\nimport type {ErrorBody} from '../../../zero-protocol/src/error.ts';\nimport type {UpstreamWithUnparsedAnalyzeQuery} from '../../../zero-protocol/src/up.ts';\nimport type {Mutagen} from '../services/mutagen/mutagen.ts';\nimport type {Pusher} from '../services/mutagen/pusher.ts';\nimport {\n  type ConnectionContextManager,\n  type ConnectionSelector,\n} from '../services/view-syncer/connection-context-manager.ts';\nimport {type ViewSyncer} from '../services/view-syncer/view-syncer.ts';\nimport {clientGroupLogID} from '../types/client-group-log-id.ts';\nimport {ProtocolErrorWithLevel} from '../types/error-with-level.ts';\nimport type {ConnectParams} from './connect-params.ts';\nimport type {HandlerResult, MessageHandler} from './connection.ts';\n\nconst tracer = trace.getTracer('syncer-ws-server', version);\n\n/**\n * Wraps a function in an OTEL context extracted from a W3C traceparent header.\n * This enables distributed tracing from the client through zero-cache to\n * the user's API server.\n */\nfunction withTraceparent<T>(traceparent: string | undefined, fn: () => T): T {\n  if (!traceparent) {\n    return fn();\n  }\n  const extracted = propagation.extract(ROOT_CONTEXT, {traceparent});\n  return context.with(extracted, fn);\n}\n\nexport class SyncerWsMessageHandler implements MessageHandler {\n  readonly #viewSyncer: ViewSyncer;\n  readonly #mutagen: Mutagen | undefined;\n  readonly #mutationLock: Lock;\n  readonly #lc: LogContext;\n  readonly #clientGroupID: string;\n  readonly #connectionSelector: ConnectionSelector;\n  readonly #connContextManager: ConnectionContextManager;\n  readonly #pusher: Pusher | undefined;\n  #initialized = false;\n\n  constructor(\n    lc: LogContext,\n    connectParams: ConnectParams,\n    connContextManager: ConnectionContextManager,\n    viewSyncer: ViewSyncer,\n    mutagen: Mutagen | undefined,\n    pusher: Pusher | undefined,\n  ) {\n    const {clientGroupID, clientID, wsID} = connectParams;\n    this.#viewSyncer = viewSyncer;\n    this.#mutagen = mutagen;\n    this.#connContextManager = connContextManager;\n    this.#mutationLock = new Lock();\n    this.#lc = lc\n      .withContext('connection')\n      .withContext('clientID', clientID)\n      .withContext('clientGroupID', clientGroupLogID(clientGroupID))\n      .withContext('wsID', wsID);\n    this.#clientGroupID = clientGroupID;\n    this.#pusher = pusher;\n    this.#connectionSelector = {\n      clientID,\n      wsID,\n    };\n  }\n\n  async handleMessage(\n    msg: UpstreamWithUnparsedAnalyzeQuery,\n  ): Promise<HandlerResult[]> {\n    const lc = this.#lc;\n    const msgType = msg[0];\n    const viewSyncer = this.#viewSyncer;\n    switch (msgType) {\n      case 'ping':\n        lc.error?.('Ping is not supported at this layer by Zero');\n        break;\n      case 'pull':\n        lc.error?.('Pull is not supported by Zero');\n        break;\n      case 'push': {\n        return withTraceparent(msg[1].traceparent, () =>\n          startAsyncSpan<HandlerResult[]>(\n            tracer,\n            'connection.push',\n            async () => {\n              const {clientGroupID, mutations} = msg[1];\n              if (clientGroupID !== this.#clientGroupID) {\n                return [\n                  {\n                    type: 'fatal',\n                    error: {\n                      kind: ErrorKind.InvalidPush,\n                      // Neither ID is included: the message is also logged\n                      // as the reason the connection is closed.\n                      message:\n                        'clientGroupID in mutation does not match clientGroupID of connection',\n                      origin: ErrorOrigin.ZeroCache,\n                    },\n                  } satisfies HandlerResult,\n                ];\n              }\n\n              if (mutations.length === 0) {\n                return [\n                  {\n                    type: 'ok',\n                  },\n                ];\n              }\n\n              // The client only ever sends 1 mutation per push.\n              // #pusher will throw if it sees a CRUD mutation.\n              // #mutagen will throw if it see a custom mutation.\n              if (mutations[0].type === 'custom') {\n                if (!this.#pusher) {\n                  return [\n                    {\n                      type: 'fatal',\n                      error: {\n                        kind: ErrorKind.InvalidPush,\n                        message:\n                          'A ZERO_MUTATE_URL must be set in order to process custom mutations.',\n                        origin: ErrorOrigin.ZeroCache,\n                      },\n                    } satisfies HandlerResult,\n                  ];\n                }\n                return [\n                  this.#pusher.enqueuePush(this.#connectionSelector, msg[1]),\n                ];\n              }\n\n              const mutagen = this.#mutagen;\n              if (!mutagen) {\n                return [\n                  {\n                    type: 'fatal',\n                    error: {\n                      kind: ErrorKind.InvalidPush,\n                      message: `Support for legacy CRUD mutations is disabled`,\n                      origin: ErrorOrigin.ZeroCache,\n                    },\n                  } satisfies HandlerResult,\n                ];\n              }\n\n              const auth = this.#connContextManager.mustGetConnectionContext(\n                this.#connectionSelector,\n              ).auth;\n              assert(\n                auth?.type !== 'opaque',\n                'Only JWT auth is supported for CRUD mutations',\n              );\n\n              // Hold a ref on the mutagen for the duration of the push so that\n              // the connection closing mid-push does not stop the service\n              // (closing its replica handle) between two of its mutations.\n              // (The connection holds a ref while open, and messages are not\n              // dispatched after it has closed, so the service is running\n              // here.)\n              mutagen.ref();\n              try {\n                // Hold a connection-level lock while processing mutations so that:\n                // 1. Mutations are processed in the order in which they are received and\n                // 2. A single view syncer connection cannot hog multiple upstream connections.\n                const ret = await this.#mutationLock.withLock(async () => {\n                  const errors: ErrorBody[] = [];\n                  for (const mutation of mutations) {\n                    const maybeError = await mutagen.processMutation(\n                      mutation,\n                      auth?.decoded,\n                      this.#pusher !== undefined,\n                    );\n                    if (maybeError !== undefined) {\n                      errors.push({\n                        kind: maybeError[0],\n                        message: maybeError[1],\n                        origin: ErrorOrigin.ZeroCache,\n                      });\n                    }\n                  }\n                  if (errors.length > 0) {\n                    return {type: 'transient', errors} satisfies HandlerResult;\n                  }\n                  return {type: 'ok'} satisfies HandlerResult;\n                });\n                return [ret];\n              } finally {\n                mutagen.unref();\n              }\n            },\n          ),\n        );\n      }\n      case 'changeDesiredQueries':\n        await withTraceparent(msg[1].traceparent, () =>\n          startAsyncSpan(tracer, 'connection.changeDesiredQueries', () =>\n            viewSyncer.changeDesiredQueries(this.#connectionSelector, msg),\n          ),\n        );\n        break;\n      case 'updateAuth':\n        await startAsyncSpan(tracer, 'connection.updateAuth', async () => {\n          const initialConnCtx =\n            this.#connContextManager.mustGetConnectionContext(\n              this.#connectionSelector,\n            );\n          const updatedConnCtx = await this.#connContextManager.updateAuth(\n            this.#connectionSelector,\n            msg[1],\n          );\n          const authRevisionChanged =\n            updatedConnCtx.revision !== initialConnCtx.revision;\n\n          await viewSyncer.updateAuth(\n            this.#connectionSelector,\n            msg,\n            authRevisionChanged,\n          );\n        });\n        break;\n      case 'deleteClients': {\n        const deletedClientIDs = await startAsyncSpan(\n          tracer,\n          'connection.deleteClients',\n          () => viewSyncer.deleteClients(this.#connectionSelector, msg),\n        );\n        if (this.#pusher && deletedClientIDs.length > 0) {\n          await this.#pusher.deleteClientMutations(\n            this.#connectionSelector,\n            deletedClientIDs,\n          );\n        }\n        break;\n      }\n      case 'initConnection': {\n        // A socket is initialized once, from the sec-protocol header or from\n        // its first initConnection message. A second one is a client bug, so\n        // reject it before it replaces this socket's view-syncer and pusher\n        // streams.\n        if (this.#initialized) {\n          throw new ProtocolErrorWithLevel(\n            {\n              kind: ErrorKind.InvalidMessage,\n              message: 'Connection was already initialized',\n              origin: ErrorOrigin.ZeroCache,\n            },\n            'warn',\n          );\n        }\n        this.#initialized = true;\n        this.#connContextManager.initConnection(\n          this.#connectionSelector,\n          msg[1],\n        );\n        return withTraceparent(msg[1].traceparent, () => {\n          const ret: HandlerResult[] = [\n            {\n              type: 'stream',\n              source: 'viewSyncer',\n              stream: startSpan(tracer, 'connection.initConnection', () =>\n                viewSyncer.initConnection(this.#connectionSelector, msg),\n              ),\n            },\n          ];\n\n          // Given we support both CRUD and Custom mutators,\n          // we do not initialize the `pusher` unless the user has opted\n          // into custom mutations. We detect that by checking\n          // if the pushURL has been set.\n          if (this.#pusher) {\n            ret.push({\n              type: 'stream',\n              source: 'pusher',\n              stream: this.#pusher.initConnection(this.#connectionSelector),\n            });\n          }\n\n          return ret;\n        });\n      }\n      case 'closeConnection':\n        // This message is deprecated and no longer used.\n        break;\n\n      case 'inspect':\n        await startAsyncSpan(tracer, 'connection.inspect', () =>\n          viewSyncer.inspect(this.#connectionSelector, msg),\n        );\n        break;\n\n      case 'ackMutationResponses':\n        if (this.#pusher) {\n          await this.#pusher.ackMutationResponses(\n            this.#connectionSelector,\n            msg[1],\n          );\n        }\n        break;\n\n      default:\n        unreachable(msgType);\n    }\n\n    return [{type: 'ok'}];\n  }\n}\n"],"mappings":";;;;;;;;;;;;AAsBA,IAAM,SAAS,MAAM,UAAU,oBAAoB,OAAO;;;;;;AAO1D,SAAS,gBAAmB,aAAiC,IAAgB;CAC3E,IAAI,CAAC,aACH,OAAO,GAAG;CAEZ,MAAM,YAAY,YAAY,QAAQ,cAAc,EAAC,YAAW,CAAC;CACjE,OAAO,QAAQ,KAAK,WAAW,EAAE;AACnC;AAEA,IAAa,yBAAb,MAA8D;CAC5D;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA,eAAe;CAEf,YACE,IACA,eACA,oBACA,YACA,SACA,QACA;EACA,MAAM,EAAC,eAAe,UAAU,SAAQ;EACxC,KAAK,cAAc;EACnB,KAAK,WAAW;EAChB,KAAK,sBAAsB;EAC3B,KAAK,gBAAgB,IAAI,KAAK;EAC9B,KAAK,MAAM,GACR,YAAY,YAAY,CAAC,CACzB,YAAY,YAAY,QAAQ,CAAC,CACjC,YAAY,iBAAiB,iBAAiB,aAAa,CAAC,CAAC,CAC7D,YAAY,QAAQ,IAAI;EAC3B,KAAK,iBAAiB;EACtB,KAAK,UAAU;EACf,KAAK,sBAAsB;GACzB;GACA;EACF;CACF;CAEA,MAAM,cACJ,KAC0B;EAC1B,MAAM,KAAK,KAAK;EAChB,MAAM,UAAU,IAAI;EACpB,MAAM,aAAa,KAAK;EACxB,QAAQ,SAAR;GACE,KAAK;IACH,GAAG,QAAQ,6CAA6C;IACxD;GACF,KAAK;IACH,GAAG,QAAQ,+BAA+B;IAC1C;GACF,KAAK,QACH,OAAO,gBAAgB,IAAI,EAAE,CAAC,mBAC5B,eACE,QACA,mBACA,YAAY;IACV,MAAM,EAAC,eAAe,cAAa,IAAI;IACvC,IAAI,kBAAkB,KAAK,gBACzB,OAAO,CACL;KACE,MAAM;KACN,OAAO;MACL,MAAMA;MAGN,SACE;MACF,QAAQC;KACV;IACF,CACF;IAGF,IAAI,UAAU,WAAW,GACvB,OAAO,CACL,EACE,MAAM,KACR,CACF;IAMF,IAAI,UAAU,EAAE,CAAC,SAAS,UAAU;KAClC,IAAI,CAAC,KAAK,SACR,OAAO,CACL;MACE,MAAM;MACN,OAAO;OACL,MAAMD;OACN,SACE;OACF,QAAQC;MACV;KACF,CACF;KAEF,OAAO,CACL,KAAK,QAAQ,YAAY,KAAK,qBAAqB,IAAI,EAAE,CAC3D;IACF;IAEA,MAAM,UAAU,KAAK;IACrB,IAAI,CAAC,SACH,OAAO,CACL;KACE,MAAM;KACN,OAAO;MACL,MAAMD;MACN,SAAS;MACT,QAAQC;KACV;IACF,CACF;IAGF,MAAM,OAAO,KAAK,oBAAoB,yBACpC,KAAK,mBACP,CAAC,CAAC;IACF,OACE,MAAM,SAAS,UACf,+CACF;IAQA,QAAQ,IAAI;IACZ,IAAI;KAyBF,OAAO,CAAC,MArBU,KAAK,cAAc,SAAS,YAAY;MACxD,MAAM,SAAsB,CAAC;MAC7B,KAAK,MAAM,YAAY,WAAW;OAChC,MAAM,aAAa,MAAM,QAAQ,gBAC/B,UACA,MAAM,SACN,KAAK,YAAY,KAAA,CACnB;OACA,IAAI,eAAe,KAAA,GACjB,OAAO,KAAK;QACV,MAAM,WAAW;QACjB,SAAS,WAAW;QACpB,QAAQA;OACV,CAAC;MAEL;MACA,IAAI,OAAO,SAAS,GAClB,OAAO;OAAC,MAAM;OAAa;MAAM;MAEnC,OAAO,EAAC,MAAM,KAAI;KACpB,CAAC,CACU;IACb,UAAU;KACR,QAAQ,MAAM;IAChB;GACF,CACF,CACF;GAEF,KAAK;IACH,MAAM,gBAAgB,IAAI,EAAE,CAAC,mBAC3B,eAAe,QAAQ,yCACrB,WAAW,qBAAqB,KAAK,qBAAqB,GAAG,CAC/D,CACF;IACA;GACF,KAAK;IACH,MAAM,eAAe,QAAQ,yBAAyB,YAAY;KAChE,MAAM,iBACJ,KAAK,oBAAoB,yBACvB,KAAK,mBACP;KAKF,MAAM,uBACJ,MAL2B,KAAK,oBAAoB,WACpD,KAAK,qBACL,IAAI,EACN,EAAA,CAEiB,aAAa,eAAe;KAE7C,MAAM,WAAW,WACf,KAAK,qBACL,KACA,mBACF;IACF,CAAC;IACD;GACF,KAAK,iBAAiB;IACpB,MAAM,mBAAmB,MAAM,eAC7B,QACA,kCACM,WAAW,cAAc,KAAK,qBAAqB,GAAG,CAC9D;IACA,IAAI,KAAK,WAAW,iBAAiB,SAAS,GAC5C,MAAM,KAAK,QAAQ,sBACjB,KAAK,qBACL,gBACF;IAEF;GACF;GACA,KAAK;IAKH,IAAI,KAAK,cACP,MAAM,IAAI,uBACR;KACE,MAAMC;KACN,SAAS;KACT,QAAQD;IACV,GACA,MACF;IAEF,KAAK,eAAe;IACpB,KAAK,oBAAoB,eACvB,KAAK,qBACL,IAAI,EACN;IACA,OAAO,gBAAgB,IAAI,EAAE,CAAC,mBAAmB;KAC/C,MAAM,MAAuB,CAC3B;MACE,MAAM;MACN,QAAQ;MACR,QAAQ,UAAU,QAAQ,mCACxB,WAAW,eAAe,KAAK,qBAAqB,GAAG,CACzD;KACF,CACF;KAMA,IAAI,KAAK,SACP,IAAI,KAAK;MACP,MAAM;MACN,QAAQ;MACR,QAAQ,KAAK,QAAQ,eAAe,KAAK,mBAAmB;KAC9D,CAAC;KAGH,OAAO;IACT,CAAC;GAEH,KAAK,mBAEH;GAEF,KAAK;IACH,MAAM,eAAe,QAAQ,4BAC3B,WAAW,QAAQ,KAAK,qBAAqB,GAAG,CAClD;IACA;GAEF,KAAK;IACH,IAAI,KAAK,SACP,MAAM,KAAK,QAAQ,qBACjB,KAAK,qBACL,IAAI,EACN;IAEF;GAEF,SACE,YAAY,OAAO;EACvB;EAEA,OAAO,CAAC,EAAC,MAAM,KAAI,CAAC;CACtB;AACF"}

package/out/zero-cache/src/workers/syncer.js

@@ -6,6 +6,7 @@ import { ProtocolError, isProtocolError } from "../../../zero-protocol/src/error
 import "../config/zero-config.js";
 import { getOrCreateCounter, getOrCreateGauge, getOrCreateNativeHistogram, getOrCreateUpDownCounter } from "../observability/metrics.js";
 import { installWebSocketReceiver } from "../types/websocket-handoff.js";
+import { clientGroupLogID } from "../types/client-group-log-id.js";
 import { Connection, sendError } from "./connection.js";
 import { expiredAffinityCookieHeader, rehome } from "./rehome.js";
 import { createNotifierFrom, subscribeTo } from "./replicator.js";
@@ -341,27 +342,26 @@ var Syncer = class {
 		ws.once("close", () => {
 			this.#webSocketOpenConnections.add(-1, metricAttributes);
 		});
-		this.#lc.debug?.("creating connection", params.clientGroupID, params.clientID);
+		this.#lc.debug?.("creating connection", clientGroupLogID(params.clientGroupID), params.clientID);
 		recordConnectionAttempted();
 		const { clientID, clientGroupID, auth, userID } = params;
 		const hasProvidedAuth = auth !== void 0 && auth !== "";
 		const incomingUserID = userID ?? null;
-		if (hasProvidedAuth) {
-			const tokenOptions = tokenConfigOptions(this.#config.auth ?? {});
-			const hasPushOrMutate = this.#config?.push?.url !== void 0 || this.#config?.mutate?.url !== void 0;
-			const hasQueries = this.#config?.query?.url !== void 0 || this.#config?.getQueries?.url !== void 0;
-			if (!(tokenOptions.length === 1) && !(hasPushOrMutate && hasQueries)) {
-				recordConnectionFailure("configuration");
-				throw new Error("Exactly one of jwk, secret, or jwksUrl must be set in order to verify tokens but actually the following were set: " + JSON.stringify(tokenOptions) + ". You may also set both ZERO_MUTATE_URL and ZERO_QUERY_URL to enable custom mutations and queries without passing token verification options.");
-			}
+		const tokenOptions = tokenConfigOptions(this.#config.auth ?? {});
+		const hasPushOrMutate = this.#config?.push?.url !== void 0 || this.#config?.mutate?.url !== void 0;
+		const hasQueries = this.#config?.query?.url !== void 0 || this.#config?.getQueries?.url !== void 0;
+		const hasCustomEndpoints = hasPushOrMutate && hasQueries;
+		if (tokenOptions.length > 1 || hasProvidedAuth && tokenOptions.length === 0 && !hasCustomEndpoints) {
+			recordConnectionFailure("configuration");
+			throw new Error("Exactly one of jwk, secret, or jwksUrl must be set in order to verify tokens but actually the following were set: " + JSON.stringify(tokenOptions) + ". You may also set both ZERO_MUTATE_URL and ZERO_QUERY_URL to enable custom mutations and queries without passing token verification options.");
 		}
 		let initialAuth;
 		try {
-			initialAuth = await resolveAuth(this.#lc.withContext("clientGroupID", clientGroupID).withContext("clientID", clientID), void 0, incomingUserID, auth, this.#validateLegacyJWT);
+			initialAuth = await resolveAuth(this.#lc.withContext("clientGroupID", clientGroupLogID(clientGroupID)).withContext("clientID", clientID), void 0, incomingUserID, auth, this.#validateLegacyJWT);
 		} catch (e) {
 			if (isProtocolError(e)) {
 				this.#lc.warn?.("Rejecting sync connection during initial auth resolution", {
-					clientGroupID,
+					clientGroupID: clientGroupLogID(clientGroupID),
 					clientID,
 					incomingUserID,
 					hasProvidedAuth,
@@ -377,7 +377,7 @@ var Syncer = class {
 		}
 		if (ws.readyState !== WebSocket.OPEN) {
 			this.#lc.debug?.("websocket closed while resolving auth", {
-				clientGroupID,
+				clientGroupID: clientGroupLogID(clientGroupID),
 				clientID,
 				readyState: ws.readyState
 			});
@@ -405,7 +405,7 @@ var Syncer = class {
 		}
 		const existing = this.#connections.get(clientID);
 		if (existing) {
-			if (existing.wsID === params.wsID) this.#lc.warn?.(`duplicate connection for client ${clientID} with the same wsID ${params.wsID}, closing existing connection`, { clientGroupID });
+			if (existing.wsID === params.wsID) this.#lc.warn?.(`duplicate connection for client ${clientID} with the same wsID ${params.wsID}, closing existing connection`, { clientGroupID: clientGroupLogID(clientGroupID) });
 			this.#lc.debug?.(`client ${clientID} already connected, closing existing connection`);
 			existing.close(`replaced by ${params.wsID}`);
 		}
@@ -444,7 +444,7 @@ var Syncer = class {
 			recordConnectionSuccess();
 		} else recordConnectionFailure("protocol_version");
 		if (params.initConnectionMsg) {
-			this.#lc.debug?.("handling init connection message from sec header", params.clientGroupID, params.clientID);
+			this.#lc.debug?.("handling init connection message from sec header", clientGroupLogID(params.clientGroupID), params.clientID);
 			await connection.handleInitConnection(JSON.stringify(params.initConnectionMsg));
 		}
 	};
@@ -491,7 +491,7 @@ var Syncer = class {
 		while (this.#viewSyncers.size) {
 			await this.#drainCoordinator.forceDrainTimeout;
 			for (const vs of this.#viewSyncers.getServices()) {
-				this.#lc.debug?.(`draining view-syncer ${vs.id} (forced)`);
+				this.#lc.debug?.(`draining view-syncer ${clientGroupLogID(vs.id)} (forced)`);
 				vs.stop();
 				break;
 			}