@rocicorp/zero 1.9.0-canary.3 → 1.9.0-canary.4

package/out/zero-cache/src/services/change-streamer/change-streamer-http.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-http.js","names":["#lc","#opts","#changeStreamer","#backupMonitor","#subscribe","#reserveSnapshot","#receiveWebsocket","#getBackupMonitor","#ensureChangeStreamerStarted","#changeStreamerStarted","#shardID","#changeDB","#changeStreamerURI","#resolveChangeStreamer"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport websocket from '@fastify/websocket';\nimport type {LogContext} from '@rocicorp/logger';\nimport WebSocket from 'ws';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport type {IncomingMessageSubset} from '../../types/http.ts';\nimport {pgClient, type PostgresDB} from '../../types/pg.ts';\nimport {type Worker} from '../../types/processes.ts';\nimport {type ShardID} from '../../types/shards.ts';\nimport {\n  streamIn,\n  streamOut,\n  streamOutStringified,\n  type Source,\n} from '../../types/streams.ts';\nimport {URLParams} from '../../types/url-params.ts';\nimport {installWebSocketReceiver} from '../../types/websocket-handoff.ts';\nimport {closeWithError, PROTOCOL_ERROR} from '../../types/ws.ts';\nimport {HttpService} from '../http-service.ts';\nimport type {BackupMonitor} from './backup-monitor.ts';\nimport {\n  downstreamSchema,\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type ChangeStreamerService,\n  type Downstream,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport {discoverChangeStreamerAddress} from './schema/tables.ts';\nimport {snapshotMessageSchema, type SnapshotMessage} from './snapshot.ts';\n\nconst MIN_SUPPORTED_PROTOCOL_VERSION = 1;\n\nconst SNAPSHOT_PATH_PATTERN = '/replication/:version/snapshot';\nconst CHANGES_PATH_PATTERN = '/replication/:version/changes';\nconst PATH_REGEX = /\\/replication\\/v(?<version>\\d+)\\/(changes|snapshot)$/;\n\nconst SNAPSHOT_PATH = `/replication/v${PROTOCOL_VERSION}/snapshot`;\nconst CHANGES_PATH = `/replication/v${PROTOCOL_VERSION}/changes`;\n\ntype Options = {\n  port: number;\n  keepaliveTimeoutMs: number | undefined;\n  startupDelayMs: number;\n};\n\nexport class ChangeStreamerHttpServer extends HttpService {\n  readonly id = 'change-streamer-http-server';\n  readonly #lc: LogContext;\n  readonly #opts: Options;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #backupMonitor: BackupMonitor | null;\n\n  constructor(\n    lc: LogContext,\n    opts: Options,\n    parent: Worker,\n    changeStreamer: ChangeStreamerService,\n    backupMonitor: BackupMonitor | null,\n  ) {\n    super('change-streamer-http-server', lc, opts, async fastify => {\n      await fastify.register(websocket);\n\n      fastify.get(CHANGES_PATH_PATTERN, {websocket: true}, this.#subscribe);\n      fastify.get(\n        SNAPSHOT_PATH_PATTERN,\n        {websocket: true},\n        this.#reserveSnapshot,\n      );\n\n      installWebSocketReceiver<'snapshot' | 'changes'>(\n        lc,\n        fastify.websocketServer,\n        this.#receiveWebsocket,\n        parent,\n      );\n    });\n\n    this.#lc = lc;\n    this.#opts = opts;\n    this.#changeStreamer = changeStreamer;\n    this.#backupMonitor = backupMonitor;\n  }\n\n  #getBackupMonitor() {\n    return must(\n      this.#backupMonitor,\n      'replication-manager is not configured with a ZERO_LITESTREAM_BACKUP_URL',\n    );\n  }\n\n  // Called when receiving a web socket via the main dispatcher handoff.\n  readonly #receiveWebsocket = (\n    ws: WebSocket,\n    action: 'changes' | 'snapshot',\n    msg: IncomingMessageSubset,\n  ) => {\n    switch (action) {\n      case 'snapshot':\n        return this.#reserveSnapshot(ws, msg);\n      case 'changes':\n        return this.#subscribe(ws, msg);\n      default:\n        closeWithError(\n          this._lc,\n          ws,\n          `invalid action \"${action}\" received in handoff`,\n        );\n        return;\n    }\n  };\n\n  readonly #reserveSnapshot = (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const url = new URL(\n        req.url ?? '',\n        req.headers.origin ?? 'http://localhost',\n      );\n      checkProtocolVersion(url.pathname);\n      const taskID = url.searchParams.get('taskID');\n      if (!taskID) {\n        throw new Error('Missing taskID in snapshot request');\n      }\n      const downstream =\n        this.#getBackupMonitor().startSnapshotReservation(taskID);\n      void streamOut(this._lc, downstream, ws);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  readonly #subscribe = async (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const ctx = getSubscriberContext(req);\n      if (ctx.mode === 'serving') {\n        this.#ensureChangeStreamerStarted('incoming subscription');\n      }\n\n      const downstream = await this.#changeStreamer.subscribe(ctx);\n      if (ctx.initial && ctx.taskID && this.#backupMonitor) {\n        // Now that the change-streamer knows about the subscriber and watermark,\n        // end the reservation to safely resume scheduling cleanup.\n        this.#backupMonitor.endReservation(ctx.taskID);\n      }\n      void streamOutStringified(this._lc, downstream, ws);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  #changeStreamerStarted = false;\n\n  #ensureChangeStreamerStarted(reason: string) {\n    if (!this.#changeStreamerStarted && this._state.shouldRun()) {\n      this.#lc.info?.(`starting ChangeStreamerService: ${reason}`);\n      void this.#changeStreamer\n        .run()\n        .catch(e =>\n          this.#lc.warn?.(`ChangeStreamerService ended with error`, e),\n        )\n        .finally(() => this.stop());\n\n      this.#changeStreamerStarted = true;\n    }\n  }\n\n  protected override _onStart(): void {\n    const {startupDelayMs} = this.#opts;\n    this._state.setTimeout(\n      () =>\n        this.#ensureChangeStreamerStarted(\n          `startup delay elapsed (${startupDelayMs} ms)`,\n        ),\n      startupDelayMs,\n    );\n  }\n\n  protected override async _onStop(): Promise<void> {\n    if (this.#changeStreamerStarted) {\n      await this.#changeStreamer.stop();\n    }\n  }\n}\n\nexport class ChangeStreamerHttpClient implements ChangeStreamer {\n  readonly #lc: LogContext;\n  readonly #shardID: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #changeStreamerURI: string | undefined;\n\n  constructor(\n    lc: LogContext,\n    shardID: ShardID,\n    changeDB: string,\n    changeStreamerURI: string | undefined,\n  ) {\n    this.#lc = lc;\n    this.#shardID = shardID;\n    // Create a pg client with a single short-lived connection for the purpose\n    // of change-streamer discovery (i.e. ChangeDB as DNS).\n    this.#changeDB = pgClient(lc, changeDB, 'change-streamer-discovery', {\n      max: 1,\n      ['idle_timeout']: 15,\n    });\n    this.#changeStreamerURI = changeStreamerURI;\n  }\n\n  async #resolveChangeStreamer(path: string) {\n    let baseURL = this.#changeStreamerURI;\n    if (!baseURL) {\n      const address = await discoverChangeStreamerAddress(\n        this.#shardID,\n        this.#changeDB,\n      );\n      if (!address) {\n        throw new Error(`no change-streamer is running`);\n      }\n      baseURL = address.includes('://') ? `${address}/` : `ws://${address}/`;\n    }\n    const uri = new URL(path, baseURL);\n    this.#lc.info?.(`connecting to change-streamer@${uri}`);\n    return uri;\n  }\n\n  async reserveSnapshot(taskID: string): Promise<Source<SnapshotMessage>> {\n    const uri = await this.#resolveChangeStreamer(SNAPSHOT_PATH);\n\n    const params = new URLSearchParams({taskID});\n    const ws = new WebSocket(uri + `?${params.toString()}`);\n\n    return streamIn(this.#lc, ws, snapshotMessageSchema);\n  }\n\n  async subscribe(ctx: SubscriberContext): Promise<Source<Downstream>> {\n    const uri = await this.#resolveChangeStreamer(CHANGES_PATH);\n\n    const params = getParams(ctx);\n    const ws = new WebSocket(uri + `?${params.toString()}`);\n\n    return streamIn(this.#lc, ws, downstreamSchema);\n  }\n}\n\ntype RequestHeaders = Pick<IncomingMessage, 'url' | 'headers'>;\n\nexport function getSubscriberContext(req: RequestHeaders): SubscriberContext {\n  const url = new URL(req.url ?? '', req.headers.origin ?? 'http://localhost');\n  const protocolVersion = checkProtocolVersion(url.pathname);\n  const params = new URLParams(url);\n\n  return {\n    protocolVersion,\n    id: params.get('id', true),\n    taskID: params.get('taskID', false),\n    mode: params.get('mode', false) === 'backup' ? 'backup' : 'serving',\n    replicaVersion: params.get('replicaVersion', true),\n    watermark: params.get('watermark', true),\n    initial: params.getBoolean('initial'),\n  };\n}\n\nfunction checkProtocolVersion(pathname: string): number {\n  const match = PATH_REGEX.exec(pathname);\n  if (!match) {\n    throw new Error(`invalid path: ${pathname}`);\n  }\n  const v = Number(match.groups?.version);\n  if (\n    Number.isNaN(v) ||\n    v > PROTOCOL_VERSION ||\n    v < MIN_SUPPORTED_PROTOCOL_VERSION\n  ) {\n    throw new Error(\n      `Cannot service client at protocol v${v}. ` +\n        `Supported protocols: [v${MIN_SUPPORTED_PROTOCOL_VERSION} ... v${PROTOCOL_VERSION}]`,\n    );\n  }\n  return v;\n}\n\n// This is called from the client-side (i.e. the replicator).\nfunction getParams(ctx: SubscriberContext): URLSearchParams {\n  // The protocolVersion is hard-coded into the CHANGES_PATH.\n  const {protocolVersion, ...stringParams} = ctx;\n  assert(\n    protocolVersion === PROTOCOL_VERSION,\n    `replicator should be setting protocolVersion to ${PROTOCOL_VERSION}`,\n  );\n  return new URLSearchParams({\n    ...stringParams,\n    taskID: ctx.taskID ? ctx.taskID : '',\n    initial: ctx.initial ? 'true' : 'false',\n  });\n}\n"],"mappings":";;;;;;;;;;;;;;;AAgCA,IAAM,iCAAiC;AAEvC,IAAM,wBAAwB;AAC9B,IAAM,uBAAuB;AAC7B,IAAM,aAAa;AAEnB,IAAM,gBAAgB;AACtB,IAAM,eAAe;AAQrB,IAAa,2BAAb,cAA8C,YAAY;CACxD,KAAc;CACd;CACA;CACA;CACA;CAEA,YACE,IACA,MACA,QACA,gBACA,eACA;EACA,MAAM,+BAA+B,IAAI,MAAM,OAAM,YAAW;GAC9D,MAAM,QAAQ,SAAS,SAAS;GAEhC,QAAQ,IAAI,sBAAsB,EAAC,WAAW,KAAI,GAAG,KAAKI,UAAU;GACpE,QAAQ,IACN,uBACA,EAAC,WAAW,KAAI,GAChB,KAAKC,gBACP;GAEA,yBACE,IACA,QAAQ,iBACR,KAAKC,mBACL,MACF;EACF,CAAC;EAED,KAAKN,MAAM;EACX,KAAKC,QAAQ;EACb,KAAKC,kBAAkB;EACvB,KAAKC,iBAAiB;CACxB;CAEA,oBAAoB;EAClB,OAAO,KACL,KAAKA,gBACL,yEACF;CACF;CAGA,qBACE,IACA,QACA,QACG;EACH,QAAQ,QAAR;GACE,KAAK,YACH,OAAO,KAAKE,iBAAiB,IAAI,GAAG;GACtC,KAAK,WACH,OAAO,KAAKD,WAAW,IAAI,GAAG;GAChC;IACE,eACE,KAAK,KACL,IACA,mBAAmB,OAAO,sBAC5B;IACA;EACJ;CACF;CAEA,oBAA6B,IAAe,QAAwB;EAClE,IAAI;GACF,MAAM,MAAM,IAAI,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,kBACxB;GACA,qBAAqB,IAAI,QAAQ;GACjC,MAAM,SAAS,IAAI,aAAa,IAAI,QAAQ;GAC5C,IAAI,CAAC,QACH,MAAM,IAAI,MAAM,oCAAoC;GAEtD,MAAM,aACJ,KAAKG,kBAAkB,EAAE,yBAAyB,MAAM;GAC1D,UAAe,KAAK,KAAK,YAAY,EAAE;EACzC,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;CAEA,aAAsB,OAAO,IAAe,QAAwB;EAClE,IAAI;GACF,MAAM,MAAM,qBAAqB,GAAG;GACpC,IAAI,IAAI,SAAS,WACf,KAAKC,6BAA6B,uBAAuB;GAG3D,MAAM,aAAa,MAAM,KAAKN,gBAAgB,UAAU,GAAG;GAC3D,IAAI,IAAI,WAAW,IAAI,UAAU,KAAKC,gBAGpC,KAAKA,eAAe,eAAe,IAAI,MAAM;GAE/C,qBAA0B,KAAK,KAAK,YAAY,EAAE;EACpD,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;CAEA,yBAAyB;CAEzB,6BAA6B,QAAgB;EAC3C,IAAI,CAAC,KAAKM,0BAA0B,KAAK,OAAO,UAAU,GAAG;GAC3D,KAAKT,IAAI,OAAO,mCAAmC,QAAQ;GAC3D,KAAUE,gBACP,IAAI,EACJ,OAAM,MACL,KAAKF,IAAI,OAAO,0CAA0C,CAAC,CAC7D,EACC,cAAc,KAAK,KAAK,CAAC;GAE5B,KAAKS,yBAAyB;EAChC;CACF;CAEA,WAAoC;EAClC,MAAM,EAAC,mBAAkB,KAAKR;EAC9B,KAAK,OAAO,iBAER,KAAKO,6BACH,0BAA0B,eAAe,KAC3C,GACF,cACF;CACF;CAEA,MAAyB,UAAyB;EAChD,IAAI,KAAKC,wBACP,MAAM,KAAKP,gBAAgB,KAAK;CAEpC;AACF;AAEA,IAAa,2BAAb,MAAgE;CAC9D;CACA;CACA;CACA;CAEA,YACE,IACA,SACA,UACA,mBACA;EACA,KAAKF,MAAM;EACX,KAAKU,WAAW;EAGhB,KAAKC,YAAY,SAAS,IAAI,UAAU,6BAA6B;GACnE,KAAK;IACJ,iBAAiB;EACpB,CAAC;EACD,KAAKC,qBAAqB;CAC5B;CAEA,MAAMC,uBAAuB,MAAc;EACzC,IAAI,UAAU,KAAKD;EACnB,IAAI,CAAC,SAAS;GACZ,MAAM,UAAU,MAAM,8BACpB,KAAKF,UACL,KAAKC,SACP;GACA,IAAI,CAAC,SACH,MAAM,IAAI,MAAM,+BAA+B;GAEjD,UAAU,QAAQ,SAAS,KAAK,IAAI,GAAG,QAAQ,KAAK,QAAQ,QAAQ;EACtE;EACA,MAAM,MAAM,IAAI,IAAI,MAAM,OAAO;EACjC,KAAKX,IAAI,OAAO,iCAAiC,KAAK;EACtD,OAAO;CACT;CAEA,MAAM,gBAAgB,QAAkD;EAItE,MAAM,KAAK,IAAI,YAAU,MAHP,KAAKa,uBAAuB,aAAa,IAG5B,IAAI,IADhB,gBAAgB,EAAC,OAAM,CACP,EAAO,SAAS,GAAG;EAEtD,OAAO,SAAS,KAAKb,KAAK,IAAI,qBAAqB;CACrD;CAEA,MAAM,UAAU,KAAqD;EAInE,MAAM,KAAK,IAAI,YAAU,MAHP,KAAKa,uBAAuB,YAAY,IAG3B,IADhB,UAAU,GACU,EAAO,SAAS,GAAG;EAEtD,OAAO,SAAS,KAAKb,KAAK,IAAI,gBAAgB;CAChD;AACF;AAIA,SAAgB,qBAAqB,KAAwC;CAC3E,MAAM,MAAM,IAAI,IAAI,IAAI,OAAO,IAAI,IAAI,QAAQ,UAAU,kBAAkB;CAC3E,MAAM,kBAAkB,qBAAqB,IAAI,QAAQ;CACzD,MAAM,SAAS,IAAI,UAAU,GAAG;CAEhC,OAAO;EACL;EACA,IAAI,OAAO,IAAI,MAAM,IAAI;EACzB,QAAQ,OAAO,IAAI,UAAU,KAAK;EAClC,MAAM,OAAO,IAAI,QAAQ,KAAK,MAAM,WAAW,WAAW;EAC1D,gBAAgB,OAAO,IAAI,kBAAkB,IAAI;EACjD,WAAW,OAAO,IAAI,aAAa,IAAI;EACvC,SAAS,OAAO,WAAW,SAAS;CACtC;AACF;AAEA,SAAS,qBAAqB,UAA0B;CACtD,MAAM,QAAQ,WAAW,KAAK,QAAQ;CACtC,IAAI,CAAC,OACH,MAAM,IAAI,MAAM,iBAAiB,UAAU;CAE7C,MAAM,IAAI,OAAO,MAAM,QAAQ,OAAO;CACtC,IACE,OAAO,MAAM,CAAC,KACd,IAAA,KACA,IAAI,gCAEJ,MAAM,IAAI,MACR,sCAAsC,EAAE,2BACZ,+BAA+B,SAC7D;CAEF,OAAO;AACT;AAGA,SAAS,UAAU,KAAyC;CAE1D,MAAM,EAAC,iBAAiB,GAAG,iBAAgB;CAC3C,OACE,oBAAA,GACA,mDACF;CACA,OAAO,IAAI,gBAAgB;EACzB,GAAG;EACH,QAAQ,IAAI,SAAS,IAAI,SAAS;EAClC,SAAS,IAAI,UAAU,SAAS;CAClC,CAAC;AACH"}
+{"version":3,"file":"change-streamer-http.js","names":["#lc","#opts","#changeStreamer","#backupMonitor","#subscribe","#reserveSnapshot","#receiveWebsocket","#getBackupMonitor","#ensureChangeStreamerStarted","#changeStreamerStarted","#shardID","#changeDB","#changeStreamerURI","#resolveChangeStreamer"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport websocket from '@fastify/websocket';\nimport type {LogContext} from '@rocicorp/logger';\nimport WebSocket from 'ws';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport type {IncomingMessageSubset} from '../../types/http.ts';\nimport {pgClient, type PostgresDB} from '../../types/pg.ts';\nimport {type Worker} from '../../types/processes.ts';\nimport {type ShardID} from '../../types/shards.ts';\nimport {\n  streamIn,\n  streamOut,\n  streamOutStringified,\n  type Source,\n} from '../../types/streams.ts';\nimport {URLParams} from '../../types/url-params.ts';\nimport {installWebSocketReceiver} from '../../types/websocket-handoff.ts';\nimport {closeWithError, PROTOCOL_ERROR} from '../../types/ws.ts';\nimport {HttpService} from '../http-service.ts';\nimport type {BackupMonitor} from './backup-monitor.ts';\nimport {\n  downstreamSchema,\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type ChangeStreamerService,\n  type Downstream,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport {discoverChangeStreamerAddress} from './schema/tables.ts';\nimport {snapshotMessageSchema, type SnapshotMessage} from './snapshot.ts';\n\nconst MIN_SUPPORTED_PROTOCOL_VERSION = 1;\n\nconst SNAPSHOT_PATH_PATTERN = '/replication/:version/snapshot';\nconst CHANGES_PATH_PATTERN = '/replication/:version/changes';\nconst PATH_REGEX = /\\/replication\\/v(?<version>\\d+)\\/(changes|snapshot)$/;\n\nconst SNAPSHOT_PATH = `/replication/v${PROTOCOL_VERSION}/snapshot`;\nconst CHANGES_PATH = `/replication/v${PROTOCOL_VERSION}/changes`;\n\ntype Options = {\n  port: number;\n  keepaliveTimeoutMs: number | undefined;\n  startupDelayMs: number;\n};\n\nexport class ChangeStreamerHttpServer extends HttpService {\n  readonly id = 'change-streamer-http-server';\n  readonly #lc: LogContext;\n  readonly #opts: Options;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #backupMonitor: BackupMonitor | null;\n\n  constructor(\n    lc: LogContext,\n    opts: Options,\n    parent: Worker,\n    changeStreamer: ChangeStreamerService,\n    backupMonitor: BackupMonitor | null,\n  ) {\n    super('change-streamer-http-server', lc, opts, async fastify => {\n      await fastify.register(websocket);\n\n      fastify.get(CHANGES_PATH_PATTERN, {websocket: true}, this.#subscribe);\n      fastify.get(\n        SNAPSHOT_PATH_PATTERN,\n        {websocket: true},\n        this.#reserveSnapshot,\n      );\n\n      installWebSocketReceiver<'snapshot' | 'changes'>(\n        lc,\n        fastify.websocketServer,\n        this.#receiveWebsocket,\n        parent,\n      );\n    });\n\n    this.#lc = lc;\n    this.#opts = opts;\n    this.#changeStreamer = changeStreamer;\n    this.#backupMonitor = backupMonitor;\n  }\n\n  #getBackupMonitor() {\n    return must(\n      this.#backupMonitor,\n      'replication-manager is not configured with a ZERO_LITESTREAM_BACKUP_URL',\n    );\n  }\n\n  // Called when receiving a web socket via the main dispatcher handoff.\n  readonly #receiveWebsocket = (\n    ws: WebSocket,\n    action: 'changes' | 'snapshot',\n    msg: IncomingMessageSubset,\n  ) => {\n    switch (action) {\n      case 'snapshot':\n        return this.#reserveSnapshot(ws, msg);\n      case 'changes':\n        return this.#subscribe(ws, msg);\n      default:\n        closeWithError(\n          this._lc,\n          ws,\n          `invalid action \"${action}\" received in handoff`,\n        );\n        return;\n    }\n  };\n\n  readonly #reserveSnapshot = (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const url = new URL(\n        req.url ?? '',\n        req.headers.origin ?? 'http://localhost',\n      );\n      checkProtocolVersion(url.pathname);\n      const taskID = url.searchParams.get('taskID');\n      if (!taskID) {\n        throw new Error('Missing taskID in snapshot request');\n      }\n      const downstream =\n        this.#getBackupMonitor().startSnapshotReservation(taskID);\n      void streamOut(this._lc, downstream, ws);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  readonly #subscribe = async (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const ctx = getSubscriberContext(req);\n      if (ctx.mode === 'serving') {\n        this.#ensureChangeStreamerStarted('incoming subscription');\n      }\n\n      const downstream = await this.#changeStreamer.subscribe(ctx);\n      if (ctx.initial && ctx.taskID && this.#backupMonitor) {\n        // Now that the change-streamer knows about the subscriber and watermark,\n        // end the reservation to safely resume scheduling cleanup.\n        this.#backupMonitor.endReservation(ctx.taskID);\n      }\n      void streamOutStringified(this._lc, downstream, ws);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  #changeStreamerStarted = false;\n\n  #ensureChangeStreamerStarted(reason: string) {\n    if (!this.#changeStreamerStarted && this._state.shouldRun()) {\n      this.#lc.info?.(`starting ChangeStreamerService: ${reason}`);\n      void this.#changeStreamer\n        .run()\n        .catch(e =>\n          this.#lc.warn?.(`ChangeStreamerService ended with error`, e),\n        )\n        .finally(() => this.stop());\n\n      this.#changeStreamerStarted = true;\n    }\n  }\n\n  protected override _onStart(): void {\n    const {startupDelayMs} = this.#opts;\n    this._state.setTimeout(\n      () =>\n        this.#ensureChangeStreamerStarted(\n          `startup delay elapsed (${startupDelayMs} ms)`,\n        ),\n      startupDelayMs,\n    );\n  }\n\n  protected override async _onStop(): Promise<void> {\n    if (this.#changeStreamerStarted) {\n      await this.#changeStreamer.stop();\n    }\n  }\n}\n\nexport class ChangeStreamerHttpClient implements ChangeStreamer {\n  readonly #lc: LogContext;\n  readonly #shardID: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #changeStreamerURI: string | undefined;\n\n  constructor(\n    lc: LogContext,\n    shardID: ShardID,\n    changeDB: string,\n    changeStreamerURI: string | undefined,\n  ) {\n    this.#lc = lc;\n    this.#shardID = shardID;\n    // Create a pg client with a single short-lived connection for the purpose\n    // of change-streamer discovery (i.e. ChangeDB as DNS).\n    this.#changeDB = pgClient(lc, changeDB, 'change-streamer-discovery', {\n      max: 1,\n      ['idle_timeout']: 15,\n    });\n    this.#changeStreamerURI = changeStreamerURI;\n  }\n\n  async #resolveChangeStreamer(path: string) {\n    let baseURL = this.#changeStreamerURI;\n    if (!baseURL) {\n      const address = await discoverChangeStreamerAddress(\n        this.#shardID,\n        this.#changeDB,\n      );\n      if (!address) {\n        throw new Error(`no change-streamer is running`);\n      }\n      baseURL = address.includes('://') ? `${address}/` : `ws://${address}/`;\n    }\n    const uri = new URL(path, baseURL);\n    this.#lc.info?.(`connecting to change-streamer@${uri}`);\n    return uri;\n  }\n\n  async reserveSnapshot(taskID: string): Promise<Source<SnapshotMessage>> {\n    const uri = await this.#resolveChangeStreamer(SNAPSHOT_PATH);\n\n    const params = new URLSearchParams({taskID});\n    const ws = new WebSocket(uri + `?${params.toString()}`);\n\n    return streamIn(this.#lc, ws, snapshotMessageSchema);\n  }\n\n  async subscribe(ctx: SubscriberContext): Promise<Source<Downstream>> {\n    const uri = await this.#resolveChangeStreamer(CHANGES_PATH);\n\n    const params = getParams(ctx);\n    const ws = new WebSocket(uri + `?${params.toString()}`);\n\n    return streamIn(this.#lc, ws, downstreamSchema);\n  }\n}\n\ntype RequestHeaders = Pick<IncomingMessage, 'url' | 'headers'>;\n\nexport function getSubscriberContext(req: RequestHeaders): SubscriberContext {\n  const url = new URL(req.url ?? '', req.headers.origin ?? 'http://localhost');\n  const protocolVersion = checkProtocolVersion(url.pathname);\n  const params = new URLParams(url);\n\n  return {\n    protocolVersion,\n    id: params.get('id', true),\n    taskID: params.get('taskID', false),\n    mode: params.get('mode', false) === 'backup' ? 'backup' : 'serving',\n    replicaVersion: params.get('replicaVersion', true),\n    watermark: params.get('watermark', true),\n    initial: params.getBoolean('initial'),\n  };\n}\n\nfunction checkProtocolVersion(pathname: string): number {\n  const match = PATH_REGEX.exec(pathname);\n  if (!match) {\n    throw new Error(`invalid path: ${pathname}`);\n  }\n  const v = Number(match.groups?.version);\n  if (\n    Number.isNaN(v) ||\n    v > PROTOCOL_VERSION ||\n    v < MIN_SUPPORTED_PROTOCOL_VERSION\n  ) {\n    throw new Error(\n      `Cannot service client at protocol v${v}. ` +\n        `Supported protocols: [v${MIN_SUPPORTED_PROTOCOL_VERSION} ... v${PROTOCOL_VERSION}]`,\n    );\n  }\n  return v;\n}\n\n// This is called from the client-side (i.e. the replicator).\nfunction getParams(ctx: SubscriberContext): URLSearchParams {\n  // The protocolVersion is hard-coded into the CHANGES_PATH.\n  const {protocolVersion, ...stringParams} = ctx;\n  assert(\n    protocolVersion === PROTOCOL_VERSION,\n    `replicator should be setting protocolVersion to ${PROTOCOL_VERSION}`,\n  );\n  return new URLSearchParams({\n    ...stringParams,\n    taskID: ctx.taskID ? ctx.taskID : '',\n    initial: ctx.initial ? 'true' : 'false',\n  });\n}\n"],"mappings":";;;;;;;;;;;;;;;AAgCA,IAAM,iCAAiC;AAEvC,IAAM,wBAAwB;AAC9B,IAAM,uBAAuB;AAC7B,IAAM,aAAa;AAEnB,IAAM,gBAAgB;AACtB,IAAM,eAAe;AAQrB,IAAa,2BAAb,cAA8C,YAAY;CACxD,KAAc;CACd;CACA;CACA;CACA;CAEA,YACE,IACA,MACA,QACA,gBACA,eACA;EACA,MAAM,+BAA+B,IAAI,MAAM,OAAM,YAAW;GAC9D,MAAM,QAAQ,SAAS,SAAS;GAEhC,QAAQ,IAAI,sBAAsB,EAAC,WAAW,KAAI,GAAG,KAAKI,UAAU;GACpE,QAAQ,IACN,uBACA,EAAC,WAAW,KAAI,GAChB,KAAKC,gBACP;GAEA,yBACE,IACA,QAAQ,iBACR,KAAKC,mBACL,MACF;EACF,CAAC;EAED,KAAKN,MAAM;EACX,KAAKC,QAAQ;EACb,KAAKC,kBAAkB;EACvB,KAAKC,iBAAiB;CACxB;CAEA,oBAAoB;EAClB,OAAO,KACL,KAAKA,gBACL,yEACF;CACF;CAGA,qBACE,IACA,QACA,QACG;EACH,QAAQ,QAAR;GACE,KAAK,YACH,OAAO,KAAKE,iBAAiB,IAAI,GAAG;GACtC,KAAK,WACH,OAAO,KAAKD,WAAW,IAAI,GAAG;GAChC;IACE,eACE,KAAK,KACL,IACA,mBAAmB,OAAO,sBAC5B;IACA;EACJ;CACF;CAEA,oBAA6B,IAAe,QAAwB;EAClE,IAAI;GACF,MAAM,MAAM,IAAI,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,kBACxB;GACA,qBAAqB,IAAI,QAAQ;GACjC,MAAM,SAAS,IAAI,aAAa,IAAI,QAAQ;GAC5C,IAAI,CAAC,QACH,MAAM,IAAI,MAAM,oCAAoC;GAEtD,MAAM,aACJ,KAAKG,kBAAkB,EAAE,yBAAyB,MAAM;GAC1D,UAAe,KAAK,KAAK,YAAY,EAAE;EACzC,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;CAEA,aAAsB,OAAO,IAAe,QAAwB;EAClE,IAAI;GACF,MAAM,MAAM,qBAAqB,GAAG;GACpC,IAAI,IAAI,SAAS,WACf,KAAKC,6BAA6B,uBAAuB;GAG3D,MAAM,aAAa,MAAM,KAAKN,gBAAgB,UAAU,GAAG;GAC3D,IAAI,IAAI,WAAW,IAAI,UAAU,KAAKC,gBAGpC,KAAKA,eAAe,eAAe,IAAI,MAAM;GAE/C,qBAA0B,KAAK,KAAK,YAAY,EAAE;EACpD,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;CAEA,yBAAyB;CAEzB,6BAA6B,QAAgB;EAC3C,IAAI,CAAC,KAAKM,0BAA0B,KAAK,OAAO,UAAU,GAAG;GAC3D,KAAKT,IAAI,OAAO,mCAAmC,QAAQ;GAC3D,KAAUE,gBACP,IAAI,EACJ,OAAM,MACL,KAAKF,IAAI,OAAO,0CAA0C,CAAC,CAC7D,EACC,cAAc,KAAK,KAAK,CAAC;GAE5B,KAAKS,yBAAyB;EAChC;CACF;CAEA,WAAoC;EAClC,MAAM,EAAC,mBAAkB,KAAKR;EAC9B,KAAK,OAAO,iBAER,KAAKO,6BACH,0BAA0B,eAAe,KAC3C,GACF,cACF;CACF;CAEA,MAAyB,UAAyB;EAChD,IAAI,KAAKC,wBACP,MAAM,KAAKP,gBAAgB,KAAK;CAEpC;AACF;AAEA,IAAa,2BAAb,MAAgE;CAC9D;CACA;CACA;CACA;CAEA,YACE,IACA,SACA,UACA,mBACA;EACA,KAAKF,MAAM;EACX,KAAKU,WAAW;EAGhB,KAAKC,YAAY,SAAS,IAAI,UAAU,6BAA6B;GACnE,KAAK;IACJ,iBAAiB;EACpB,CAAC;EACD,KAAKC,qBAAqB;CAC5B;CAEA,MAAMC,uBAAuB,MAAc;EACzC,IAAI,UAAU,KAAKD;EACnB,IAAI,CAAC,SAAS;GACZ,MAAM,UAAU,MAAM,8BACpB,KAAKF,UACL,KAAKC,SACP;GACA,IAAI,CAAC,SACH,MAAM,IAAI,MAAM,+BAA+B;GAEjD,UAAU,QAAQ,SAAS,KAAK,IAAI,GAAG,QAAQ,KAAK,QAAQ,QAAQ;EACtE;EACA,MAAM,MAAM,IAAI,IAAI,MAAM,OAAO;EACjC,KAAKX,IAAI,OAAO,iCAAiC,KAAK;EACtD,OAAO;CACT;CAEA,MAAM,gBAAgB,QAAkD;EAItE,MAAM,KAAK,IAAI,UAAU,MAHP,KAAKa,uBAAuB,aAAa,IAG5B,IAAI,IADhB,gBAAgB,EAAC,OAAM,CACP,EAAO,SAAS,GAAG;EAEtD,OAAO,SAAS,KAAKb,KAAK,IAAI,qBAAqB;CACrD;CAEA,MAAM,UAAU,KAAqD;EAInE,MAAM,KAAK,IAAI,UAAU,MAHP,KAAKa,uBAAuB,YAAY,IAG3B,IADhB,UAAU,GACU,EAAO,SAAS,GAAG;EAEtD,OAAO,SAAS,KAAKb,KAAK,IAAI,gBAAgB;CAChD;AACF;AAIA,SAAgB,qBAAqB,KAAwC;CAC3E,MAAM,MAAM,IAAI,IAAI,IAAI,OAAO,IAAI,IAAI,QAAQ,UAAU,kBAAkB;CAC3E,MAAM,kBAAkB,qBAAqB,IAAI,QAAQ;CACzD,MAAM,SAAS,IAAI,UAAU,GAAG;CAEhC,OAAO;EACL;EACA,IAAI,OAAO,IAAI,MAAM,IAAI;EACzB,QAAQ,OAAO,IAAI,UAAU,KAAK;EAClC,MAAM,OAAO,IAAI,QAAQ,KAAK,MAAM,WAAW,WAAW;EAC1D,gBAAgB,OAAO,IAAI,kBAAkB,IAAI;EACjD,WAAW,OAAO,IAAI,aAAa,IAAI;EACvC,SAAS,OAAO,WAAW,SAAS;CACtC;AACF;AAEA,SAAS,qBAAqB,UAA0B;CACtD,MAAM,QAAQ,WAAW,KAAK,QAAQ;CACtC,IAAI,CAAC,OACH,MAAM,IAAI,MAAM,iBAAiB,UAAU;CAE7C,MAAM,IAAI,OAAO,MAAM,QAAQ,OAAO;CACtC,IACE,OAAO,MAAM,CAAC,KACd,IAAA,KACA,IAAI,gCAEJ,MAAM,IAAI,MACR,sCAAsC,EAAE,2BACZ,+BAA+B,SAC7D;CAEF,OAAO;AACT;AAGA,SAAS,UAAU,KAAyC;CAE1D,MAAM,EAAC,iBAAiB,GAAG,iBAAgB;CAC3C,OACE,oBAAA,GACA,mDACF;CACA,OAAO,IAAI,gBAAgB;EACzB,GAAG;EACH,QAAQ,IAAI,SAAS,IAAI,SAAS;EAClC,SAAS,IAAI,UAAU,SAAS;CAClC,CAAC;AACH"}

package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-service.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAWjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAClD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAGnD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAEL,KAAK,gBAAgB,EAEtB,MAAM,iDAAiD,CAAC;AACzD,OAAO,EAGL,KAAK,0BAA0B,EAChC,MAAM,qCAAqC,CAAC;AAC7C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,2CAA2C,CAAC;AAMjF,OAAO,EACL,KAAK,qBAAqB,EAG3B,MAAM,sBAAsB,CAAC;AAS9B,OAAO,EAEL,KAAK,SAAS,EACd,KAAK,aAAa,IAAI,aAAa,EACpC,MAAM,aAAa,CAAC;AAGrB,MAAM,MAAM,aAAa,GAAG,aAAa,GAAG;IAC1C,kCAAkC,EAAE,MAAM,CAAC;CAC5C,CAAC;AAEF;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,QAAQ,EAAE,UAAU,EACpB,YAAY,EAAE,YAAY,EAC1B,0BAA0B,EAAE,0BAA0B,EACtD,iBAAiB,EAAE,iBAAiB,EACpC,SAAS,EAAE,SAAS,GAAG,IAAI,EAC3B,SAAS,EAAE,OAAO,EAClB,IAAI,EAAE,aAAa,EACnB,YAAY,oBAAa,GACxB,OAAO,CAAC,qBAAqB,CAAC,CA4BhC;AAID,MAAM,MAAM,SAAS,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;AAEnD;;;;;;;;;;;;;GAaG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,MAAM;IACjB,GAAG,EAAE,SAAS;IACd,IAAI,EAAE,MAAM;CACb,CAAC"}
+{"version":3,"file":"change-streamer-service.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAWjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAClD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAInD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAEL,KAAK,gBAAgB,EAEtB,MAAM,iDAAiD,CAAC;AACzD,OAAO,EAGL,KAAK,0BAA0B,EAChC,MAAM,qCAAqC,CAAC;AAC7C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,2CAA2C,CAAC;AAMjF,OAAO,EACL,KAAK,qBAAqB,EAG3B,MAAM,sBAAsB,CAAC;AAS9B,OAAO,EAEL,KAAK,SAAS,EACd,KAAK,aAAa,IAAI,aAAa,EACpC,MAAM,aAAa,CAAC;AAGrB,MAAM,MAAM,aAAa,GAAG,aAAa,GAAG;IAC1C,kCAAkC,EAAE,MAAM,CAAC;CAC5C,CAAC;AAEF;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,QAAQ,EAAE,UAAU,EACpB,YAAY,EAAE,YAAY,EAC1B,0BAA0B,EAAE,0BAA0B,EACtD,iBAAiB,EAAE,iBAAiB,EACpC,SAAS,EAAE,SAAS,GAAG,IAAI,EAC3B,SAAS,EAAE,OAAO,EAClB,IAAI,EAAE,aAAa,EACnB,YAAY,oBAAa,GACxB,OAAO,CAAC,qBAAqB,CAAC,CA4BhC;AAID,MAAM,MAAM,SAAS,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;AAEnD;;;;;;;;;;;;;GAaG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,MAAM;IACjB,GAAG,EAAE,SAAS;IACd,IAAI,EAAE,MAAM;CACb,CAAC"}

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js

@@ -8,6 +8,7 @@ import { publishCriticalEvent } from "../../observability/events.js";
 import { AutoResetSignal, ensureReplicationConfig, markResetRequired } from "./schema/tables.js";
 import { Subscription } from "../../types/subscription.js";
 import { publishReplicationError, replicationStatusError } from "../replicator/replication-status.js";
+import { orTimeout } from "../../types/timeout.js";
 import "./change-streamer.js";
 import { Forwarder } from "./forwarder.js";
 import { initChangeStreamerSchema } from "./schema/init.js";
@@ -182,6 +183,7 @@ var ChangeStreamerImpl = class {
 	#changeCounter = getOrCreateCounter("replication", "changes", "Count of replicated changes (DML or DDL statements)");
 	#latestStatus;
 	#purgeLock;
+	#fatalSourceError;
 	#stream;
 	constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB, replicaVersion, source, replicationStatusPublisher, initialPurgeLock, autoReset, opts, setTimeoutFn = setTimeout) {
 		this.id = `change-streamer`;
@@ -217,6 +219,17 @@ var ChangeStreamerImpl = class {
 			try {
 				const { lastWatermark, backfillRequests } = await this.#storer.getStartStreamInitializationParameters();
 				const stream = await this.#source.startStream(lastWatermark, backfillRequests);
+				const waitForFlowControl = async (wait) => {
+					if (!stream.sourceTerminated) {
+						await wait;
+						return;
+					}
+					const sourceError = await Promise.race([wait.then(() => void 0), stream.sourceTerminated]);
+					if (sourceError) {
+						this.#fatalSourceError = sourceError;
+						throw sourceError;
+					}
+				};
 				this.#storer.run().catch((e) => stream.changes.cancel(e));
 				this.#stream = stream;
 				if (this.#state.resetBackoff() > REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS) this.#replicationStatusPublisher.publish(this.#lc, "Replicating", `Replicating from ${lastWatermark}`);
@@ -255,12 +268,12 @@ var ChangeStreamerImpl = class {
 					unflushedBytes += json.length;
 					if (unflushedBytes < flushBytesThreshold) this.#forwarder.forward(entry);
 					else {
-						await this.#forwarder.forwardWithFlowControl(entry);
+						await waitForFlowControl(this.#forwarder.forwardWithFlowControl(entry));
 						unflushedBytes = 0;
 					}
 					if (type === "commit" || type === "rollback") watermark = null;
 					const readyForMore = this.#storer.readyForMore();
-					if (readyForMore) await readyForMore;
+					if (readyForMore) await waitForFlowControl(readyForMore);
 				}
 			} catch (e) {
 				err = e;
@@ -268,6 +281,10 @@ var ChangeStreamerImpl = class {
 				this.#stream?.changes.cancel();
 				this.#stream = void 0;
 			}
+			if (this.#fatalSourceError && err === this.#fatalSourceError) {
+				this.#forwarder.stopProgressMonitor();
+				throw err;
+			}
 			if (watermark) {
 				this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);
 				this.#storer.abort();
@@ -366,6 +383,10 @@ var ChangeStreamerImpl = class {
 	async stop(err) {
 		this.#state.stop(this.#lc, err);
 		this.#stream?.changes.cancel();
+		if (this.#fatalSourceError) {
+			await orTimeout(Promise.allSettled([this.#storer.stop(), this.#source.stop()]), 100);
+			return;
+		}
 		await this.#storer.stop();
 		await this.#source.stop();
 	}

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-service.js","names":["#lc","#shard","#changeDB","#replicaVersion","#source","#storer","#forwarder","#replicationStatusPublisher","#autoReset","#state","#initialWatermarks","#serving","#txCounter","#changeCounter","#stream","#purgeLock","#latestStatus","#handleControlMessage","#purgeOldChanges"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"sourcesContent":["import {getDefaultHighWaterMark} from 'node:stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {publishCriticalEvent} from '../../observability/events.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {\n  min,\n  type AtLeastOne,\n  type LexiVersion,\n} from '../../types/lexi-version.ts';\nimport type {PostgresDB} from '../../types/pg.ts';\nimport type {ShardID} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {\n  ChangeSource,\n  ChangeStream,\n} from '../change-source/change-source.ts';\nimport {\n  type ChangeStreamControl,\n  type ChangeStreamData,\n  type Rollback,\n} from '../change-source/protocol/current/downstream.ts';\nimport {\n  publishReplicationError,\n  replicationStatusError,\n  type ReplicationStatusPublisher,\n} from '../replicator/replication-status.ts';\nimport type {SubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n  DEFAULT_MAX_RETRY_DELAY_MS,\n  RunningState,\n  UnrecoverableError,\n} from '../running-state.ts';\nimport {\n  type ChangeStreamerService,\n  type Status,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {Forwarder} from './forwarder.ts';\nimport {initChangeStreamerSchema} from './schema/init.ts';\nimport {\n  AutoResetSignal,\n  ensureReplicationConfig,\n  markResetRequired,\n} from './schema/tables.ts';\nimport {\n  Storer,\n  type PurgeLock,\n  type TuningOptions as StorerOptions,\n} from './storer.ts';\nimport {Subscriber} from './subscriber.ts';\n\nexport type TuningOptions = StorerOptions & {\n  flowControlConsensusPaddingSeconds: number;\n};\n\n/**\n * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.\n */\nexport async function initializeStreamer(\n  lc: LogContext,\n  shard: ShardID,\n  taskID: string,\n  discoveryAddress: string,\n  discoveryProtocol: string,\n  changeDB: PostgresDB,\n  changeSource: ChangeSource,\n  replicationStatusPublisher: ReplicationStatusPublisher,\n  subscriptionState: SubscriptionState,\n  purgeLock: PurgeLock | null,\n  autoReset: boolean,\n  opts: TuningOptions,\n  setTimeoutFn = setTimeout,\n): Promise<ChangeStreamerService> {\n  // Make sure the ChangeLog DB is set up.\n  await initChangeStreamerSchema(lc, changeDB, shard);\n  await ensureReplicationConfig(\n    lc,\n    changeDB,\n    subscriptionState,\n    shard,\n    autoReset,\n    setTimeoutFn,\n  );\n\n  const {replicaVersion} = subscriptionState;\n  return new ChangeStreamerImpl(\n    lc,\n    shard,\n    taskID,\n    discoveryAddress,\n    discoveryProtocol,\n    changeDB,\n    replicaVersion,\n    changeSource,\n    replicationStatusPublisher,\n    purgeLock,\n    autoReset,\n    opts,\n    setTimeoutFn,\n  );\n}\n\nconst REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS = 5000;\n\nexport type ChangeTag = ChangeStreamData[1]['tag'];\n\n/**\n * Internally all Downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering for:\n * 1. Replaying new changes in the Storer\n * 2. Filtering old changes in the Subscriber\n *\n * However, only the watermark for `Commit` messages are exposed to\n * subscribers, as that is the only semantically correct watermark to\n * use for tracking a position in a replication stream.\n *\n * Additionally, the ChangeStreamData is eagerly stringified once, after which\n * the string is passed to the changeLog and all subscribers, eliminating\n * redundant stringification and reducing GC churn.\n */\nexport type WatermarkedChange = [\n  watermark: string,\n  tag: ChangeTag,\n  json: string,\n];\n\n/**\n * Upstream-agnostic dispatch of messages in a {@link ChangeStreamMessage} to a\n * {@link Forwarder} and {@link Storer} to execute the forward-store-ack\n * procedure described in {@link ChangeStreamer}.\n *\n * ### Subscriber Catchup\n *\n * Connecting clients first need to be \"caught up\" to the current watermark\n * (from stored change log entries) before new entries are forwarded to\n * them. This is non-trivial because the replication stream may be in the\n * middle of a pending streamed Transaction for which some entries have\n * already been forwarded but are not yet committed to the store.\n *\n *\n * ```\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * | Historic changes in storage |  Pending (streamed) tx  |   Next tx\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n *                                           Replication stream\n *                                           >  >  >  >  >  >  >  >  >\n *           ^  ---> required catchup --->   ^\n * Subscriber watermark               Subscription begins\n * ```\n *\n * Preemptively buffering the changes of every pending transaction\n * would be wasteful and consume too much memory for large transactions.\n *\n * Instead, the streamer synchronously dispatches changes and subscriptions\n * to the {@link Forwarder} and the {@link Storer} such that the two\n * components are aligned as to where in the stream the subscription started.\n * The two components then coordinate catchup and handoff via the\n * {@link Subscriber} object with the following algorithm:\n *\n * * If the streamer is in the middle of a pending Transaction, the\n *   Subscriber is \"queued\" on both the Forwarder and the Storer. In this\n *   state, new changes are *not* forwarded to the Subscriber, and catchup\n *   is not yet executed.\n * * Once the commit message for the pending Transaction is processed\n *   by the Storer, it begins catchup on the Subscriber (with a READONLY\n *   snapshot so that it does not block subsequent storage operations).\n *   This catchup is thus guaranteed to load the change log entries of\n *   that last Transaction.\n * * When the Forwarder processes that same commit message, it moves the\n *   Subscriber from the \"queued\" to the \"active\" set of clients such that\n *   the Subscriber begins receiving new changes, starting from the next\n *   Transaction.\n * * The Subscriber does not forward those changes, however, if its catchup\n *   is not complete. Until then, it buffers the changes in memory.\n * * Once catchup is complete, the buffered changes are immediately sent\n *   and the Subscriber henceforth forwards changes as they are received.\n *\n * In the (common) case where the streamer is not in the middle of a pending\n * transaction when a subscription begins, the Storer begins catchup\n * immediately and the Forwarder directly adds the Subscriber to its active\n * set. However, the Subscriber still buffers any forwarded messages until\n * its catchup is complete.\n *\n * ### Watermarks and ordering\n *\n * The ChangeStreamerService depends on its {@link ChangeSource} to send\n * changes in contiguous [`begin`, `data` ..., `data`, `commit`] sequences\n * in commit order. This follows Postgres's Logical Replication Protocol\n * Message Flow:\n *\n * https://www.postgresql.org/docs/16/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW\n *\n * > The logical replication protocol sends individual transactions one by one.\n * > This means that all messages between a pair of Begin and Commit messages belong to the same transaction.\n *\n * In order to correctly replay (new) and filter (old) messages to subscribers\n * at different points in the replication stream, these changes must be assigned\n * watermarks such that they preserve the order in which they were received\n * from the ChangeSource.\n *\n * A previous implementation incorrectly derived these watermarks from the Postgres\n * Log Sequence Numbers (LSN) of each message. However, LSNs from concurrent,\n * non-conflicting transactions can overlap, which can result in a `begin` message\n * with an earlier LSN arriving after a `commit` message. For example, the\n * changes for these transactions:\n *\n * ```\n * LSN:   1     2     3  4    5   6   7     8   9      10\n * tx1: begin  data     data     data     commit\n * tx2:             begin    data    data      data  commit\n * ```\n *\n * will arrive as:\n *\n * ```\n * begin1, data2, data4, data6, commit8, begin3, data5, data7, data9, commit10\n * ```\n *\n * Thus, LSN of non-commit messages are not suitable for tracking the sorting\n * order of the replication stream.\n *\n * Instead, the ChangeStreamer uses the following algorithm for deterministic\n * catchup and filtering of changes:\n *\n * * A `commit` message is assigned to a watermark corresponding to its LSN.\n *   These are guaranteed to be in commit order by definition.\n *\n * * `begin` and `data` messages are assigned to the watermark of the\n *   preceding `commit` (the previous transaction, or the replication\n *   slot's starting LSN) plus 1. This guarantees that they will be sorted\n *   after the previously commit transaction even if their LSNs came before it.\n *   This is referred to as the `preCommitWatermark`.\n *\n * * In the ChangeLog DB, messages have a secondary sort column `pos`, which is\n *   the position of the message within its transaction, with the `begin` message\n *   starting at `0`. This guarantees that `begin` and `data` messages will be\n *   fetched in the original ChangeSource order during catchup.\n *\n * `begin` and `data` messages share the same watermark, but this is sufficient for\n * Subscriber filtering because subscribers only know about the `commit` watermarks\n * exposed in the `Downstream` `Commit` message. The Subscriber object thus compares\n * the internal watermarks of the incoming messages against the commit watermark of\n * the caller, updating the watermark at every `Commit` message that is forwarded.\n *\n * ### Cleanup\n *\n * As mentioned in the {@link ChangeStreamer} documentation: \"the ChangeStreamer\n * uses a combination of [the \"initial\", i.e. backup-derived watermark and] ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\"\n *\n * More concretely:\n *\n * * The `initial`, backup-derived watermark is the earliest to which cleanup\n *   should ever happen.\n *\n * * However, it is possible for the replica backup to be *ahead* of a connected\n *   subscriber; and if a network error causes that subscriber to retry from its\n *   last watermark, the change streamer must support it.\n *\n * Thus, before cleaning up to an `initial` backup-derived watermark, the change\n * streamer first confirms that all connected subscribers have also passed\n * that watermark.\n */\nclass ChangeStreamerImpl implements ChangeStreamerService {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #replicaVersion: string;\n  readonly #source: ChangeSource;\n  readonly #storer: Storer;\n  readonly #forwarder: Forwarder;\n  readonly #replicationStatusPublisher: ReplicationStatusPublisher;\n\n  readonly #autoReset: boolean;\n  readonly #state: RunningState;\n  readonly #initialWatermarks = new Set<string>();\n\n  // Starting the (Postgres) ChangeStream results in killing the previous\n  // Postgres subscriber, potentially creating a gap in which the old\n  // change-streamer has shut down and the new change-streamer has not yet\n  // been recognized as \"healthy\" (and thus does not get any requests).\n  //\n  // To minimize this gap, delay starting the ChangeStream until the first\n  // request from a `serving` replicator, indicating that higher level\n  // load-balancing / routing logic has begun routing requests to this task.\n  readonly #serving = resolver();\n\n  readonly #txCounter = getOrCreateCounter(\n    'replication',\n    'transactions',\n    'Count of replicated transactions',\n  );\n  readonly #changeCounter = getOrCreateCounter(\n    'replication',\n    'changes',\n    'Count of replicated changes (DML or DDL statements)',\n  );\n\n  #latestStatus: Status;\n  #purgeLock: PurgeLock | null;\n  #stream: ChangeStream | undefined;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    taskID: string,\n    discoveryAddress: string,\n    discoveryProtocol: string,\n    changeDB: PostgresDB,\n    replicaVersion: string,\n    source: ChangeSource,\n    replicationStatusPublisher: ReplicationStatusPublisher,\n    initialPurgeLock: PurgeLock | null,\n    autoReset: boolean,\n    opts: TuningOptions,\n    setTimeoutFn = setTimeout,\n  ) {\n    this.id = `change-streamer`;\n    this.#lc = lc.withContext('component', 'change-streamer');\n    this.#shard = shard;\n    this.#changeDB = changeDB;\n    this.#replicaVersion = replicaVersion;\n    this.#source = source;\n    this.#storer = new Storer(\n      lc,\n      shard,\n      taskID,\n      discoveryAddress,\n      discoveryProtocol,\n      changeDB,\n      replicaVersion,\n      consumed => this.#stream?.acks.push(['status', consumed[1], consumed[2]]),\n      err => this.stop(err),\n      opts,\n    );\n    this.#forwarder = new Forwarder(lc, {\n      flowControlConsensusPaddingSeconds:\n        opts.flowControlConsensusPaddingSeconds,\n    });\n    this.#replicationStatusPublisher = replicationStatusPublisher;\n    this.#purgeLock = initialPurgeLock;\n    this.#autoReset = autoReset;\n    this.#state = new RunningState(this.id, undefined, setTimeoutFn);\n    this.#latestStatus = {tag: 'status'};\n  }\n\n  async run() {\n    this.#lc.info?.('starting change stream');\n\n    this.#forwarder.startProgressMonitor();\n\n    const lagReport = await this.#source.startLagReporter();\n    if (lagReport) {\n      this.#latestStatus.lagReport = lagReport;\n    }\n\n    // Once this change-streamer acquires \"ownership\" of the change DB,\n    // it is safe to start the storer.\n    await this.#storer.assumeOwnership(this.#purgeLock);\n    this.#purgeLock = null;\n\n    // The threshold in (estimated number of) bytes to send() on subscriber\n    // websockets before `await`-ing the I/O buffers to be ready for more.\n    const flushBytesThreshold = getDefaultHighWaterMark(false);\n\n    while (this.#state.shouldRun()) {\n      let err: unknown;\n      let watermark: string | null = null;\n      let unflushedBytes = 0;\n      try {\n        const {lastWatermark, backfillRequests} =\n          await this.#storer.getStartStreamInitializationParameters();\n        const stream = await this.#source.startStream(\n          lastWatermark,\n          backfillRequests,\n        );\n        this.#storer.run().catch(e => stream.changes.cancel(e));\n\n        this.#stream = stream;\n        if (\n          this.#state.resetBackoff() >\n          REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS\n        ) {\n          // After recovering from a backoff for which a replication status\n          // error was published, publish an OK status\n          this.#replicationStatusPublisher.publish(\n            this.#lc,\n            'Replicating',\n            `Replicating from ${lastWatermark}`,\n          );\n        }\n        watermark = null;\n\n        for await (const change of stream.changes) {\n          const [type, msg] = change;\n          switch (type) {\n            case 'status':\n              if (msg.ack) {\n                this.#storer.status(change); // storer acks once it gets through its queue\n              }\n              if (msg.lagReport) {\n                // Lag reports are not stored in the cdc change log, but rather\n                // only forwarded on \"live\" connections. When a new subscriber\n                // is catching up, it is initialized with the #latestStatus\n                // from which it can measure lag while catching up.\n                this.#latestStatus.lagReport = msg.lagReport;\n                this.#forwarder.sendStatus(this.#latestStatus);\n              }\n              continue;\n            case 'control':\n              await this.#handleControlMessage(msg);\n              continue; // control messages are not stored/forwarded\n            case 'begin':\n              watermark = change[2].commitWatermark;\n              break;\n            case 'commit':\n              if (watermark !== change[2].watermark) {\n                throw new UnrecoverableError(\n                  `commit watermark ${change[2].watermark} does not match 'begin' watermark ${watermark}`,\n                );\n              }\n              this.#txCounter.add(1);\n              break;\n            default:\n              if (type === 'data') {\n                this.#changeCounter.add(1);\n              }\n              if (watermark === null) {\n                throw new UnrecoverableError(\n                  `${type} change (${msg.tag}) received before 'begin' message`,\n                );\n              }\n              break;\n          }\n\n          const json = this.#storer.store(watermark, change);\n          const entry: WatermarkedChange = [watermark, change[1].tag, json];\n          unflushedBytes += json.length;\n          if (unflushedBytes < flushBytesThreshold) {\n            // pipeline changes until flushBytesThreshold\n            this.#forwarder.forward(entry);\n          } else {\n            // Wait for messages to clear socket buffers to ensure that they\n            // make their way to subscribers. Without this `await`, the\n            // messages end up being buffered in this process, which:\n            // (1) results in memory pressure and increased GC activity\n            // (2) prevents subscribers from processing the messages as they\n            //     arrive, instead getting them in a large batch after being\n            //     idle while they were queued (causing further delays).\n            await this.#forwarder.forwardWithFlowControl(entry);\n            unflushedBytes = 0;\n          }\n\n          if (type === 'commit' || type === 'rollback') {\n            watermark = null;\n          }\n\n          // Allow the storer to exert back pressure.\n          const readyForMore = this.#storer.readyForMore();\n          if (readyForMore) {\n            await readyForMore;\n          }\n        }\n      } catch (e) {\n        err = e;\n      } finally {\n        this.#stream?.changes.cancel();\n        this.#stream = undefined;\n      }\n\n      // When the change stream is interrupted, abort any pending transaction.\n      if (watermark) {\n        this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);\n        this.#storer.abort();\n        this.#forwarder.forward([watermark, 'rollback', ROLLBACK_JSON]);\n      }\n\n      // Backoff and drain any pending entries in the storer before reconnecting.\n      await Promise.all([\n        this.#storer.stop(),\n        this.#state.backoff(this.#lc, err),\n        this.#state.retryDelay > REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS\n          ? publishCriticalEvent(\n              this.#lc,\n              replicationStatusError(this.#lc, 'Replicating', err),\n            )\n          : promiseVoid,\n      ]);\n    }\n\n    this.#forwarder.stopProgressMonitor();\n    this.#lc.info?.('ChangeStreamer stopped');\n  }\n\n  async #handleControlMessage(msg: ChangeStreamControl[1]) {\n    this.#lc.info?.('received control message', msg);\n    const {tag} = msg;\n\n    switch (tag) {\n      case 'reset-required':\n        await markResetRequired(this.#changeDB, this.#shard);\n        await publishReplicationError(\n          this.#lc,\n          'Replicating',\n          msg.message ?? 'Resync required',\n          msg.errorDetails,\n        );\n        if (this.#autoReset) {\n          this.#lc.warn?.('shutting down for auto-reset');\n          await this.stop(new AutoResetSignal());\n        }\n        break;\n      default:\n        unreachable(tag);\n    }\n  }\n\n  subscribe(ctx: SubscriberContext): Promise<Source<string>> {\n    const {protocolVersion, id, mode, replicaVersion, watermark} = ctx;\n    if (mode === 'serving') {\n      this.#serving.resolve();\n    }\n    const downstream = Subscription.create<string>({\n      cleanup: () => this.#forwarder.remove(subscriber),\n    });\n    const subscriber = new Subscriber(\n      protocolVersion,\n      id,\n      watermark,\n      downstream,\n      () => this.#latestStatus,\n    );\n    if (replicaVersion !== this.#replicaVersion) {\n      this.#lc.warn?.(\n        `rejecting subscriber at replica version ${replicaVersion}`,\n      );\n      subscriber.close(\n        ErrorType.WrongReplicaVersion,\n        `current replica version is ${\n          this.#replicaVersion\n        } (requested ${replicaVersion})`,\n      );\n    } else {\n      this.#lc.debug?.(`adding subscriber ${subscriber.id}`);\n\n      this.#forwarder.add(subscriber);\n      this.#storer.catchup(subscriber, mode);\n    }\n    return Promise.resolve(downstream);\n  }\n\n  scheduleCleanup(watermark: string) {\n    const origSize = this.#initialWatermarks.size;\n    this.#initialWatermarks.add(watermark);\n\n    if (origSize === 0) {\n      this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n    }\n  }\n\n  async getChangeLogState(): Promise<{\n    replicaVersion: string;\n    minWatermark: string;\n  }> {\n    const minWatermark = await this.#storer.getMinWatermarkForCatchup();\n    if (!minWatermark) {\n      this.#lc.warn?.(\n        `Unexpected empty changeLog. Resync if \"Local replica watermark\" errors arise`,\n      );\n    }\n    return {\n      replicaVersion: this.#replicaVersion,\n      minWatermark: minWatermark ?? this.#replicaVersion,\n    };\n  }\n\n  /**\n   * Makes a best effort to purge the change log. In the event of a database\n   * error, exceptions will be logged and swallowed, so this method is safe\n   * to run in a timeout.\n   */\n  async #purgeOldChanges(): Promise<void> {\n    const initial = [...this.#initialWatermarks];\n    if (initial.length === 0) {\n      this.#lc.warn?.('No initial watermarks to check for cleanup'); // Not expected.\n      return;\n    }\n    const current = [...this.#forwarder.getAcks()];\n    if (current.length === 0) {\n      // Also not expected, but possible (e.g. subscriber connects, then disconnects).\n      // Bail to be safe.\n      this.#lc.warn?.('No subscribers to confirm cleanup');\n      return;\n    }\n    try {\n      const earliestInitial = min(...(initial as AtLeastOne<LexiVersion>));\n      const earliestCurrent = min(...(current as AtLeastOne<LexiVersion>));\n      if (earliestCurrent < earliestInitial) {\n        this.#lc.info?.(\n          `At least one client is behind backup (${earliestCurrent} < ${earliestInitial})`,\n        );\n      } else {\n        this.#lc.info?.(`Purging changes before ${earliestInitial} ...`);\n        const start = performance.now();\n        const deleted = await this.#storer.purgeRecordsBefore(earliestInitial);\n        const elapsed = (performance.now() - start).toFixed(2);\n        this.#lc.info?.(\n          `Purged ${deleted} changes before ${earliestInitial} (${elapsed} ms)`,\n        );\n        this.#initialWatermarks.delete(earliestInitial);\n      }\n    } catch (e) {\n      this.#lc.warn?.(`error purging change log`, e);\n    } finally {\n      if (this.#initialWatermarks.size) {\n        // If there are unpurged watermarks to check, schedule the next purge.\n        this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n      }\n    }\n  }\n\n  async stop(err?: unknown) {\n    this.#state.stop(this.#lc, err);\n    this.#stream?.changes.cancel();\n    await this.#storer.stop();\n    await this.#source.stop();\n  }\n}\n\n// The delay between receiving an initial, backup-based watermark\n// and performing a check of whether to purge records before it.\n// This delay should be long enough to handle situations like the following:\n//\n// 1. `litestream restore` downloads a backup for the `replication-manager`\n// 2. `replication-manager` starts up and runs this `change-streamer`\n// 3. `zero-cache`s that are running on a different replica connect to this\n//    `change-streamer` after exponential backoff retries.\n//\n// It is possible for a `zero-cache`[3] to be behind the backup restored [1].\n// This cleanup delay (30 seconds) is thus set to be a value comfortably\n// longer than the max delay for exponential backoff (10 seconds) in\n// `services/running-state.ts`. This allows the `zero-cache` [3] to reconnect\n// so that the `change-streamer` can track its progress and know when it has\n// surpassed the initial watermark of the backup [1].\nconst CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS * 3;\n\nconst ROLLBACK_JSON = JSON.stringify([\n  'rollback',\n  {tag: 'rollback'},\n] satisfies Rollback);\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AA+DA,eAAsB,mBACpB,IACA,OACA,QACA,kBACA,mBACA,UACA,cACA,4BACA,mBACA,WACA,WACA,MACA,eAAe,YACiB;CAEhC,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAClD,MAAM,wBACJ,IACA,UACA,mBACA,OACA,WACA,YACF;CAEA,MAAM,EAAC,mBAAkB;CACzB,OAAO,IAAI,mBACT,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,cACA,4BACA,WACA,WACA,MACA,YACF;AACF;AAEA,IAAM,8CAA8C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkKpD,IAAM,qBAAN,MAA0D;CACxD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA;CACA;CACA,qCAA8B,IAAI,IAAY;CAU9C,WAAoB,SAAS;CAE7B,aAAsB,mBACpB,eACA,gBACA,kCACF;CACA,iBAA0B,mBACxB,eACA,WACA,qDACF;CAEA;CACA;CACA;CAEA,YACE,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,QACA,4BACA,kBACA,WACA,MACA,eAAe,YACf;EACA,KAAK,KAAK;EACV,KAAKA,MAAM,GAAG,YAAY,aAAa,iBAAiB;EACxD,KAAKC,SAAS;EACd,KAAKC,YAAY;EACjB,KAAKC,kBAAkB;EACvB,KAAKC,UAAU;EACf,KAAKC,UAAU,IAAI,OACjB,IACA,OACA,QACA,kBACA,mBACA,UACA,iBACA,aAAY,KAAKS,SAAS,KAAK,KAAK;GAAC;GAAU,SAAS;GAAI,SAAS;EAAE,CAAC,IACxE,QAAO,KAAK,KAAK,GAAG,GACpB,IACF;EACA,KAAKR,aAAa,IAAI,UAAU,IAAI,EAClC,oCACE,KAAK,mCACT,CAAC;EACD,KAAKC,8BAA8B;EACnC,KAAKQ,aAAa;EAClB,KAAKP,aAAa;EAClB,KAAKC,SAAS,IAAI,aAAa,KAAK,IAAI,KAAA,GAAW,YAAY;EAC/D,KAAKO,gBAAgB,EAAC,KAAK,SAAQ;CACrC;CAEA,MAAM,MAAM;EACV,KAAKhB,IAAI,OAAO,wBAAwB;EAExC,KAAKM,WAAW,qBAAqB;EAErC,MAAM,YAAY,MAAM,KAAKF,QAAQ,iBAAiB;EACtD,IAAI,WACF,KAAKY,cAAc,YAAY;EAKjC,MAAM,KAAKX,QAAQ,gBAAgB,KAAKU,UAAU;EAClD,KAAKA,aAAa;EAIlB,MAAM,sBAAsB,wBAAwB,KAAK;EAEzD,OAAO,KAAKN,OAAO,UAAU,GAAG;GAC9B,IAAI;GACJ,IAAI,YAA2B;GAC/B,IAAI,iBAAiB;GACrB,IAAI;IACF,MAAM,EAAC,eAAe,qBACpB,MAAM,KAAKJ,QAAQ,uCAAuC;IAC5D,MAAM,SAAS,MAAM,KAAKD,QAAQ,YAChC,eACA,gBACF;IACA,KAAKC,QAAQ,IAAI,EAAE,OAAM,MAAK,OAAO,QAAQ,OAAO,CAAC,CAAC;IAEtD,KAAKS,UAAU;IACf,IACE,KAAKL,OAAO,aAAa,IACzB,6CAIA,KAAKF,4BAA4B,QAC/B,KAAKP,KACL,eACA,oBAAoB,eACtB;IAEF,YAAY;IAEZ,WAAW,MAAM,UAAU,OAAO,SAAS;KACzC,MAAM,CAAC,MAAM,OAAO;KACpB,QAAQ,MAAR;MACE,KAAK;OACH,IAAI,IAAI,KACN,KAAKK,QAAQ,OAAO,MAAM;OAE5B,IAAI,IAAI,WAAW;QAKjB,KAAKW,cAAc,YAAY,IAAI;QACnC,KAAKV,WAAW,WAAW,KAAKU,aAAa;OAC/C;OACA;MACF,KAAK;OACH,MAAM,KAAKC,sBAAsB,GAAG;OACpC;MACF,KAAK;OACH,YAAY,OAAO,GAAG;OACtB;MACF,KAAK;OACH,IAAI,cAAc,OAAO,GAAG,WAC1B,MAAM,IAAI,mBACR,oBAAoB,OAAO,GAAG,UAAU,oCAAoC,WAC9E;OAEF,KAAKL,WAAW,IAAI,CAAC;OACrB;MACF;OACE,IAAI,SAAS,QACX,KAAKC,eAAe,IAAI,CAAC;OAE3B,IAAI,cAAc,MAChB,MAAM,IAAI,mBACR,GAAG,KAAK,WAAW,IAAI,IAAI,kCAC7B;OAEF;KACJ;KAEA,MAAM,OAAO,KAAKR,QAAQ,MAAM,WAAW,MAAM;KACjD,MAAM,QAA2B;MAAC;MAAW,OAAO,GAAG;MAAK;KAAI;KAChE,kBAAkB,KAAK;KACvB,IAAI,iBAAiB,qBAEnB,KAAKC,WAAW,QAAQ,KAAK;UACxB;MAQL,MAAM,KAAKA,WAAW,uBAAuB,KAAK;MAClD,iBAAiB;KACnB;KAEA,IAAI,SAAS,YAAY,SAAS,YAChC,YAAY;KAId,MAAM,eAAe,KAAKD,QAAQ,aAAa;KAC/C,IAAI,cACF,MAAM;IAEV;GACF,SAAS,GAAG;IACV,MAAM;GACR,UAAU;IACR,KAAKS,SAAS,QAAQ,OAAO;IAC7B,KAAKA,UAAU,KAAA;GACjB;GAGA,IAAI,WAAW;IACb,KAAKd,IAAI,OAAO,oCAAoC,WAAW;IAC/D,KAAKK,QAAQ,MAAM;IACnB,KAAKC,WAAW,QAAQ;KAAC;KAAW;KAAY;IAAa,CAAC;GAChE;GAGA,MAAM,QAAQ,IAAI;IAChB,KAAKD,QAAQ,KAAK;IAClB,KAAKI,OAAO,QAAQ,KAAKT,KAAK,GAAG;IACjC,KAAKS,OAAO,aAAa,8CACrB,qBACE,KAAKT,KACL,uBAAuB,KAAKA,KAAK,eAAe,GAAG,CACrD,IACA;GACN,CAAC;EACH;EAEA,KAAKM,WAAW,oBAAoB;EACpC,KAAKN,IAAI,OAAO,wBAAwB;CAC1C;CAEA,MAAMiB,sBAAsB,KAA6B;EACvD,KAAKjB,IAAI,OAAO,4BAA4B,GAAG;EAC/C,MAAM,EAAC,QAAO;EAEd,QAAQ,KAAR;GACE,KAAK;IACH,MAAM,kBAAkB,KAAKE,WAAW,KAAKD,MAAM;IACnD,MAAM,wBACJ,KAAKD,KACL,eACA,IAAI,WAAW,mBACf,IAAI,YACN;IACA,IAAI,KAAKQ,YAAY;KACnB,KAAKR,IAAI,OAAO,8BAA8B;KAC9C,MAAM,KAAK,KAAK,IAAI,gBAAgB,CAAC;IACvC;IACA;GACF,SACE,YAAY,GAAG;EACnB;CACF;CAEA,UAAU,KAAiD;EACzD,MAAM,EAAC,iBAAiB,IAAI,MAAM,gBAAgB,cAAa;EAC/D,IAAI,SAAS,WACX,KAAKW,SAAS,QAAQ;EAExB,MAAM,aAAa,aAAa,OAAe,EAC7C,eAAe,KAAKL,WAAW,OAAO,UAAU,EAClD,CAAC;EACD,MAAM,aAAa,IAAI,WACrB,iBACA,IACA,WACA,kBACM,KAAKU,aACb;EACA,IAAI,mBAAmB,KAAKb,iBAAiB;GAC3C,KAAKH,IAAI,OACP,2CAA2C,gBAC7C;GACA,WAAW,MACT,GACA,8BACE,KAAKG,gBACN,cAAc,eAAe,EAChC;EACF,OAAO;GACL,KAAKH,IAAI,QAAQ,qBAAqB,WAAW,IAAI;GAErD,KAAKM,WAAW,IAAI,UAAU;GAC9B,KAAKD,QAAQ,QAAQ,YAAY,IAAI;EACvC;EACA,OAAO,QAAQ,QAAQ,UAAU;CACnC;CAEA,gBAAgB,WAAmB;EACjC,MAAM,WAAW,KAAKK,mBAAmB;EACzC,KAAKA,mBAAmB,IAAI,SAAS;EAErC,IAAI,aAAa,GACf,KAAKD,OAAO,iBAAiB,KAAKS,iBAAiB,GAAG,gBAAgB;CAE1E;CAEA,MAAM,oBAGH;EACD,MAAM,eAAe,MAAM,KAAKb,QAAQ,0BAA0B;EAClE,IAAI,CAAC,cACH,KAAKL,IAAI,OACP,8EACF;EAEF,OAAO;GACL,gBAAgB,KAAKG;GACrB,cAAc,gBAAgB,KAAKA;EACrC;CACF;;;;;;CAOA,MAAMe,mBAAkC;EACtC,MAAM,UAAU,CAAC,GAAG,KAAKR,kBAAkB;EAC3C,IAAI,QAAQ,WAAW,GAAG;GACxB,KAAKV,IAAI,OAAO,4CAA4C;GAC5D;EACF;EACA,MAAM,UAAU,CAAC,GAAG,KAAKM,WAAW,QAAQ,CAAC;EAC7C,IAAI,QAAQ,WAAW,GAAG;GAGxB,KAAKN,IAAI,OAAO,mCAAmC;GACnD;EACF;EACA,IAAI;GACF,MAAM,kBAAkB,IAAI,GAAI,OAAmC;GACnE,MAAM,kBAAkB,IAAI,GAAI,OAAmC;GACnE,IAAI,kBAAkB,iBACpB,KAAKA,IAAI,OACP,yCAAyC,gBAAgB,KAAK,gBAAgB,EAChF;QACK;IACL,KAAKA,IAAI,OAAO,0BAA0B,gBAAgB,KAAK;IAC/D,MAAM,QAAQ,YAAY,IAAI;IAC9B,MAAM,UAAU,MAAM,KAAKK,QAAQ,mBAAmB,eAAe;IACrE,MAAM,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;IACrD,KAAKL,IAAI,OACP,UAAU,QAAQ,kBAAkB,gBAAgB,IAAI,QAAQ,KAClE;IACA,KAAKU,mBAAmB,OAAO,eAAe;GAChD;EACF,SAAS,GAAG;GACV,KAAKV,IAAI,OAAO,4BAA4B,CAAC;EAC/C,UAAU;GACR,IAAI,KAAKU,mBAAmB,MAE1B,KAAKD,OAAO,iBAAiB,KAAKS,iBAAiB,GAAG,gBAAgB;EAE1E;CACF;CAEA,MAAM,KAAK,KAAe;EACxB,KAAKT,OAAO,KAAK,KAAKT,KAAK,GAAG;EAC9B,KAAKc,SAAS,QAAQ,OAAO;EAC7B,MAAM,KAAKT,QAAQ,KAAK;EACxB,MAAM,KAAKD,QAAQ,KAAK;CAC1B;AACF;AAiBA,IAAM,mBAAmB,6BAA6B;AAEtD,IAAM,gBAAgB,KAAK,UAAU,CACnC,YACA,EAAC,KAAK,WAAU,CAClB,CAAoB"}
+{"version":3,"file":"change-streamer-service.js","names":["#lc","#shard","#changeDB","#replicaVersion","#source","#storer","#forwarder","#replicationStatusPublisher","#autoReset","#state","#initialWatermarks","#serving","#txCounter","#changeCounter","#stream","#purgeLock","#latestStatus","#fatalSourceError","#handleControlMessage","#purgeOldChanges"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"sourcesContent":["import {getDefaultHighWaterMark} from 'node:stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {publishCriticalEvent} from '../../observability/events.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {\n  min,\n  type AtLeastOne,\n  type LexiVersion,\n} from '../../types/lexi-version.ts';\nimport type {PostgresDB} from '../../types/pg.ts';\nimport type {ShardID} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {orTimeout} from '../../types/timeout.ts';\nimport type {\n  ChangeSource,\n  ChangeStream,\n} from '../change-source/change-source.ts';\nimport {\n  type ChangeStreamControl,\n  type ChangeStreamData,\n  type Rollback,\n} from '../change-source/protocol/current/downstream.ts';\nimport {\n  publishReplicationError,\n  replicationStatusError,\n  type ReplicationStatusPublisher,\n} from '../replicator/replication-status.ts';\nimport type {SubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n  DEFAULT_MAX_RETRY_DELAY_MS,\n  RunningState,\n  UnrecoverableError,\n} from '../running-state.ts';\nimport {\n  type ChangeStreamerService,\n  type Status,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {Forwarder} from './forwarder.ts';\nimport {initChangeStreamerSchema} from './schema/init.ts';\nimport {\n  AutoResetSignal,\n  ensureReplicationConfig,\n  markResetRequired,\n} from './schema/tables.ts';\nimport {\n  Storer,\n  type PurgeLock,\n  type TuningOptions as StorerOptions,\n} from './storer.ts';\nimport {Subscriber} from './subscriber.ts';\n\nexport type TuningOptions = StorerOptions & {\n  flowControlConsensusPaddingSeconds: number;\n};\n\n/**\n * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.\n */\nexport async function initializeStreamer(\n  lc: LogContext,\n  shard: ShardID,\n  taskID: string,\n  discoveryAddress: string,\n  discoveryProtocol: string,\n  changeDB: PostgresDB,\n  changeSource: ChangeSource,\n  replicationStatusPublisher: ReplicationStatusPublisher,\n  subscriptionState: SubscriptionState,\n  purgeLock: PurgeLock | null,\n  autoReset: boolean,\n  opts: TuningOptions,\n  setTimeoutFn = setTimeout,\n): Promise<ChangeStreamerService> {\n  // Make sure the ChangeLog DB is set up.\n  await initChangeStreamerSchema(lc, changeDB, shard);\n  await ensureReplicationConfig(\n    lc,\n    changeDB,\n    subscriptionState,\n    shard,\n    autoReset,\n    setTimeoutFn,\n  );\n\n  const {replicaVersion} = subscriptionState;\n  return new ChangeStreamerImpl(\n    lc,\n    shard,\n    taskID,\n    discoveryAddress,\n    discoveryProtocol,\n    changeDB,\n    replicaVersion,\n    changeSource,\n    replicationStatusPublisher,\n    purgeLock,\n    autoReset,\n    opts,\n    setTimeoutFn,\n  );\n}\n\nconst REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS = 5000;\n\nexport type ChangeTag = ChangeStreamData[1]['tag'];\n\n/**\n * Internally all Downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering for:\n * 1. Replaying new changes in the Storer\n * 2. Filtering old changes in the Subscriber\n *\n * However, only the watermark for `Commit` messages are exposed to\n * subscribers, as that is the only semantically correct watermark to\n * use for tracking a position in a replication stream.\n *\n * Additionally, the ChangeStreamData is eagerly stringified once, after which\n * the string is passed to the changeLog and all subscribers, eliminating\n * redundant stringification and reducing GC churn.\n */\nexport type WatermarkedChange = [\n  watermark: string,\n  tag: ChangeTag,\n  json: string,\n];\n\n/**\n * Upstream-agnostic dispatch of messages in a {@link ChangeStreamMessage} to a\n * {@link Forwarder} and {@link Storer} to execute the forward-store-ack\n * procedure described in {@link ChangeStreamer}.\n *\n * ### Subscriber Catchup\n *\n * Connecting clients first need to be \"caught up\" to the current watermark\n * (from stored change log entries) before new entries are forwarded to\n * them. This is non-trivial because the replication stream may be in the\n * middle of a pending streamed Transaction for which some entries have\n * already been forwarded but are not yet committed to the store.\n *\n *\n * ```\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * | Historic changes in storage |  Pending (streamed) tx  |   Next tx\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n *                                           Replication stream\n *                                           >  >  >  >  >  >  >  >  >\n *           ^  ---> required catchup --->   ^\n * Subscriber watermark               Subscription begins\n * ```\n *\n * Preemptively buffering the changes of every pending transaction\n * would be wasteful and consume too much memory for large transactions.\n *\n * Instead, the streamer synchronously dispatches changes and subscriptions\n * to the {@link Forwarder} and the {@link Storer} such that the two\n * components are aligned as to where in the stream the subscription started.\n * The two components then coordinate catchup and handoff via the\n * {@link Subscriber} object with the following algorithm:\n *\n * * If the streamer is in the middle of a pending Transaction, the\n *   Subscriber is \"queued\" on both the Forwarder and the Storer. In this\n *   state, new changes are *not* forwarded to the Subscriber, and catchup\n *   is not yet executed.\n * * Once the commit message for the pending Transaction is processed\n *   by the Storer, it begins catchup on the Subscriber (with a READONLY\n *   snapshot so that it does not block subsequent storage operations).\n *   This catchup is thus guaranteed to load the change log entries of\n *   that last Transaction.\n * * When the Forwarder processes that same commit message, it moves the\n *   Subscriber from the \"queued\" to the \"active\" set of clients such that\n *   the Subscriber begins receiving new changes, starting from the next\n *   Transaction.\n * * The Subscriber does not forward those changes, however, if its catchup\n *   is not complete. Until then, it buffers the changes in memory.\n * * Once catchup is complete, the buffered changes are immediately sent\n *   and the Subscriber henceforth forwards changes as they are received.\n *\n * In the (common) case where the streamer is not in the middle of a pending\n * transaction when a subscription begins, the Storer begins catchup\n * immediately and the Forwarder directly adds the Subscriber to its active\n * set. However, the Subscriber still buffers any forwarded messages until\n * its catchup is complete.\n *\n * ### Watermarks and ordering\n *\n * The ChangeStreamerService depends on its {@link ChangeSource} to send\n * changes in contiguous [`begin`, `data` ..., `data`, `commit`] sequences\n * in commit order. This follows Postgres's Logical Replication Protocol\n * Message Flow:\n *\n * https://www.postgresql.org/docs/16/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW\n *\n * > The logical replication protocol sends individual transactions one by one.\n * > This means that all messages between a pair of Begin and Commit messages belong to the same transaction.\n *\n * In order to correctly replay (new) and filter (old) messages to subscribers\n * at different points in the replication stream, these changes must be assigned\n * watermarks such that they preserve the order in which they were received\n * from the ChangeSource.\n *\n * A previous implementation incorrectly derived these watermarks from the Postgres\n * Log Sequence Numbers (LSN) of each message. However, LSNs from concurrent,\n * non-conflicting transactions can overlap, which can result in a `begin` message\n * with an earlier LSN arriving after a `commit` message. For example, the\n * changes for these transactions:\n *\n * ```\n * LSN:   1     2     3  4    5   6   7     8   9      10\n * tx1: begin  data     data     data     commit\n * tx2:             begin    data    data      data  commit\n * ```\n *\n * will arrive as:\n *\n * ```\n * begin1, data2, data4, data6, commit8, begin3, data5, data7, data9, commit10\n * ```\n *\n * Thus, LSN of non-commit messages are not suitable for tracking the sorting\n * order of the replication stream.\n *\n * Instead, the ChangeStreamer uses the following algorithm for deterministic\n * catchup and filtering of changes:\n *\n * * A `commit` message is assigned to a watermark corresponding to its LSN.\n *   These are guaranteed to be in commit order by definition.\n *\n * * `begin` and `data` messages are assigned to the watermark of the\n *   preceding `commit` (the previous transaction, or the replication\n *   slot's starting LSN) plus 1. This guarantees that they will be sorted\n *   after the previously commit transaction even if their LSNs came before it.\n *   This is referred to as the `preCommitWatermark`.\n *\n * * In the ChangeLog DB, messages have a secondary sort column `pos`, which is\n *   the position of the message within its transaction, with the `begin` message\n *   starting at `0`. This guarantees that `begin` and `data` messages will be\n *   fetched in the original ChangeSource order during catchup.\n *\n * `begin` and `data` messages share the same watermark, but this is sufficient for\n * Subscriber filtering because subscribers only know about the `commit` watermarks\n * exposed in the `Downstream` `Commit` message. The Subscriber object thus compares\n * the internal watermarks of the incoming messages against the commit watermark of\n * the caller, updating the watermark at every `Commit` message that is forwarded.\n *\n * ### Cleanup\n *\n * As mentioned in the {@link ChangeStreamer} documentation: \"the ChangeStreamer\n * uses a combination of [the \"initial\", i.e. backup-derived watermark and] ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\"\n *\n * More concretely:\n *\n * * The `initial`, backup-derived watermark is the earliest to which cleanup\n *   should ever happen.\n *\n * * However, it is possible for the replica backup to be *ahead* of a connected\n *   subscriber; and if a network error causes that subscriber to retry from its\n *   last watermark, the change streamer must support it.\n *\n * Thus, before cleaning up to an `initial` backup-derived watermark, the change\n * streamer first confirms that all connected subscribers have also passed\n * that watermark.\n */\nclass ChangeStreamerImpl implements ChangeStreamerService {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #replicaVersion: string;\n  readonly #source: ChangeSource;\n  readonly #storer: Storer;\n  readonly #forwarder: Forwarder;\n  readonly #replicationStatusPublisher: ReplicationStatusPublisher;\n\n  readonly #autoReset: boolean;\n  readonly #state: RunningState;\n  readonly #initialWatermarks = new Set<string>();\n\n  // Starting the (Postgres) ChangeStream results in killing the previous\n  // Postgres subscriber, potentially creating a gap in which the old\n  // change-streamer has shut down and the new change-streamer has not yet\n  // been recognized as \"healthy\" (and thus does not get any requests).\n  //\n  // To minimize this gap, delay starting the ChangeStream until the first\n  // request from a `serving` replicator, indicating that higher level\n  // load-balancing / routing logic has begun routing requests to this task.\n  readonly #serving = resolver();\n\n  readonly #txCounter = getOrCreateCounter(\n    'replication',\n    'transactions',\n    'Count of replicated transactions',\n  );\n  readonly #changeCounter = getOrCreateCounter(\n    'replication',\n    'changes',\n    'Count of replicated changes (DML or DDL statements)',\n  );\n\n  #latestStatus: Status;\n  #purgeLock: PurgeLock | null;\n  #fatalSourceError: Error | undefined;\n  #stream: ChangeStream | undefined;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    taskID: string,\n    discoveryAddress: string,\n    discoveryProtocol: string,\n    changeDB: PostgresDB,\n    replicaVersion: string,\n    source: ChangeSource,\n    replicationStatusPublisher: ReplicationStatusPublisher,\n    initialPurgeLock: PurgeLock | null,\n    autoReset: boolean,\n    opts: TuningOptions,\n    setTimeoutFn = setTimeout,\n  ) {\n    this.id = `change-streamer`;\n    this.#lc = lc.withContext('component', 'change-streamer');\n    this.#shard = shard;\n    this.#changeDB = changeDB;\n    this.#replicaVersion = replicaVersion;\n    this.#source = source;\n    this.#storer = new Storer(\n      lc,\n      shard,\n      taskID,\n      discoveryAddress,\n      discoveryProtocol,\n      changeDB,\n      replicaVersion,\n      consumed => this.#stream?.acks.push(['status', consumed[1], consumed[2]]),\n      err => this.stop(err),\n      opts,\n    );\n    this.#forwarder = new Forwarder(lc, {\n      flowControlConsensusPaddingSeconds:\n        opts.flowControlConsensusPaddingSeconds,\n    });\n    this.#replicationStatusPublisher = replicationStatusPublisher;\n    this.#purgeLock = initialPurgeLock;\n    this.#autoReset = autoReset;\n    this.#state = new RunningState(this.id, undefined, setTimeoutFn);\n    this.#latestStatus = {tag: 'status'};\n  }\n\n  async run() {\n    this.#lc.info?.('starting change stream');\n\n    this.#forwarder.startProgressMonitor();\n\n    const lagReport = await this.#source.startLagReporter();\n    if (lagReport) {\n      this.#latestStatus.lagReport = lagReport;\n    }\n\n    // Once this change-streamer acquires \"ownership\" of the change DB,\n    // it is safe to start the storer.\n    await this.#storer.assumeOwnership(this.#purgeLock);\n    this.#purgeLock = null;\n\n    // The threshold in (estimated number of) bytes to send() on subscriber\n    // websockets before `await`-ing the I/O buffers to be ready for more.\n    const flushBytesThreshold = getDefaultHighWaterMark(false);\n\n    while (this.#state.shouldRun()) {\n      let err: unknown;\n      let watermark: string | null = null;\n      let unflushedBytes = 0;\n      try {\n        const {lastWatermark, backfillRequests} =\n          await this.#storer.getStartStreamInitializationParameters();\n        const stream = await this.#source.startStream(\n          lastWatermark,\n          backfillRequests,\n        );\n        const waitForFlowControl = async (wait: Promise<void>) => {\n          if (!stream.sourceTerminated) {\n            await wait;\n            return;\n          }\n          const sourceError = await Promise.race([\n            wait.then(() => undefined),\n            stream.sourceTerminated,\n          ]);\n          if (sourceError) {\n            this.#fatalSourceError = sourceError;\n            throw sourceError;\n          }\n        };\n        this.#storer.run().catch(e => stream.changes.cancel(e));\n\n        this.#stream = stream;\n        if (\n          this.#state.resetBackoff() >\n          REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS\n        ) {\n          // After recovering from a backoff for which a replication status\n          // error was published, publish an OK status\n          this.#replicationStatusPublisher.publish(\n            this.#lc,\n            'Replicating',\n            `Replicating from ${lastWatermark}`,\n          );\n        }\n        watermark = null;\n\n        for await (const change of stream.changes) {\n          const [type, msg] = change;\n          switch (type) {\n            case 'status':\n              if (msg.ack) {\n                this.#storer.status(change); // storer acks once it gets through its queue\n              }\n              if (msg.lagReport) {\n                // Lag reports are not stored in the cdc change log, but rather\n                // only forwarded on \"live\" connections. When a new subscriber\n                // is catching up, it is initialized with the #latestStatus\n                // from which it can measure lag while catching up.\n                this.#latestStatus.lagReport = msg.lagReport;\n                this.#forwarder.sendStatus(this.#latestStatus);\n              }\n              continue;\n            case 'control':\n              await this.#handleControlMessage(msg);\n              continue; // control messages are not stored/forwarded\n            case 'begin':\n              watermark = change[2].commitWatermark;\n              break;\n            case 'commit':\n              if (watermark !== change[2].watermark) {\n                throw new UnrecoverableError(\n                  `commit watermark ${change[2].watermark} does not match 'begin' watermark ${watermark}`,\n                );\n              }\n              this.#txCounter.add(1);\n              break;\n            default:\n              if (type === 'data') {\n                this.#changeCounter.add(1);\n              }\n              if (watermark === null) {\n                throw new UnrecoverableError(\n                  `${type} change (${msg.tag}) received before 'begin' message`,\n                );\n              }\n              break;\n          }\n\n          const json = this.#storer.store(watermark, change);\n          const entry: WatermarkedChange = [watermark, change[1].tag, json];\n          unflushedBytes += json.length;\n          if (unflushedBytes < flushBytesThreshold) {\n            // pipeline changes until flushBytesThreshold\n            this.#forwarder.forward(entry);\n          } else {\n            // Wait for messages to clear socket buffers to ensure that they\n            // make their way to subscribers. Without this `await`, the\n            // messages end up being buffered in this process, which:\n            // (1) results in memory pressure and increased GC activity\n            // (2) prevents subscribers from processing the messages as they\n            //     arrive, instead getting them in a large batch after being\n            //     idle while they were queued (causing further delays).\n            const forwarded = this.#forwarder.forwardWithFlowControl(entry);\n            await waitForFlowControl(forwarded);\n            unflushedBytes = 0;\n          }\n\n          if (type === 'commit' || type === 'rollback') {\n            watermark = null;\n          }\n\n          // Allow the storer to exert back pressure.\n          const readyForMore = this.#storer.readyForMore();\n          if (readyForMore) {\n            await waitForFlowControl(readyForMore);\n          }\n        }\n      } catch (e) {\n        err = e;\n      } finally {\n        this.#stream?.changes.cancel();\n        this.#stream = undefined;\n      }\n\n      if (this.#fatalSourceError && err === this.#fatalSourceError) {\n        // A blocked flow-control wait may not drain; restart from durable state.\n        this.#forwarder.stopProgressMonitor();\n        throw err;\n      }\n\n      // When the change stream is interrupted, abort any pending transaction.\n      if (watermark) {\n        this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);\n        this.#storer.abort();\n        this.#forwarder.forward([watermark, 'rollback', ROLLBACK_JSON]);\n      }\n\n      // Backoff and drain any pending entries in the storer before reconnecting.\n      await Promise.all([\n        this.#storer.stop(),\n        this.#state.backoff(this.#lc, err),\n        this.#state.retryDelay > REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS\n          ? publishCriticalEvent(\n              this.#lc,\n              replicationStatusError(this.#lc, 'Replicating', err),\n            )\n          : promiseVoid,\n      ]);\n    }\n\n    this.#forwarder.stopProgressMonitor();\n    this.#lc.info?.('ChangeStreamer stopped');\n  }\n\n  async #handleControlMessage(msg: ChangeStreamControl[1]) {\n    this.#lc.info?.('received control message', msg);\n    const {tag} = msg;\n\n    switch (tag) {\n      case 'reset-required':\n        await markResetRequired(this.#changeDB, this.#shard);\n        await publishReplicationError(\n          this.#lc,\n          'Replicating',\n          msg.message ?? 'Resync required',\n          msg.errorDetails,\n        );\n        if (this.#autoReset) {\n          this.#lc.warn?.('shutting down for auto-reset');\n          await this.stop(new AutoResetSignal());\n        }\n        break;\n      default:\n        unreachable(tag);\n    }\n  }\n\n  subscribe(ctx: SubscriberContext): Promise<Source<string>> {\n    const {protocolVersion, id, mode, replicaVersion, watermark} = ctx;\n    if (mode === 'serving') {\n      this.#serving.resolve();\n    }\n    const downstream = Subscription.create<string>({\n      cleanup: () => this.#forwarder.remove(subscriber),\n    });\n    const subscriber = new Subscriber(\n      protocolVersion,\n      id,\n      watermark,\n      downstream,\n      () => this.#latestStatus,\n    );\n    if (replicaVersion !== this.#replicaVersion) {\n      this.#lc.warn?.(\n        `rejecting subscriber at replica version ${replicaVersion}`,\n      );\n      subscriber.close(\n        ErrorType.WrongReplicaVersion,\n        `current replica version is ${\n          this.#replicaVersion\n        } (requested ${replicaVersion})`,\n      );\n    } else {\n      this.#lc.debug?.(`adding subscriber ${subscriber.id}`);\n\n      this.#forwarder.add(subscriber);\n      this.#storer.catchup(subscriber, mode);\n    }\n    return Promise.resolve(downstream);\n  }\n\n  scheduleCleanup(watermark: string) {\n    const origSize = this.#initialWatermarks.size;\n    this.#initialWatermarks.add(watermark);\n\n    if (origSize === 0) {\n      this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n    }\n  }\n\n  async getChangeLogState(): Promise<{\n    replicaVersion: string;\n    minWatermark: string;\n  }> {\n    const minWatermark = await this.#storer.getMinWatermarkForCatchup();\n    if (!minWatermark) {\n      this.#lc.warn?.(\n        `Unexpected empty changeLog. Resync if \"Local replica watermark\" errors arise`,\n      );\n    }\n    return {\n      replicaVersion: this.#replicaVersion,\n      minWatermark: minWatermark ?? this.#replicaVersion,\n    };\n  }\n\n  /**\n   * Makes a best effort to purge the change log. In the event of a database\n   * error, exceptions will be logged and swallowed, so this method is safe\n   * to run in a timeout.\n   */\n  async #purgeOldChanges(): Promise<void> {\n    const initial = [...this.#initialWatermarks];\n    if (initial.length === 0) {\n      this.#lc.warn?.('No initial watermarks to check for cleanup'); // Not expected.\n      return;\n    }\n    const current = [...this.#forwarder.getAcks()];\n    if (current.length === 0) {\n      // Also not expected, but possible (e.g. subscriber connects, then disconnects).\n      // Bail to be safe.\n      this.#lc.warn?.('No subscribers to confirm cleanup');\n      return;\n    }\n    try {\n      const earliestInitial = min(...(initial as AtLeastOne<LexiVersion>));\n      const earliestCurrent = min(...(current as AtLeastOne<LexiVersion>));\n      if (earliestCurrent < earliestInitial) {\n        this.#lc.info?.(\n          `At least one client is behind backup (${earliestCurrent} < ${earliestInitial})`,\n        );\n      } else {\n        this.#lc.info?.(`Purging changes before ${earliestInitial} ...`);\n        const start = performance.now();\n        const deleted = await this.#storer.purgeRecordsBefore(earliestInitial);\n        const elapsed = (performance.now() - start).toFixed(2);\n        this.#lc.info?.(\n          `Purged ${deleted} changes before ${earliestInitial} (${elapsed} ms)`,\n        );\n        this.#initialWatermarks.delete(earliestInitial);\n      }\n    } catch (e) {\n      this.#lc.warn?.(`error purging change log`, e);\n    } finally {\n      if (this.#initialWatermarks.size) {\n        // If there are unpurged watermarks to check, schedule the next purge.\n        this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n      }\n    }\n  }\n\n  async stop(err?: unknown) {\n    this.#state.stop(this.#lc, err);\n    this.#stream?.changes.cancel();\n    if (this.#fatalSourceError) {\n      // Cleanup is best-effort before intentional process replacement. A\n      // wedged Storer may be unable to stop without the process exiting.\n      await orTimeout(\n        Promise.allSettled([this.#storer.stop(), this.#source.stop()]),\n        100,\n      );\n      return;\n    }\n    await this.#storer.stop();\n    await this.#source.stop();\n  }\n}\n\n// The delay between receiving an initial, backup-based watermark\n// and performing a check of whether to purge records before it.\n// This delay should be long enough to handle situations like the following:\n//\n// 1. `litestream restore` downloads a backup for the `replication-manager`\n// 2. `replication-manager` starts up and runs this `change-streamer`\n// 3. `zero-cache`s that are running on a different replica connect to this\n//    `change-streamer` after exponential backoff retries.\n//\n// It is possible for a `zero-cache`[3] to be behind the backup restored [1].\n// This cleanup delay (30 seconds) is thus set to be a value comfortably\n// longer than the max delay for exponential backoff (10 seconds) in\n// `services/running-state.ts`. This allows the `zero-cache` [3] to reconnect\n// so that the `change-streamer` can track its progress and know when it has\n// surpassed the initial watermark of the backup [1].\nconst CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS * 3;\n\nconst ROLLBACK_JSON = JSON.stringify([\n  'rollback',\n  {tag: 'rollback'},\n] satisfies Rollback);\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AAgEA,eAAsB,mBACpB,IACA,OACA,QACA,kBACA,mBACA,UACA,cACA,4BACA,mBACA,WACA,WACA,MACA,eAAe,YACiB;CAEhC,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAClD,MAAM,wBACJ,IACA,UACA,mBACA,OACA,WACA,YACF;CAEA,MAAM,EAAC,mBAAkB;CACzB,OAAO,IAAI,mBACT,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,cACA,4BACA,WACA,WACA,MACA,YACF;AACF;AAEA,IAAM,8CAA8C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkKpD,IAAM,qBAAN,MAA0D;CACxD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA;CACA;CACA,qCAA8B,IAAI,IAAY;CAU9C,WAAoB,SAAS;CAE7B,aAAsB,mBACpB,eACA,gBACA,kCACF;CACA,iBAA0B,mBACxB,eACA,WACA,qDACF;CAEA;CACA;CACA;CACA;CAEA,YACE,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,QACA,4BACA,kBACA,WACA,MACA,eAAe,YACf;EACA,KAAK,KAAK;EACV,KAAKA,MAAM,GAAG,YAAY,aAAa,iBAAiB;EACxD,KAAKC,SAAS;EACd,KAAKC,YAAY;EACjB,KAAKC,kBAAkB;EACvB,KAAKC,UAAU;EACf,KAAKC,UAAU,IAAI,OACjB,IACA,OACA,QACA,kBACA,mBACA,UACA,iBACA,aAAY,KAAKS,SAAS,KAAK,KAAK;GAAC;GAAU,SAAS;GAAI,SAAS;EAAE,CAAC,IACxE,QAAO,KAAK,KAAK,GAAG,GACpB,IACF;EACA,KAAKR,aAAa,IAAI,UAAU,IAAI,EAClC,oCACE,KAAK,mCACT,CAAC;EACD,KAAKC,8BAA8B;EACnC,KAAKQ,aAAa;EAClB,KAAKP,aAAa;EAClB,KAAKC,SAAS,IAAI,aAAa,KAAK,IAAI,KAAA,GAAW,YAAY;EAC/D,KAAKO,gBAAgB,EAAC,KAAK,SAAQ;CACrC;CAEA,MAAM,MAAM;EACV,KAAKhB,IAAI,OAAO,wBAAwB;EAExC,KAAKM,WAAW,qBAAqB;EAErC,MAAM,YAAY,MAAM,KAAKF,QAAQ,iBAAiB;EACtD,IAAI,WACF,KAAKY,cAAc,YAAY;EAKjC,MAAM,KAAKX,QAAQ,gBAAgB,KAAKU,UAAU;EAClD,KAAKA,aAAa;EAIlB,MAAM,sBAAsB,wBAAwB,KAAK;EAEzD,OAAO,KAAKN,OAAO,UAAU,GAAG;GAC9B,IAAI;GACJ,IAAI,YAA2B;GAC/B,IAAI,iBAAiB;GACrB,IAAI;IACF,MAAM,EAAC,eAAe,qBACpB,MAAM,KAAKJ,QAAQ,uCAAuC;IAC5D,MAAM,SAAS,MAAM,KAAKD,QAAQ,YAChC,eACA,gBACF;IACA,MAAM,qBAAqB,OAAO,SAAwB;KACxD,IAAI,CAAC,OAAO,kBAAkB;MAC5B,MAAM;MACN;KACF;KACA,MAAM,cAAc,MAAM,QAAQ,KAAK,CACrC,KAAK,WAAW,KAAA,CAAS,GACzB,OAAO,gBACT,CAAC;KACD,IAAI,aAAa;MACf,KAAKa,oBAAoB;MACzB,MAAM;KACR;IACF;IACA,KAAKZ,QAAQ,IAAI,EAAE,OAAM,MAAK,OAAO,QAAQ,OAAO,CAAC,CAAC;IAEtD,KAAKS,UAAU;IACf,IACE,KAAKL,OAAO,aAAa,IACzB,6CAIA,KAAKF,4BAA4B,QAC/B,KAAKP,KACL,eACA,oBAAoB,eACtB;IAEF,YAAY;IAEZ,WAAW,MAAM,UAAU,OAAO,SAAS;KACzC,MAAM,CAAC,MAAM,OAAO;KACpB,QAAQ,MAAR;MACE,KAAK;OACH,IAAI,IAAI,KACN,KAAKK,QAAQ,OAAO,MAAM;OAE5B,IAAI,IAAI,WAAW;QAKjB,KAAKW,cAAc,YAAY,IAAI;QACnC,KAAKV,WAAW,WAAW,KAAKU,aAAa;OAC/C;OACA;MACF,KAAK;OACH,MAAM,KAAKE,sBAAsB,GAAG;OACpC;MACF,KAAK;OACH,YAAY,OAAO,GAAG;OACtB;MACF,KAAK;OACH,IAAI,cAAc,OAAO,GAAG,WAC1B,MAAM,IAAI,mBACR,oBAAoB,OAAO,GAAG,UAAU,oCAAoC,WAC9E;OAEF,KAAKN,WAAW,IAAI,CAAC;OACrB;MACF;OACE,IAAI,SAAS,QACX,KAAKC,eAAe,IAAI,CAAC;OAE3B,IAAI,cAAc,MAChB,MAAM,IAAI,mBACR,GAAG,KAAK,WAAW,IAAI,IAAI,kCAC7B;OAEF;KACJ;KAEA,MAAM,OAAO,KAAKR,QAAQ,MAAM,WAAW,MAAM;KACjD,MAAM,QAA2B;MAAC;MAAW,OAAO,GAAG;MAAK;KAAI;KAChE,kBAAkB,KAAK;KACvB,IAAI,iBAAiB,qBAEnB,KAAKC,WAAW,QAAQ,KAAK;UACxB;MASL,MAAM,mBADY,KAAKA,WAAW,uBAAuB,KAChC,CAAS;MAClC,iBAAiB;KACnB;KAEA,IAAI,SAAS,YAAY,SAAS,YAChC,YAAY;KAId,MAAM,eAAe,KAAKD,QAAQ,aAAa;KAC/C,IAAI,cACF,MAAM,mBAAmB,YAAY;IAEzC;GACF,SAAS,GAAG;IACV,MAAM;GACR,UAAU;IACR,KAAKS,SAAS,QAAQ,OAAO;IAC7B,KAAKA,UAAU,KAAA;GACjB;GAEA,IAAI,KAAKG,qBAAqB,QAAQ,KAAKA,mBAAmB;IAE5D,KAAKX,WAAW,oBAAoB;IACpC,MAAM;GACR;GAGA,IAAI,WAAW;IACb,KAAKN,IAAI,OAAO,oCAAoC,WAAW;IAC/D,KAAKK,QAAQ,MAAM;IACnB,KAAKC,WAAW,QAAQ;KAAC;KAAW;KAAY;IAAa,CAAC;GAChE;GAGA,MAAM,QAAQ,IAAI;IAChB,KAAKD,QAAQ,KAAK;IAClB,KAAKI,OAAO,QAAQ,KAAKT,KAAK,GAAG;IACjC,KAAKS,OAAO,aAAa,8CACrB,qBACE,KAAKT,KACL,uBAAuB,KAAKA,KAAK,eAAe,GAAG,CACrD,IACA;GACN,CAAC;EACH;EAEA,KAAKM,WAAW,oBAAoB;EACpC,KAAKN,IAAI,OAAO,wBAAwB;CAC1C;CAEA,MAAMkB,sBAAsB,KAA6B;EACvD,KAAKlB,IAAI,OAAO,4BAA4B,GAAG;EAC/C,MAAM,EAAC,QAAO;EAEd,QAAQ,KAAR;GACE,KAAK;IACH,MAAM,kBAAkB,KAAKE,WAAW,KAAKD,MAAM;IACnD,MAAM,wBACJ,KAAKD,KACL,eACA,IAAI,WAAW,mBACf,IAAI,YACN;IACA,IAAI,KAAKQ,YAAY;KACnB,KAAKR,IAAI,OAAO,8BAA8B;KAC9C,MAAM,KAAK,KAAK,IAAI,gBAAgB,CAAC;IACvC;IACA;GACF,SACE,YAAY,GAAG;EACnB;CACF;CAEA,UAAU,KAAiD;EACzD,MAAM,EAAC,iBAAiB,IAAI,MAAM,gBAAgB,cAAa;EAC/D,IAAI,SAAS,WACX,KAAKW,SAAS,QAAQ;EAExB,MAAM,aAAa,aAAa,OAAe,EAC7C,eAAe,KAAKL,WAAW,OAAO,UAAU,EAClD,CAAC;EACD,MAAM,aAAa,IAAI,WACrB,iBACA,IACA,WACA,kBACM,KAAKU,aACb;EACA,IAAI,mBAAmB,KAAKb,iBAAiB;GAC3C,KAAKH,IAAI,OACP,2CAA2C,gBAC7C;GACA,WAAW,MACT,GACA,8BACE,KAAKG,gBACN,cAAc,eAAe,EAChC;EACF,OAAO;GACL,KAAKH,IAAI,QAAQ,qBAAqB,WAAW,IAAI;GAErD,KAAKM,WAAW,IAAI,UAAU;GAC9B,KAAKD,QAAQ,QAAQ,YAAY,IAAI;EACvC;EACA,OAAO,QAAQ,QAAQ,UAAU;CACnC;CAEA,gBAAgB,WAAmB;EACjC,MAAM,WAAW,KAAKK,mBAAmB;EACzC,KAAKA,mBAAmB,IAAI,SAAS;EAErC,IAAI,aAAa,GACf,KAAKD,OAAO,iBAAiB,KAAKU,iBAAiB,GAAG,gBAAgB;CAE1E;CAEA,MAAM,oBAGH;EACD,MAAM,eAAe,MAAM,KAAKd,QAAQ,0BAA0B;EAClE,IAAI,CAAC,cACH,KAAKL,IAAI,OACP,8EACF;EAEF,OAAO;GACL,gBAAgB,KAAKG;GACrB,cAAc,gBAAgB,KAAKA;EACrC;CACF;;;;;;CAOA,MAAMgB,mBAAkC;EACtC,MAAM,UAAU,CAAC,GAAG,KAAKT,kBAAkB;EAC3C,IAAI,QAAQ,WAAW,GAAG;GACxB,KAAKV,IAAI,OAAO,4CAA4C;GAC5D;EACF;EACA,MAAM,UAAU,CAAC,GAAG,KAAKM,WAAW,QAAQ,CAAC;EAC7C,IAAI,QAAQ,WAAW,GAAG;GAGxB,KAAKN,IAAI,OAAO,mCAAmC;GACnD;EACF;EACA,IAAI;GACF,MAAM,kBAAkB,IAAI,GAAI,OAAmC;GACnE,MAAM,kBAAkB,IAAI,GAAI,OAAmC;GACnE,IAAI,kBAAkB,iBACpB,KAAKA,IAAI,OACP,yCAAyC,gBAAgB,KAAK,gBAAgB,EAChF;QACK;IACL,KAAKA,IAAI,OAAO,0BAA0B,gBAAgB,KAAK;IAC/D,MAAM,QAAQ,YAAY,IAAI;IAC9B,MAAM,UAAU,MAAM,KAAKK,QAAQ,mBAAmB,eAAe;IACrE,MAAM,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;IACrD,KAAKL,IAAI,OACP,UAAU,QAAQ,kBAAkB,gBAAgB,IAAI,QAAQ,KAClE;IACA,KAAKU,mBAAmB,OAAO,eAAe;GAChD;EACF,SAAS,GAAG;GACV,KAAKV,IAAI,OAAO,4BAA4B,CAAC;EAC/C,UAAU;GACR,IAAI,KAAKU,mBAAmB,MAE1B,KAAKD,OAAO,iBAAiB,KAAKU,iBAAiB,GAAG,gBAAgB;EAE1E;CACF;CAEA,MAAM,KAAK,KAAe;EACxB,KAAKV,OAAO,KAAK,KAAKT,KAAK,GAAG;EAC9B,KAAKc,SAAS,QAAQ,OAAO;EAC7B,IAAI,KAAKG,mBAAmB;GAG1B,MAAM,UACJ,QAAQ,WAAW,CAAC,KAAKZ,QAAQ,KAAK,GAAG,KAAKD,QAAQ,KAAK,CAAC,CAAC,GAC7D,GACF;GACA;EACF;EACA,MAAM,KAAKC,QAAQ,KAAK;EACxB,MAAM,KAAKD,QAAQ,KAAK;CAC1B;AACF;AAiBA,IAAM,mBAAmB,6BAA6B;AAEtD,IAAM,gBAAgB,KAAK,UAAU,CACnC,YACA,EAAC,KAAK,WAAU,CAClB,CAAoB"}

package/out/zero-cache/src/workers/connection.js

@@ -7,7 +7,7 @@ import "../../../zero-protocol/src/protocol-version.js";
 import { getOrCreateCounter } from "../observability/metrics.js";
 import { ProtocolErrorWithLevel, getLogLevel, wrapWithProtocolError } from "../types/error-with-level.js";
 import { upstreamSchema } from "../../../zero-protocol/src/up.js";
-import WebSocket$1, { createWebSocketStream } from "ws";
+import WebSocket, { createWebSocketStream } from "ws";
 import { Readable, Writable, pipeline } from "node:stream";
 //#region ../zero-cache/src/workers/connection.ts
 var DOWNSTREAM_MSG_INTERVAL_MS = 6e3;
@@ -192,7 +192,7 @@ var Connection = class {
 	}
 };
 function send(lc, ws, data, callback) {
-	if (ws.readyState === WebSocket$1.OPEN) ws.send(JSON.stringify(data), callback === "ignore-backpressure" ? void 0 : callback);
+	if (ws.readyState === WebSocket.OPEN) ws.send(JSON.stringify(data), callback === "ignore-backpressure" ? void 0 : callback);
 	else {
 		lc.debug?.(`Dropping outbound message on ws (state: ${ws.readyState})`, { dropped: data });
 		if (callback !== "ignore-backpressure") callback(new ProtocolErrorWithLevel({

package/out/zero-cache/src/workers/connection.js.map

@@ -1 +1 @@
-{"version":3,"file":"connection.js","names":["#ws","#wsID","#protocolVersion","#lc","#onClose","#messageHandler","#downstreamMsgTimer","#webSocketErrors","#handleClose","#handleError","#proxyInbound","#maybeSendPong","#closeWithError","#closed","#viewSyncerOutboundStream","#pusherOutboundStream","#handleMessage","#handleMessageResult","#closeWithThrown","#proxyOutbound","#recordWebSocketError","#lastDownstreamMsgTime"],"sources":["../../../../../zero-cache/src/workers/connection.ts"],"sourcesContent":["import {pipeline, Readable, Writable} from 'node:stream';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport type {CloseEvent, Data, ErrorEvent} from 'ws';\nimport WebSocket, {createWebSocketStream} from 'ws';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport * as valita from '../../../shared/src/valita.ts';\nimport type {ConnectedMessage} from '../../../zero-protocol/src/connect.ts';\nimport type {Downstream} from '../../../zero-protocol/src/down.ts';\nimport {ErrorKind} from '../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../zero-protocol/src/error-origin.ts';\nimport type {ErrorBody} from '../../../zero-protocol/src/error.ts';\nimport {\n  isProtocolError,\n  type ProtocolError,\n} from '../../../zero-protocol/src/error.ts';\nimport {\n  MIN_SERVER_SUPPORTED_SYNC_PROTOCOL,\n  PROTOCOL_VERSION,\n} from '../../../zero-protocol/src/protocol-version.ts';\nimport {upstreamSchema, type Upstream} from '../../../zero-protocol/src/up.ts';\nimport {getOrCreateCounter} from '../observability/metrics.ts';\nimport {\n  ProtocolErrorWithLevel,\n  getLogLevel,\n  wrapWithProtocolError,\n} from '../types/error-with-level.ts';\nimport type {Source} from '../types/streams.ts';\nimport type {ConnectParams} from './connect-params.ts';\n\nexport type HandlerResult =\n  | {\n      type: 'ok';\n    }\n  | {\n      type: 'fatal';\n      error: ErrorBody;\n    }\n  | {\n      type: 'transient';\n      errors: ErrorBody[];\n    }\n  | StreamResult;\n\nexport type StreamResult = {\n  type: 'stream';\n  source: 'viewSyncer' | 'pusher';\n  stream: Source<Downstream>;\n};\n\nexport interface MessageHandler {\n  handleMessage(msg: Upstream): Promise<HandlerResult[]>;\n}\n\n// Ensures that a downstream message is sent at least every interval, sending a\n// 'pong' if necessary. This is set to be slightly longer than the client-side\n// PING_INTERVAL of 5 seconds, so that in the common case, 'pong's are sent in\n// response to client-initiated 'ping's. However, if the inbound stream is\n// backed up because a command is taking a long time to process, the pings\n// will be stuck in the queue (i.e. back-pressured), in which case pongs will\n// be manually sent to notify the client of server liveness.\n//\n// This is equivalent to what is done for Postgres keepalives on the\n// replication stream (which can similarly be back-pressured):\n// https://github.com/rocicorp/mono/blob/f98cb369a2dbb15650328859c732db358f187ef0/packages/zero-cache/src/services/change-source/pg/logical-replication/stream.ts#L21\nconst DOWNSTREAM_MSG_INTERVAL_MS = 6_000;\nconst PROTOCOL_VERSION_ATTRIBUTE = 'protocol.version';\nconst EVENT_TYPE_ATTRIBUTE = 'event.type';\n\n/**\n * Represents a connection between the client and server.\n *\n * Handles incoming messages on the connection and dispatches\n * them to the correct service.\n *\n * Listens to the ViewSyncer and sends messages to the client.\n */\nexport class Connection {\n  readonly #ws: WebSocket;\n  readonly #wsID: string;\n  readonly #protocolVersion: number;\n  readonly #lc: LogContext;\n  readonly #onClose: () => void;\n  readonly #messageHandler: MessageHandler;\n  readonly #downstreamMsgTimer: NodeJS.Timeout | undefined;\n  readonly #webSocketErrors = getOrCreateCounter(\n    'sync',\n    'websocket.errors',\n    'Client WebSocket error events.',\n  );\n\n  #viewSyncerOutboundStream: Source<Downstream> | undefined;\n  #pusherOutboundStream: Source<Downstream> | undefined;\n  #closed = false;\n\n  constructor(\n    lc: LogContext,\n    connectParams: ConnectParams,\n    ws: WebSocket,\n    messageHandler: MessageHandler,\n    onClose: () => void,\n  ) {\n    const {clientGroupID, clientID, wsID, protocolVersion} = connectParams;\n    this.#messageHandler = messageHandler;\n\n    this.#ws = ws;\n    this.#wsID = wsID;\n    this.#protocolVersion = protocolVersion;\n\n    this.#lc = lc\n      .withContext('connection')\n      .withContext('clientID', clientID)\n      .withContext('clientGroupID', clientGroupID)\n      .withContext('wsID', wsID);\n    this.#lc.debug?.('new connection');\n    this.#onClose = onClose;\n\n    this.#ws.addEventListener('close', this.#handleClose);\n    this.#ws.addEventListener('error', this.#handleError);\n\n    this.#proxyInbound();\n    this.#downstreamMsgTimer = setInterval(\n      this.#maybeSendPong,\n      DOWNSTREAM_MSG_INTERVAL_MS / 2,\n    );\n  }\n\n  /**\n   * Checks the protocol version and errors for unsupported protocols,\n   * sending the initial `connected` response on success.\n   *\n   * This is early in the connection lifecycle because {@link #handleMessage}\n   * will only parse messages with schema(s) of supported protocol versions.\n   */\n  init(): boolean {\n    if (\n      this.#protocolVersion > PROTOCOL_VERSION ||\n      this.#protocolVersion < MIN_SERVER_SUPPORTED_SYNC_PROTOCOL\n    ) {\n      this.#closeWithError({\n        kind: ErrorKind.VersionNotSupported,\n        message: `server is at sync protocol v${PROTOCOL_VERSION} and does not support v${\n          this.#protocolVersion\n        }. The ${\n          this.#protocolVersion > PROTOCOL_VERSION ? 'server' : 'client'\n        } must be updated to a newer release.`,\n        origin: ErrorOrigin.ZeroCache,\n      });\n    } else {\n      const connectedMessage: ConnectedMessage = [\n        'connected',\n        {wsid: this.#wsID, timestamp: Date.now()},\n      ];\n      this.send(connectedMessage, 'ignore-backpressure');\n      return true;\n    }\n    return false;\n  }\n\n  close(reason: string, ...args: unknown[]) {\n    if (this.#closed) {\n      return;\n    }\n    this.#closed = true;\n    this.#lc.info?.(`closing connection: ${reason}`, ...args);\n    this.#ws.removeEventListener('close', this.#handleClose);\n    this.#ws.removeEventListener('error', this.#handleError);\n    this.#viewSyncerOutboundStream?.cancel();\n    this.#viewSyncerOutboundStream = undefined;\n    this.#pusherOutboundStream?.cancel();\n    this.#pusherOutboundStream = undefined;\n    this.#onClose();\n    if (this.#ws.readyState !== this.#ws.CLOSED) {\n      this.#ws.close();\n    }\n    clearTimeout(this.#downstreamMsgTimer);\n\n    // spin down services if we have\n    // no more client connections for the client group?\n  }\n\n  handleInitConnection(initConnectionMsg: string) {\n    return this.#handleMessage({data: initConnectionMsg});\n  }\n\n  #handleMessage = async (event: {data: Data}) => {\n    const data = event.data.toString();\n    if (this.#closed) {\n      this.#lc.debug?.('Ignoring message received after closed', data);\n      return;\n    }\n\n    let msg;\n    try {\n      const value = JSON.parse(data);\n      msg = valita.parse(value, upstreamSchema);\n    } catch (e) {\n      const errorBody = {\n        kind: ErrorKind.InvalidMessage,\n        message: String(e),\n        origin: ErrorOrigin.ZeroCache,\n      } as const;\n      this.#closeWithError(\n        errorBody,\n        new ProtocolErrorWithLevel(errorBody, 'warn'),\n      );\n      return;\n    }\n\n    try {\n      const msgType = msg[0];\n      if (msgType === 'ping') {\n        this.send(['pong', {}], 'ignore-backpressure');\n        return;\n      }\n\n      const result = await this.#messageHandler.handleMessage(msg);\n      for (const r of result) {\n        this.#handleMessageResult(r);\n      }\n    } catch (e) {\n      this.#closeWithThrown(e);\n    }\n  };\n\n  #handleMessageResult(result: HandlerResult): void {\n    switch (result.type) {\n      case 'fatal':\n        this.#closeWithError(result.error);\n        break;\n      case 'ok':\n        break;\n      case 'stream': {\n        switch (result.source) {\n          case 'viewSyncer':\n            assert(\n              this.#viewSyncerOutboundStream === undefined,\n              'Outbound stream already set for this connection!',\n            );\n            this.#viewSyncerOutboundStream = result.stream;\n            break;\n          case 'pusher':\n            assert(\n              this.#pusherOutboundStream === undefined,\n              'Outbound stream already set for this connection!',\n            );\n            this.#pusherOutboundStream = result.stream;\n            break;\n        }\n        this.#proxyOutbound(result.stream);\n        break;\n      }\n      case 'transient': {\n        for (const error of result.errors) {\n          this.sendError(error);\n        }\n      }\n    }\n  }\n\n  #handleClose = (e: CloseEvent) => {\n    const {code, reason, wasClean} = e;\n    if (!wasClean) {\n      this.#recordWebSocketError('unclean_close');\n    }\n    this.close('WebSocket close event', {code, reason, wasClean});\n  };\n\n  #handleError = (e: ErrorEvent) => {\n    this.#recordWebSocketError('error_event');\n    this.#lc.warn?.('WebSocket error event', e.message, e.error);\n  };\n\n  #recordWebSocketError(eventType: string) {\n    this.#webSocketErrors.add(1, {\n      [PROTOCOL_VERSION_ATTRIBUTE]: this.#protocolVersion,\n      [EVENT_TYPE_ATTRIBUTE]: eventType,\n    });\n  }\n\n  #proxyInbound() {\n    pipeline(\n      createWebSocketStream(this.#ws),\n      new Writable({\n        write: (data, _encoding, callback) => {\n          this.#handleMessage({data}).then(() => callback(), callback);\n        },\n      }),\n      // The done callback is not used, as #handleClose and #handleError,\n      // configured on the underlying WebSocket, provide more complete\n      // information.\n      () => {},\n    );\n  }\n\n  #proxyOutbound(outboundStream: Source<Downstream>) {\n    // Note: createWebSocketStream() is avoided here in order to control\n    //       exception handling with #closeWithThrown(). If the Writable\n    //       from createWebSocketStream() were instead used, exceptions\n    //       from the outboundStream result in the Writable closing the\n    //       the websocket before the error message can be sent.\n    pipeline(\n      Readable.from(outboundStream),\n      new Writable({\n        objectMode: true,\n        write: (downstream: Downstream, _encoding, callback) =>\n          this.send(downstream, callback),\n      }),\n      e =>\n        e\n          ? this.#closeWithThrown(e)\n          : this.close(`downstream closed by ViewSyncer`),\n    );\n  }\n\n  #closeWithThrown(e: unknown) {\n    const errorBody =\n      findProtocolError(e)?.errorBody ?? wrapWithProtocolError(e).errorBody;\n\n    this.#closeWithError(errorBody, e);\n  }\n\n  #closeWithError(errorBody: ErrorBody, thrown?: unknown) {\n    this.sendError(errorBody, thrown);\n    this.close(\n      `${errorBody.kind} (${errorBody.origin}): ${errorBody.message}`,\n      errorBody,\n    );\n  }\n\n  #lastDownstreamMsgTime = Date.now();\n\n  #maybeSendPong = () => {\n    if (Date.now() - this.#lastDownstreamMsgTime > DOWNSTREAM_MSG_INTERVAL_MS) {\n      this.#lc.debug?.('manually sending pong');\n      this.send(['pong', {}], 'ignore-backpressure');\n    }\n  };\n\n  send(\n    data: Downstream,\n    callback: ((err?: Error | null) => void) | 'ignore-backpressure',\n  ) {\n    this.#lastDownstreamMsgTime = Date.now();\n    return send(this.#lc, this.#ws, data, callback);\n  }\n\n  sendError(errorBody: ErrorBody, thrown?: unknown) {\n    sendError(this.#lc, this.#ws, errorBody, thrown);\n  }\n}\n\nexport type WebSocketLike = Pick<WebSocket, 'readyState'> & {\n  send(data: string, cb?: (err?: Error) => void): void;\n};\n\n// Exported for testing purposes.\nexport function send(\n  lc: LogContext,\n  ws: WebSocketLike,\n  data: Downstream,\n  callback: ((err?: Error | null) => void) | 'ignore-backpressure',\n) {\n  if (ws.readyState === WebSocket.OPEN) {\n    ws.send(\n      JSON.stringify(data),\n      callback === 'ignore-backpressure' ? undefined : callback,\n    );\n  } else {\n    lc.debug?.(`Dropping outbound message on ws (state: ${ws.readyState})`, {\n      dropped: data,\n    });\n    if (callback !== 'ignore-backpressure') {\n      callback(\n        new ProtocolErrorWithLevel(\n          {\n            kind: ErrorKind.Internal,\n            message: 'WebSocket closed',\n            origin: ErrorOrigin.ZeroCache,\n          },\n          'info',\n        ),\n      );\n    }\n  }\n}\n\nexport function sendError(\n  lc: LogContext,\n  ws: WebSocket,\n  errorBody: ErrorBody,\n  thrown?: unknown,\n) {\n  lc = lc.withContext('errorKind', errorBody.kind);\n\n  let logLevel: LogLevel;\n\n  // If the thrown error is a ProtocolErrorWithLevel, its explicit logLevel takes precedence\n  if (thrown instanceof ProtocolErrorWithLevel) {\n    logLevel = thrown.logLevel;\n  }\n  // Errors with errno or transient socket codes are low-level, transient I/O issues\n  // (e.g., EPIPE, ECONNRESET) and should be warnings, not errors\n  else if (\n    hasErrno(thrown) ||\n    hasTransientSocketCode(thrown) ||\n    isTransientSocketMessage(errorBody.message)\n  ) {\n    logLevel = 'warn';\n  }\n  // Fallback: check errorBody.kind for errors that weren't thrown as ProtocolErrorWithLevel\n  else if (\n    errorBody.kind === ErrorKind.ClientNotFound ||\n    errorBody.kind === ErrorKind.TransformFailed\n  ) {\n    logLevel = 'warn';\n  } else {\n    logLevel = thrown ? getLogLevel(thrown) : 'info';\n  }\n\n  lc[logLevel]?.('Sending error on WebSocket', errorBody, thrown ?? '');\n  send(lc, ws, ['error', errorBody], 'ignore-backpressure');\n}\n\nexport function findProtocolError(error: unknown): ProtocolError | undefined {\n  if (isProtocolError(error)) {\n    return error;\n  }\n  if (error instanceof Error && error.cause) {\n    return findProtocolError(error.cause);\n  }\n  return undefined;\n}\n\nfunction hasErrno(error: unknown): boolean {\n  return Boolean(\n    error &&\n    typeof error === 'object' &&\n    'errno' in error &&\n    typeof (error as {errno: unknown}).errno !== 'undefined',\n  );\n}\n\n// System error codes that indicate transient socket conditions.\n// These are checked via the `code` property on errors.\nconst TRANSIENT_SOCKET_ERROR_CODES = new Set([\n  'EPIPE',\n  'ECONNRESET',\n  'ECANCELED',\n]);\n\n// Error messages that indicate transient socket conditions but don't have\n// standard error codes (e.g., WebSocket library errors).\nconst TRANSIENT_SOCKET_MESSAGE_PATTERNS = [\n  'socket was closed while data was being compressed',\n];\n\nfunction hasTransientSocketCode(error: unknown): boolean {\n  if (!error || typeof error !== 'object') {\n    return false;\n  }\n  const maybeCode =\n    'code' in error ? String((error as {code?: unknown}).code) : undefined;\n  return Boolean(\n    maybeCode && TRANSIENT_SOCKET_ERROR_CODES.has(maybeCode.toUpperCase()),\n  );\n}\n\nfunction isTransientSocketMessage(message: string | undefined): boolean {\n  if (!message) {\n    return false;\n  }\n  const lower = message.toLowerCase();\n  return TRANSIENT_SOCKET_MESSAGE_PATTERNS.some(pattern =>\n    lower.includes(pattern),\n  );\n}\n"],"mappings":";;;;;;;;;;;;AAgEA,IAAM,6BAA6B;AACnC,IAAM,6BAA6B;AACnC,IAAM,uBAAuB;;;;;;;;;AAU7B,IAAa,aAAb,MAAwB;CACtB;CACA;CACA;CACA;CACA;CACA;CACA;CACA,mBAA4B,mBAC1B,QACA,oBACA,gCACF;CAEA;CACA;CACA,UAAU;CAEV,YACE,IACA,eACA,IACA,gBACA,SACA;EACA,MAAM,EAAC,eAAe,UAAU,MAAM,oBAAmB;EACzD,KAAKK,kBAAkB;EAEvB,KAAKL,MAAM;EACX,KAAKC,QAAQ;EACb,KAAKC,mBAAmB;EAExB,KAAKC,MAAM,GACR,YAAY,YAAY,EACxB,YAAY,YAAY,QAAQ,EAChC,YAAY,iBAAiB,aAAa,EAC1C,YAAY,QAAQ,IAAI;EAC3B,KAAKA,IAAI,QAAQ,gBAAgB;EACjC,KAAKC,WAAW;EAEhB,KAAKJ,IAAI,iBAAiB,SAAS,KAAKQ,YAAY;EACpD,KAAKR,IAAI,iBAAiB,SAAS,KAAKS,YAAY;EAEpD,KAAKC,cAAc;EACnB,KAAKJ,sBAAsB,YACzB,KAAKK,gBACL,6BAA6B,CAC/B;CACF;;;;;;;;CASA,OAAgB;EACd,IACE,KAAKT,mBAAAA,MACL,KAAKA,mBAAAA,IAEL,KAAKU,gBAAgB;GACnB,MAAM;GACN,SAAS,wDACP,KAAKV,iBACN,QACC,KAAKA,mBAAAA,KAAsC,WAAW,SACvD;GACD,QAAQ;EACV,CAAC;OACI;GACL,MAAM,mBAAqC,CACzC,aACA;IAAC,MAAM,KAAKD;IAAO,WAAW,KAAK,IAAI;GAAC,CAC1C;GACA,KAAK,KAAK,kBAAkB,qBAAqB;GACjD,OAAO;EACT;EACA,OAAO;CACT;CAEA,MAAM,QAAgB,GAAG,MAAiB;EACxC,IAAI,KAAKY,SACP;EAEF,KAAKA,UAAU;EACf,KAAKV,IAAI,OAAO,uBAAuB,UAAU,GAAG,IAAI;EACxD,KAAKH,IAAI,oBAAoB,SAAS,KAAKQ,YAAY;EACvD,KAAKR,IAAI,oBAAoB,SAAS,KAAKS,YAAY;EACvD,KAAKK,2BAA2B,OAAO;EACvC,KAAKA,4BAA4B,KAAA;EACjC,KAAKC,uBAAuB,OAAO;EACnC,KAAKA,wBAAwB,KAAA;EAC7B,KAAKX,SAAS;EACd,IAAI,KAAKJ,IAAI,eAAe,KAAKA,IAAI,QACnC,KAAKA,IAAI,MAAM;EAEjB,aAAa,KAAKM,mBAAmB;CAIvC;CAEA,qBAAqB,mBAA2B;EAC9C,OAAO,KAAKU,eAAe,EAAC,MAAM,kBAAiB,CAAC;CACtD;CAEA,iBAAiB,OAAO,UAAwB;EAC9C,MAAM,OAAO,MAAM,KAAK,SAAS;EACjC,IAAI,KAAKH,SAAS;GAChB,KAAKV,IAAI,QAAQ,0CAA0C,IAAI;GAC/D;EACF;EAEA,IAAI;EACJ,IAAI;GAEF,MAAM,MADQ,KAAK,MAAM,IACN,GAAO,cAAc;EAC1C,SAAS,GAAG;GACV,MAAM,YAAY;IAChB,MAAM;IACN,SAAS,OAAO,CAAC;IACjB,QAAQ;GACV;GACA,KAAKS,gBACH,WACA,IAAI,uBAAuB,WAAW,MAAM,CAC9C;GACA;EACF;EAEA,IAAI;GAEF,IADgB,IAAI,OACJ,QAAQ;IACtB,KAAK,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,qBAAqB;IAC7C;GACF;GAEA,MAAM,SAAS,MAAM,KAAKP,gBAAgB,cAAc,GAAG;GAC3D,KAAK,MAAM,KAAK,QACd,KAAKY,qBAAqB,CAAC;EAE/B,SAAS,GAAG;GACV,KAAKC,iBAAiB,CAAC;EACzB;CACF;CAEA,qBAAqB,QAA6B;EAChD,QAAQ,OAAO,MAAf;GACE,KAAK;IACH,KAAKN,gBAAgB,OAAO,KAAK;IACjC;GACF,KAAK,MACH;GACF,KAAK;IACH,QAAQ,OAAO,QAAf;KACE,KAAK;MACH,OACE,KAAKE,8BAA8B,KAAA,GACnC,kDACF;MACA,KAAKA,4BAA4B,OAAO;MACxC;KACF,KAAK;MACH,OACE,KAAKC,0BAA0B,KAAA,GAC/B,kDACF;MACA,KAAKA,wBAAwB,OAAO;MACpC;IACJ;IACA,KAAKI,eAAe,OAAO,MAAM;IACjC;GAEF,KAAK,aACH,KAAK,MAAM,SAAS,OAAO,QACzB,KAAK,UAAU,KAAK;EAG1B;CACF;CAEA,gBAAgB,MAAkB;EAChC,MAAM,EAAC,MAAM,QAAQ,aAAY;EACjC,IAAI,CAAC,UACH,KAAKC,sBAAsB,eAAe;EAE5C,KAAK,MAAM,yBAAyB;GAAC;GAAM;GAAQ;EAAQ,CAAC;CAC9D;CAEA,gBAAgB,MAAkB;EAChC,KAAKA,sBAAsB,aAAa;EACxC,KAAKjB,IAAI,OAAO,yBAAyB,EAAE,SAAS,EAAE,KAAK;CAC7D;CAEA,sBAAsB,WAAmB;EACvC,KAAKI,iBAAiB,IAAI,GAAG;IAC1B,6BAA6B,KAAKL;IAClC,uBAAuB;EAC1B,CAAC;CACH;CAEA,gBAAgB;EACd,SACE,sBAAsB,KAAKF,GAAG,GAC9B,IAAI,SAAS,EACX,QAAQ,MAAM,WAAW,aAAa;GACpC,KAAKgB,eAAe,EAAC,KAAI,CAAC,EAAE,WAAW,SAAS,GAAG,QAAQ;EAC7D,EACF,CAAC,SAIK,CAAC,CACT;CACF;CAEA,eAAe,gBAAoC;EAMjD,SACE,SAAS,KAAK,cAAc,GAC5B,IAAI,SAAS;GACX,YAAY;GACZ,QAAQ,YAAwB,WAAW,aACzC,KAAK,KAAK,YAAY,QAAQ;EAClC,CAAC,IACD,MACE,IACI,KAAKE,iBAAiB,CAAC,IACvB,KAAK,MAAM,iCAAiC,CACpD;CACF;CAEA,iBAAiB,GAAY;EAC3B,MAAM,YACJ,kBAAkB,CAAC,GAAG,aAAa,sBAAsB,CAAC,EAAE;EAE9D,KAAKN,gBAAgB,WAAW,CAAC;CACnC;CAEA,gBAAgB,WAAsB,QAAkB;EACtD,KAAK,UAAU,WAAW,MAAM;EAChC,KAAK,MACH,GAAG,UAAU,KAAK,IAAI,UAAU,OAAO,KAAK,UAAU,WACtD,SACF;CACF;CAEA,yBAAyB,KAAK,IAAI;CAElC,uBAAuB;EACrB,IAAI,KAAK,IAAI,IAAI,KAAKS,yBAAyB,4BAA4B;GACzE,KAAKlB,IAAI,QAAQ,uBAAuB;GACxC,KAAK,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,qBAAqB;EAC/C;CACF;CAEA,KACE,MACA,UACA;EACA,KAAKkB,yBAAyB,KAAK,IAAI;EACvC,OAAO,KAAK,KAAKlB,KAAK,KAAKH,KAAK,MAAM,QAAQ;CAChD;CAEA,UAAU,WAAsB,QAAkB;EAChD,UAAU,KAAKG,KAAK,KAAKH,KAAK,WAAW,MAAM;CACjD;AACF;AAOA,SAAgB,KACd,IACA,IACA,MACA,UACA;CACA,IAAI,GAAG,eAAe,YAAU,MAC9B,GAAG,KACD,KAAK,UAAU,IAAI,GACnB,aAAa,wBAAwB,KAAA,IAAY,QACnD;MACK;EACL,GAAG,QAAQ,2CAA2C,GAAG,WAAW,IAAI,EACtE,SAAS,KACX,CAAC;EACD,IAAI,aAAa,uBACf,SACE,IAAI,uBACF;GACE,MAAM;GACN,SAAS;GACT,QAAQ;EACV,GACA,MACF,CACF;CAEJ;AACF;AAEA,SAAgB,UACd,IACA,IACA,WACA,QACA;CACA,KAAK,GAAG,YAAY,aAAa,UAAU,IAAI;CAE/C,IAAI;CAGJ,IAAI,kBAAkB,wBACpB,WAAW,OAAO;MAIf,IACH,SAAS,MAAM,KACf,uBAAuB,MAAM,KAC7B,yBAAyB,UAAU,OAAO,GAE1C,WAAW;MAGR,IACH,UAAU,SAAS,oBACnB,UAAU,SAAS,mBAEnB,WAAW;MAEX,WAAW,SAAS,YAAY,MAAM,IAAI;CAG5C,GAAG,YAAY,8BAA8B,WAAW,UAAU,EAAE;CACpE,KAAK,IAAI,IAAI,CAAC,SAAS,SAAS,GAAG,qBAAqB;AAC1D;AAEA,SAAgB,kBAAkB,OAA2C;CAC3E,IAAI,gBAAgB,KAAK,GACvB,OAAO;CAET,IAAI,iBAAiB,SAAS,MAAM,OAClC,OAAO,kBAAkB,MAAM,KAAK;AAGxC;AAEA,SAAS,SAAS,OAAyB;CACzC,OAAO,QACL,SACA,OAAO,UAAU,YACjB,WAAW,SACX,OAAQ,MAA2B,UAAU,WAC/C;AACF;AAIA,IAAM,+BAA+B,IAAI,IAAI;CAC3C;CACA;CACA;AACF,CAAC;AAID,IAAM,oCAAoC,CACxC,mDACF;AAEA,SAAS,uBAAuB,OAAyB;CACvD,IAAI,CAAC,SAAS,OAAO,UAAU,UAC7B,OAAO;CAET,MAAM,YACJ,UAAU,QAAQ,OAAQ,MAA2B,IAAI,IAAI,KAAA;CAC/D,OAAO,QACL,aAAa,6BAA6B,IAAI,UAAU,YAAY,CAAC,CACvE;AACF;AAEA,SAAS,yBAAyB,SAAsC;CACtE,IAAI,CAAC,SACH,OAAO;CAET,MAAM,QAAQ,QAAQ,YAAY;CAClC,OAAO,kCAAkC,MAAK,YAC5C,MAAM,SAAS,OAAO,CACxB;AACF"}
+{"version":3,"file":"connection.js","names":["#ws","#wsID","#protocolVersion","#lc","#onClose","#messageHandler","#downstreamMsgTimer","#webSocketErrors","#handleClose","#handleError","#proxyInbound","#maybeSendPong","#closeWithError","#closed","#viewSyncerOutboundStream","#pusherOutboundStream","#handleMessage","#handleMessageResult","#closeWithThrown","#proxyOutbound","#recordWebSocketError","#lastDownstreamMsgTime"],"sources":["../../../../../zero-cache/src/workers/connection.ts"],"sourcesContent":["import {pipeline, Readable, Writable} from 'node:stream';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport type {CloseEvent, Data, ErrorEvent} from 'ws';\nimport WebSocket, {createWebSocketStream} from 'ws';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport * as valita from '../../../shared/src/valita.ts';\nimport type {ConnectedMessage} from '../../../zero-protocol/src/connect.ts';\nimport type {Downstream} from '../../../zero-protocol/src/down.ts';\nimport {ErrorKind} from '../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../zero-protocol/src/error-origin.ts';\nimport type {ErrorBody} from '../../../zero-protocol/src/error.ts';\nimport {\n  isProtocolError,\n  type ProtocolError,\n} from '../../../zero-protocol/src/error.ts';\nimport {\n  MIN_SERVER_SUPPORTED_SYNC_PROTOCOL,\n  PROTOCOL_VERSION,\n} from '../../../zero-protocol/src/protocol-version.ts';\nimport {upstreamSchema, type Upstream} from '../../../zero-protocol/src/up.ts';\nimport {getOrCreateCounter} from '../observability/metrics.ts';\nimport {\n  ProtocolErrorWithLevel,\n  getLogLevel,\n  wrapWithProtocolError,\n} from '../types/error-with-level.ts';\nimport type {Source} from '../types/streams.ts';\nimport type {ConnectParams} from './connect-params.ts';\n\nexport type HandlerResult =\n  | {\n      type: 'ok';\n    }\n  | {\n      type: 'fatal';\n      error: ErrorBody;\n    }\n  | {\n      type: 'transient';\n      errors: ErrorBody[];\n    }\n  | StreamResult;\n\nexport type StreamResult = {\n  type: 'stream';\n  source: 'viewSyncer' | 'pusher';\n  stream: Source<Downstream>;\n};\n\nexport interface MessageHandler {\n  handleMessage(msg: Upstream): Promise<HandlerResult[]>;\n}\n\n// Ensures that a downstream message is sent at least every interval, sending a\n// 'pong' if necessary. This is set to be slightly longer than the client-side\n// PING_INTERVAL of 5 seconds, so that in the common case, 'pong's are sent in\n// response to client-initiated 'ping's. However, if the inbound stream is\n// backed up because a command is taking a long time to process, the pings\n// will be stuck in the queue (i.e. back-pressured), in which case pongs will\n// be manually sent to notify the client of server liveness.\n//\n// This is equivalent to what is done for Postgres keepalives on the\n// replication stream (which can similarly be back-pressured):\n// https://github.com/rocicorp/mono/blob/f98cb369a2dbb15650328859c732db358f187ef0/packages/zero-cache/src/services/change-source/pg/logical-replication/stream.ts#L21\nconst DOWNSTREAM_MSG_INTERVAL_MS = 6_000;\nconst PROTOCOL_VERSION_ATTRIBUTE = 'protocol.version';\nconst EVENT_TYPE_ATTRIBUTE = 'event.type';\n\n/**\n * Represents a connection between the client and server.\n *\n * Handles incoming messages on the connection and dispatches\n * them to the correct service.\n *\n * Listens to the ViewSyncer and sends messages to the client.\n */\nexport class Connection {\n  readonly #ws: WebSocket;\n  readonly #wsID: string;\n  readonly #protocolVersion: number;\n  readonly #lc: LogContext;\n  readonly #onClose: () => void;\n  readonly #messageHandler: MessageHandler;\n  readonly #downstreamMsgTimer: NodeJS.Timeout | undefined;\n  readonly #webSocketErrors = getOrCreateCounter(\n    'sync',\n    'websocket.errors',\n    'Client WebSocket error events.',\n  );\n\n  #viewSyncerOutboundStream: Source<Downstream> | undefined;\n  #pusherOutboundStream: Source<Downstream> | undefined;\n  #closed = false;\n\n  constructor(\n    lc: LogContext,\n    connectParams: ConnectParams,\n    ws: WebSocket,\n    messageHandler: MessageHandler,\n    onClose: () => void,\n  ) {\n    const {clientGroupID, clientID, wsID, protocolVersion} = connectParams;\n    this.#messageHandler = messageHandler;\n\n    this.#ws = ws;\n    this.#wsID = wsID;\n    this.#protocolVersion = protocolVersion;\n\n    this.#lc = lc\n      .withContext('connection')\n      .withContext('clientID', clientID)\n      .withContext('clientGroupID', clientGroupID)\n      .withContext('wsID', wsID);\n    this.#lc.debug?.('new connection');\n    this.#onClose = onClose;\n\n    this.#ws.addEventListener('close', this.#handleClose);\n    this.#ws.addEventListener('error', this.#handleError);\n\n    this.#proxyInbound();\n    this.#downstreamMsgTimer = setInterval(\n      this.#maybeSendPong,\n      DOWNSTREAM_MSG_INTERVAL_MS / 2,\n    );\n  }\n\n  /**\n   * Checks the protocol version and errors for unsupported protocols,\n   * sending the initial `connected` response on success.\n   *\n   * This is early in the connection lifecycle because {@link #handleMessage}\n   * will only parse messages with schema(s) of supported protocol versions.\n   */\n  init(): boolean {\n    if (\n      this.#protocolVersion > PROTOCOL_VERSION ||\n      this.#protocolVersion < MIN_SERVER_SUPPORTED_SYNC_PROTOCOL\n    ) {\n      this.#closeWithError({\n        kind: ErrorKind.VersionNotSupported,\n        message: `server is at sync protocol v${PROTOCOL_VERSION} and does not support v${\n          this.#protocolVersion\n        }. The ${\n          this.#protocolVersion > PROTOCOL_VERSION ? 'server' : 'client'\n        } must be updated to a newer release.`,\n        origin: ErrorOrigin.ZeroCache,\n      });\n    } else {\n      const connectedMessage: ConnectedMessage = [\n        'connected',\n        {wsid: this.#wsID, timestamp: Date.now()},\n      ];\n      this.send(connectedMessage, 'ignore-backpressure');\n      return true;\n    }\n    return false;\n  }\n\n  close(reason: string, ...args: unknown[]) {\n    if (this.#closed) {\n      return;\n    }\n    this.#closed = true;\n    this.#lc.info?.(`closing connection: ${reason}`, ...args);\n    this.#ws.removeEventListener('close', this.#handleClose);\n    this.#ws.removeEventListener('error', this.#handleError);\n    this.#viewSyncerOutboundStream?.cancel();\n    this.#viewSyncerOutboundStream = undefined;\n    this.#pusherOutboundStream?.cancel();\n    this.#pusherOutboundStream = undefined;\n    this.#onClose();\n    if (this.#ws.readyState !== this.#ws.CLOSED) {\n      this.#ws.close();\n    }\n    clearTimeout(this.#downstreamMsgTimer);\n\n    // spin down services if we have\n    // no more client connections for the client group?\n  }\n\n  handleInitConnection(initConnectionMsg: string) {\n    return this.#handleMessage({data: initConnectionMsg});\n  }\n\n  #handleMessage = async (event: {data: Data}) => {\n    const data = event.data.toString();\n    if (this.#closed) {\n      this.#lc.debug?.('Ignoring message received after closed', data);\n      return;\n    }\n\n    let msg;\n    try {\n      const value = JSON.parse(data);\n      msg = valita.parse(value, upstreamSchema);\n    } catch (e) {\n      const errorBody = {\n        kind: ErrorKind.InvalidMessage,\n        message: String(e),\n        origin: ErrorOrigin.ZeroCache,\n      } as const;\n      this.#closeWithError(\n        errorBody,\n        new ProtocolErrorWithLevel(errorBody, 'warn'),\n      );\n      return;\n    }\n\n    try {\n      const msgType = msg[0];\n      if (msgType === 'ping') {\n        this.send(['pong', {}], 'ignore-backpressure');\n        return;\n      }\n\n      const result = await this.#messageHandler.handleMessage(msg);\n      for (const r of result) {\n        this.#handleMessageResult(r);\n      }\n    } catch (e) {\n      this.#closeWithThrown(e);\n    }\n  };\n\n  #handleMessageResult(result: HandlerResult): void {\n    switch (result.type) {\n      case 'fatal':\n        this.#closeWithError(result.error);\n        break;\n      case 'ok':\n        break;\n      case 'stream': {\n        switch (result.source) {\n          case 'viewSyncer':\n            assert(\n              this.#viewSyncerOutboundStream === undefined,\n              'Outbound stream already set for this connection!',\n            );\n            this.#viewSyncerOutboundStream = result.stream;\n            break;\n          case 'pusher':\n            assert(\n              this.#pusherOutboundStream === undefined,\n              'Outbound stream already set for this connection!',\n            );\n            this.#pusherOutboundStream = result.stream;\n            break;\n        }\n        this.#proxyOutbound(result.stream);\n        break;\n      }\n      case 'transient': {\n        for (const error of result.errors) {\n          this.sendError(error);\n        }\n      }\n    }\n  }\n\n  #handleClose = (e: CloseEvent) => {\n    const {code, reason, wasClean} = e;\n    if (!wasClean) {\n      this.#recordWebSocketError('unclean_close');\n    }\n    this.close('WebSocket close event', {code, reason, wasClean});\n  };\n\n  #handleError = (e: ErrorEvent) => {\n    this.#recordWebSocketError('error_event');\n    this.#lc.warn?.('WebSocket error event', e.message, e.error);\n  };\n\n  #recordWebSocketError(eventType: string) {\n    this.#webSocketErrors.add(1, {\n      [PROTOCOL_VERSION_ATTRIBUTE]: this.#protocolVersion,\n      [EVENT_TYPE_ATTRIBUTE]: eventType,\n    });\n  }\n\n  #proxyInbound() {\n    pipeline(\n      createWebSocketStream(this.#ws),\n      new Writable({\n        write: (data, _encoding, callback) => {\n          this.#handleMessage({data}).then(() => callback(), callback);\n        },\n      }),\n      // The done callback is not used, as #handleClose and #handleError,\n      // configured on the underlying WebSocket, provide more complete\n      // information.\n      () => {},\n    );\n  }\n\n  #proxyOutbound(outboundStream: Source<Downstream>) {\n    // Note: createWebSocketStream() is avoided here in order to control\n    //       exception handling with #closeWithThrown(). If the Writable\n    //       from createWebSocketStream() were instead used, exceptions\n    //       from the outboundStream result in the Writable closing the\n    //       the websocket before the error message can be sent.\n    pipeline(\n      Readable.from(outboundStream),\n      new Writable({\n        objectMode: true,\n        write: (downstream: Downstream, _encoding, callback) =>\n          this.send(downstream, callback),\n      }),\n      e =>\n        e\n          ? this.#closeWithThrown(e)\n          : this.close(`downstream closed by ViewSyncer`),\n    );\n  }\n\n  #closeWithThrown(e: unknown) {\n    const errorBody =\n      findProtocolError(e)?.errorBody ?? wrapWithProtocolError(e).errorBody;\n\n    this.#closeWithError(errorBody, e);\n  }\n\n  #closeWithError(errorBody: ErrorBody, thrown?: unknown) {\n    this.sendError(errorBody, thrown);\n    this.close(\n      `${errorBody.kind} (${errorBody.origin}): ${errorBody.message}`,\n      errorBody,\n    );\n  }\n\n  #lastDownstreamMsgTime = Date.now();\n\n  #maybeSendPong = () => {\n    if (Date.now() - this.#lastDownstreamMsgTime > DOWNSTREAM_MSG_INTERVAL_MS) {\n      this.#lc.debug?.('manually sending pong');\n      this.send(['pong', {}], 'ignore-backpressure');\n    }\n  };\n\n  send(\n    data: Downstream,\n    callback: ((err?: Error | null) => void) | 'ignore-backpressure',\n  ) {\n    this.#lastDownstreamMsgTime = Date.now();\n    return send(this.#lc, this.#ws, data, callback);\n  }\n\n  sendError(errorBody: ErrorBody, thrown?: unknown) {\n    sendError(this.#lc, this.#ws, errorBody, thrown);\n  }\n}\n\nexport type WebSocketLike = Pick<WebSocket, 'readyState'> & {\n  send(data: string, cb?: (err?: Error) => void): void;\n};\n\n// Exported for testing purposes.\nexport function send(\n  lc: LogContext,\n  ws: WebSocketLike,\n  data: Downstream,\n  callback: ((err?: Error | null) => void) | 'ignore-backpressure',\n) {\n  if (ws.readyState === WebSocket.OPEN) {\n    ws.send(\n      JSON.stringify(data),\n      callback === 'ignore-backpressure' ? undefined : callback,\n    );\n  } else {\n    lc.debug?.(`Dropping outbound message on ws (state: ${ws.readyState})`, {\n      dropped: data,\n    });\n    if (callback !== 'ignore-backpressure') {\n      callback(\n        new ProtocolErrorWithLevel(\n          {\n            kind: ErrorKind.Internal,\n            message: 'WebSocket closed',\n            origin: ErrorOrigin.ZeroCache,\n          },\n          'info',\n        ),\n      );\n    }\n  }\n}\n\nexport function sendError(\n  lc: LogContext,\n  ws: WebSocket,\n  errorBody: ErrorBody,\n  thrown?: unknown,\n) {\n  lc = lc.withContext('errorKind', errorBody.kind);\n\n  let logLevel: LogLevel;\n\n  // If the thrown error is a ProtocolErrorWithLevel, its explicit logLevel takes precedence\n  if (thrown instanceof ProtocolErrorWithLevel) {\n    logLevel = thrown.logLevel;\n  }\n  // Errors with errno or transient socket codes are low-level, transient I/O issues\n  // (e.g., EPIPE, ECONNRESET) and should be warnings, not errors\n  else if (\n    hasErrno(thrown) ||\n    hasTransientSocketCode(thrown) ||\n    isTransientSocketMessage(errorBody.message)\n  ) {\n    logLevel = 'warn';\n  }\n  // Fallback: check errorBody.kind for errors that weren't thrown as ProtocolErrorWithLevel\n  else if (\n    errorBody.kind === ErrorKind.ClientNotFound ||\n    errorBody.kind === ErrorKind.TransformFailed\n  ) {\n    logLevel = 'warn';\n  } else {\n    logLevel = thrown ? getLogLevel(thrown) : 'info';\n  }\n\n  lc[logLevel]?.('Sending error on WebSocket', errorBody, thrown ?? '');\n  send(lc, ws, ['error', errorBody], 'ignore-backpressure');\n}\n\nexport function findProtocolError(error: unknown): ProtocolError | undefined {\n  if (isProtocolError(error)) {\n    return error;\n  }\n  if (error instanceof Error && error.cause) {\n    return findProtocolError(error.cause);\n  }\n  return undefined;\n}\n\nfunction hasErrno(error: unknown): boolean {\n  return Boolean(\n    error &&\n    typeof error === 'object' &&\n    'errno' in error &&\n    typeof (error as {errno: unknown}).errno !== 'undefined',\n  );\n}\n\n// System error codes that indicate transient socket conditions.\n// These are checked via the `code` property on errors.\nconst TRANSIENT_SOCKET_ERROR_CODES = new Set([\n  'EPIPE',\n  'ECONNRESET',\n  'ECANCELED',\n]);\n\n// Error messages that indicate transient socket conditions but don't have\n// standard error codes (e.g., WebSocket library errors).\nconst TRANSIENT_SOCKET_MESSAGE_PATTERNS = [\n  'socket was closed while data was being compressed',\n];\n\nfunction hasTransientSocketCode(error: unknown): boolean {\n  if (!error || typeof error !== 'object') {\n    return false;\n  }\n  const maybeCode =\n    'code' in error ? String((error as {code?: unknown}).code) : undefined;\n  return Boolean(\n    maybeCode && TRANSIENT_SOCKET_ERROR_CODES.has(maybeCode.toUpperCase()),\n  );\n}\n\nfunction isTransientSocketMessage(message: string | undefined): boolean {\n  if (!message) {\n    return false;\n  }\n  const lower = message.toLowerCase();\n  return TRANSIENT_SOCKET_MESSAGE_PATTERNS.some(pattern =>\n    lower.includes(pattern),\n  );\n}\n"],"mappings":";;;;;;;;;;;;AAgEA,IAAM,6BAA6B;AACnC,IAAM,6BAA6B;AACnC,IAAM,uBAAuB;;;;;;;;;AAU7B,IAAa,aAAb,MAAwB;CACtB;CACA;CACA;CACA;CACA;CACA;CACA;CACA,mBAA4B,mBAC1B,QACA,oBACA,gCACF;CAEA;CACA;CACA,UAAU;CAEV,YACE,IACA,eACA,IACA,gBACA,SACA;EACA,MAAM,EAAC,eAAe,UAAU,MAAM,oBAAmB;EACzD,KAAKK,kBAAkB;EAEvB,KAAKL,MAAM;EACX,KAAKC,QAAQ;EACb,KAAKC,mBAAmB;EAExB,KAAKC,MAAM,GACR,YAAY,YAAY,EACxB,YAAY,YAAY,QAAQ,EAChC,YAAY,iBAAiB,aAAa,EAC1C,YAAY,QAAQ,IAAI;EAC3B,KAAKA,IAAI,QAAQ,gBAAgB;EACjC,KAAKC,WAAW;EAEhB,KAAKJ,IAAI,iBAAiB,SAAS,KAAKQ,YAAY;EACpD,KAAKR,IAAI,iBAAiB,SAAS,KAAKS,YAAY;EAEpD,KAAKC,cAAc;EACnB,KAAKJ,sBAAsB,YACzB,KAAKK,gBACL,6BAA6B,CAC/B;CACF;;;;;;;;CASA,OAAgB;EACd,IACE,KAAKT,mBAAAA,MACL,KAAKA,mBAAAA,IAEL,KAAKU,gBAAgB;GACnB,MAAM;GACN,SAAS,wDACP,KAAKV,iBACN,QACC,KAAKA,mBAAAA,KAAsC,WAAW,SACvD;GACD,QAAQ;EACV,CAAC;OACI;GACL,MAAM,mBAAqC,CACzC,aACA;IAAC,MAAM,KAAKD;IAAO,WAAW,KAAK,IAAI;GAAC,CAC1C;GACA,KAAK,KAAK,kBAAkB,qBAAqB;GACjD,OAAO;EACT;EACA,OAAO;CACT;CAEA,MAAM,QAAgB,GAAG,MAAiB;EACxC,IAAI,KAAKY,SACP;EAEF,KAAKA,UAAU;EACf,KAAKV,IAAI,OAAO,uBAAuB,UAAU,GAAG,IAAI;EACxD,KAAKH,IAAI,oBAAoB,SAAS,KAAKQ,YAAY;EACvD,KAAKR,IAAI,oBAAoB,SAAS,KAAKS,YAAY;EACvD,KAAKK,2BAA2B,OAAO;EACvC,KAAKA,4BAA4B,KAAA;EACjC,KAAKC,uBAAuB,OAAO;EACnC,KAAKA,wBAAwB,KAAA;EAC7B,KAAKX,SAAS;EACd,IAAI,KAAKJ,IAAI,eAAe,KAAKA,IAAI,QACnC,KAAKA,IAAI,MAAM;EAEjB,aAAa,KAAKM,mBAAmB;CAIvC;CAEA,qBAAqB,mBAA2B;EAC9C,OAAO,KAAKU,eAAe,EAAC,MAAM,kBAAiB,CAAC;CACtD;CAEA,iBAAiB,OAAO,UAAwB;EAC9C,MAAM,OAAO,MAAM,KAAK,SAAS;EACjC,IAAI,KAAKH,SAAS;GAChB,KAAKV,IAAI,QAAQ,0CAA0C,IAAI;GAC/D;EACF;EAEA,IAAI;EACJ,IAAI;GAEF,MAAM,MADQ,KAAK,MAAM,IACN,GAAO,cAAc;EAC1C,SAAS,GAAG;GACV,MAAM,YAAY;IAChB,MAAM;IACN,SAAS,OAAO,CAAC;IACjB,QAAQ;GACV;GACA,KAAKS,gBACH,WACA,IAAI,uBAAuB,WAAW,MAAM,CAC9C;GACA;EACF;EAEA,IAAI;GAEF,IADgB,IAAI,OACJ,QAAQ;IACtB,KAAK,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,qBAAqB;IAC7C;GACF;GAEA,MAAM,SAAS,MAAM,KAAKP,gBAAgB,cAAc,GAAG;GAC3D,KAAK,MAAM,KAAK,QACd,KAAKY,qBAAqB,CAAC;EAE/B,SAAS,GAAG;GACV,KAAKC,iBAAiB,CAAC;EACzB;CACF;CAEA,qBAAqB,QAA6B;EAChD,QAAQ,OAAO,MAAf;GACE,KAAK;IACH,KAAKN,gBAAgB,OAAO,KAAK;IACjC;GACF,KAAK,MACH;GACF,KAAK;IACH,QAAQ,OAAO,QAAf;KACE,KAAK;MACH,OACE,KAAKE,8BAA8B,KAAA,GACnC,kDACF;MACA,KAAKA,4BAA4B,OAAO;MACxC;KACF,KAAK;MACH,OACE,KAAKC,0BAA0B,KAAA,GAC/B,kDACF;MACA,KAAKA,wBAAwB,OAAO;MACpC;IACJ;IACA,KAAKI,eAAe,OAAO,MAAM;IACjC;GAEF,KAAK,aACH,KAAK,MAAM,SAAS,OAAO,QACzB,KAAK,UAAU,KAAK;EAG1B;CACF;CAEA,gBAAgB,MAAkB;EAChC,MAAM,EAAC,MAAM,QAAQ,aAAY;EACjC,IAAI,CAAC,UACH,KAAKC,sBAAsB,eAAe;EAE5C,KAAK,MAAM,yBAAyB;GAAC;GAAM;GAAQ;EAAQ,CAAC;CAC9D;CAEA,gBAAgB,MAAkB;EAChC,KAAKA,sBAAsB,aAAa;EACxC,KAAKjB,IAAI,OAAO,yBAAyB,EAAE,SAAS,EAAE,KAAK;CAC7D;CAEA,sBAAsB,WAAmB;EACvC,KAAKI,iBAAiB,IAAI,GAAG;IAC1B,6BAA6B,KAAKL;IAClC,uBAAuB;EAC1B,CAAC;CACH;CAEA,gBAAgB;EACd,SACE,sBAAsB,KAAKF,GAAG,GAC9B,IAAI,SAAS,EACX,QAAQ,MAAM,WAAW,aAAa;GACpC,KAAKgB,eAAe,EAAC,KAAI,CAAC,EAAE,WAAW,SAAS,GAAG,QAAQ;EAC7D,EACF,CAAC,SAIK,CAAC,CACT;CACF;CAEA,eAAe,gBAAoC;EAMjD,SACE,SAAS,KAAK,cAAc,GAC5B,IAAI,SAAS;GACX,YAAY;GACZ,QAAQ,YAAwB,WAAW,aACzC,KAAK,KAAK,YAAY,QAAQ;EAClC,CAAC,IACD,MACE,IACI,KAAKE,iBAAiB,CAAC,IACvB,KAAK,MAAM,iCAAiC,CACpD;CACF;CAEA,iBAAiB,GAAY;EAC3B,MAAM,YACJ,kBAAkB,CAAC,GAAG,aAAa,sBAAsB,CAAC,EAAE;EAE9D,KAAKN,gBAAgB,WAAW,CAAC;CACnC;CAEA,gBAAgB,WAAsB,QAAkB;EACtD,KAAK,UAAU,WAAW,MAAM;EAChC,KAAK,MACH,GAAG,UAAU,KAAK,IAAI,UAAU,OAAO,KAAK,UAAU,WACtD,SACF;CACF;CAEA,yBAAyB,KAAK,IAAI;CAElC,uBAAuB;EACrB,IAAI,KAAK,IAAI,IAAI,KAAKS,yBAAyB,4BAA4B;GACzE,KAAKlB,IAAI,QAAQ,uBAAuB;GACxC,KAAK,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,qBAAqB;EAC/C;CACF;CAEA,KACE,MACA,UACA;EACA,KAAKkB,yBAAyB,KAAK,IAAI;EACvC,OAAO,KAAK,KAAKlB,KAAK,KAAKH,KAAK,MAAM,QAAQ;CAChD;CAEA,UAAU,WAAsB,QAAkB;EAChD,UAAU,KAAKG,KAAK,KAAKH,KAAK,WAAW,MAAM;CACjD;AACF;AAOA,SAAgB,KACd,IACA,IACA,MACA,UACA;CACA,IAAI,GAAG,eAAe,UAAU,MAC9B,GAAG,KACD,KAAK,UAAU,IAAI,GACnB,aAAa,wBAAwB,KAAA,IAAY,QACnD;MACK;EACL,GAAG,QAAQ,2CAA2C,GAAG,WAAW,IAAI,EACtE,SAAS,KACX,CAAC;EACD,IAAI,aAAa,uBACf,SACE,IAAI,uBACF;GACE,MAAM;GACN,SAAS;GACT,QAAQ;EACV,GACA,MACF,CACF;CAEJ;AACF;AAEA,SAAgB,UACd,IACA,IACA,WACA,QACA;CACA,KAAK,GAAG,YAAY,aAAa,UAAU,IAAI;CAE/C,IAAI;CAGJ,IAAI,kBAAkB,wBACpB,WAAW,OAAO;MAIf,IACH,SAAS,MAAM,KACf,uBAAuB,MAAM,KAC7B,yBAAyB,UAAU,OAAO,GAE1C,WAAW;MAGR,IACH,UAAU,SAAS,oBACnB,UAAU,SAAS,mBAEnB,WAAW;MAEX,WAAW,SAAS,YAAY,MAAM,IAAI;CAG5C,GAAG,YAAY,8BAA8B,WAAW,UAAU,EAAE;CACpE,KAAK,IAAI,IAAI,CAAC,SAAS,SAAS,GAAG,qBAAqB;AAC1D;AAEA,SAAgB,kBAAkB,OAA2C;CAC3E,IAAI,gBAAgB,KAAK,GACvB,OAAO;CAET,IAAI,iBAAiB,SAAS,MAAM,OAClC,OAAO,kBAAkB,MAAM,KAAK;AAGxC;AAEA,SAAS,SAAS,OAAyB;CACzC,OAAO,QACL,SACA,OAAO,UAAU,YACjB,WAAW,SACX,OAAQ,MAA2B,UAAU,WAC/C;AACF;AAIA,IAAM,+BAA+B,IAAI,IAAI;CAC3C;CACA;CACA;AACF,CAAC;AAID,IAAM,oCAAoC,CACxC,mDACF;AAEA,SAAS,uBAAuB,OAAyB;CACvD,IAAI,CAAC,SAAS,OAAO,UAAU,UAC7B,OAAO;CAET,MAAM,YACJ,UAAU,QAAQ,OAAQ,MAA2B,IAAI,IAAI,KAAA;CAC/D,OAAO,QACL,aAAa,6BAA6B,IAAI,UAAU,YAAY,CAAC,CACvE;AACF;AAEA,SAAS,yBAAyB,SAAsC;CACtE,IAAI,CAAC,SACH,OAAO;CAET,MAAM,QAAQ,QAAQ,YAAY;CAClC,OAAO,kCAAkC,MAAK,YAC5C,MAAM,SAAS,OAAO,CACxB;AACF"}

package/out/zero-client/src/client/version.js

@@ -2,7 +2,7 @@
 /**
 * The current version of Zero.
 */
-var version = "1.9.0-canary.3";
+var version = "1.9.0-canary.4";
 //#endregion
 export { version };
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rocicorp/zero",
-  "version": "1.9.0-canary.3",
+  "version": "1.9.0-canary.4",
   "description": "Zero is a web framework for serverless web development.",
   "homepage": "https://zero.rocicorp.dev",
   "bugs": {