@rocicorp/zero 1.11.0-canary.36 → 1.11.0-canary.37

package/out/zero-cache/src/services/change-streamer/change-streamer-http.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-http.js","names":["#lc","#opts","#changeStreamer","#changes","#reserveSnapshot","#subscribe","#receiveWebsocket","#ensureChangeStreamerStarted","#forwardReservation","#changeStreamerStarted","#shardID","#changeDB","#changeStreamerURI","#resolveChangeStreamer"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport websocket from '@fastify/websocket';\nimport type {LogContext} from '@rocicorp/logger';\nimport WebSocket from 'ws';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport type {IncomingMessageSubset} from '../../types/http.ts';\nimport {pgClient, type PostgresDB} from '../../types/pg.ts';\nimport {type Worker} from '../../types/processes.ts';\nimport {type ShardID} from '../../types/shards.ts';\nimport {\n  readHead,\n  streamInternal,\n  streamInternalStringified,\n  streamInternalWithSize,\n  type Sized,\n  type Source,\n} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {URLParams} from '../../types/url-params.ts';\nimport {installWebSocketReceiver} from '../../types/websocket-handoff.ts';\nimport {closeWithError, PROTOCOL_ERROR} from '../../types/ws.ts';\nimport {HttpService, type Options as HttpOptions} from '../http-service.ts';\nimport {handleProfzRequest} from '../profz.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport {\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type ChangeStreamerService,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport {discoverChangeStreamerAddress} from './schema/tables.ts';\nimport {type SnapshotMessage} from './snapshot-message.ts';\nimport {\n  subscribeDownstreamSchema,\n  subscribeUpstreamSchema,\n  type ReservedMessage,\n  type SnapshotStatus,\n  type SubscribeContext,\n  type SubscribeDownstream,\n  type SubscribeUpstream,\n} from './subscribe.ts';\n\nconst MIN_SUPPORTED_PROTOCOL_VERSION = 4;\n\n// The merged `/subscribe` protocol was introduced at v7.\nconst MIN_SUBSCRIBE_PROTOCOL_VERSION = 7;\n\nconst SNAPSHOT_PATH_PATTERN = '/replication/:version/snapshot';\nconst CHANGES_PATH_PATTERN = '/replication/:version/changes';\nconst SUBSCRIBE_PATH_PATTERN = '/replication/:version/subscribe';\nconst PATH_REGEX =\n  /\\/replication\\/v(?<version>\\d+)\\/(changes|snapshot|subscribe)$/;\n\nconst SUBSCRIBE_PATH = `/replication/v${PROTOCOL_VERSION}/subscribe`;\n\n// Max time for a client to establish the /subscribe websocket (TCP connect\n// plus upgrade) before abandoning the attempt.\nconst CONNECT_TIMEOUT_MS = 10_000;\n\ntype Options = HttpOptions & {\n  startupDelayMs: number;\n  config?:\n    | Pick<NormalizedZeroConfig, 'adminPassword' | 'operatorPassword'>\n    | undefined;\n  getProfileWorker?: (() => Promise<Worker>) | undefined;\n};\n\nexport class ChangeStreamerHttpServer extends HttpService {\n  readonly id = 'change-streamer-http-server';\n  readonly #lc: LogContext;\n  readonly #opts: Options;\n  readonly #changeStreamer: ChangeStreamerService;\n\n  constructor(\n    lc: LogContext,\n    opts: Options,\n    parent: Worker,\n    changeStreamer: ChangeStreamerService,\n  ) {\n    super('change-streamer-http-server', lc, opts, async fastify => {\n      await fastify.register(websocket);\n\n      fastify.get(CHANGES_PATH_PATTERN, {websocket: true}, this.#changes);\n      fastify.get(\n        SNAPSHOT_PATH_PATTERN,\n        {websocket: true},\n        this.#reserveSnapshot,\n      );\n      fastify.get(SUBSCRIBE_PATH_PATTERN, {websocket: true}, this.#subscribe);\n\n      fastify.get('/profz', (req, res) =>\n        handleProfzRequest(\n          lc,\n          opts.config ?? {\n            adminPassword: undefined,\n            operatorPassword: undefined,\n          },\n          req,\n          res,\n          opts.getProfileWorker,\n          undefined,\n          'change-streamer',\n        ),\n      );\n\n      installWebSocketReceiver<'snapshot' | 'changes' | 'subscribe'>(\n        lc,\n        fastify.websocketServer,\n        this.#receiveWebsocket,\n        parent,\n      );\n    });\n\n    this.#lc = lc;\n    this.#opts = opts;\n    this.#changeStreamer = changeStreamer;\n  }\n\n  // Called when receiving a web socket via the main dispatcher handoff.\n  readonly #receiveWebsocket = (\n    ws: WebSocket,\n    action: 'changes' | 'snapshot' | 'subscribe',\n    msg: IncomingMessageSubset,\n  ) => {\n    switch (action) {\n      case 'snapshot':\n        return this.#reserveSnapshot(ws, msg);\n      case 'changes':\n        return this.#changes(ws, msg);\n      case 'subscribe':\n        return this.#subscribe(ws, msg);\n      default:\n        closeWithError(\n          this._lc,\n          ws,\n          `invalid action \"${action}\" received in handoff`,\n        );\n        return;\n    }\n  };\n\n  readonly #reserveSnapshot = async (ws: WebSocket, req: RequestHeaders) => {\n    this.#ensureChangeStreamerStarted('incoming snapshot reservation');\n    try {\n      const url = new URL(\n        req.url ?? '',\n        req.headers.origin ?? 'http://localhost',\n      );\n      checkProtocolVersion(url.pathname);\n      const taskID = url.searchParams.get('taskID');\n      if (!taskID) {\n        throw new Error('Missing taskID in snapshot request');\n      }\n      const downstream =\n        await this.#changeStreamer.startSnapshotReservation(taskID);\n      // Send-only for now: the reservation carries no inbound application\n      // messages (only transport acks, demuxed internally).\n      void streamInternal(this._lc, ws, undefined, downstream);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  readonly #changes = async (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const ctx = getSubscriberContext(req);\n      if (ctx.mode === 'serving') {\n        this.#ensureChangeStreamerStarted('incoming subscription');\n      }\n\n      const downstream = await this.#changeStreamer.subscribe(ctx);\n      // Send-only for now: the subscription carries no inbound application\n      // messages (only transport acks, demuxed internally).\n      void streamInternalStringified(this._lc, ws, undefined, downstream, {\n        batched: ctx.wsBatched,\n      });\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  /**\n   * The merged v7 `/subscribe` handler. A single bidirectional connection\n   * carries, in sequence: an optional reservation phase (`reserve-snapshot` →\n   * `reserved`) and then the subscription phase (`start-subscription` → change\n   * stream). This ensures that the subscriber subscribes to the same task from\n   * which it restored the backup.\n   */\n  readonly #subscribe = async (ws: WebSocket, req: RequestHeaders) => {\n    let reservation: Source<SnapshotMessage> | undefined;\n    const outbound = Subscription.create<string | PreSerializedBatch>();\n\n    try {\n      const url = new URL(\n        req.url ?? '',\n        req.headers.origin ?? 'http://localhost',\n      );\n      const protocolVersion = checkProtocolVersion(url.pathname);\n      if (protocolVersion < MIN_SUBSCRIBE_PROTOCOL_VERSION) {\n        throw new Error(\n          `the /subscribe endpoint requires protocol v${MIN_SUBSCRIBE_PROTOCOL_VERSION} ` +\n            `(client is at v${protocolVersion})`,\n        );\n      }\n\n      const instream = await streamInternalStringified(\n        this._lc,\n        ws,\n        subscribeUpstreamSchema,\n        outbound,\n        {batched: true},\n      );\n\n      // Keep reading upstream control messages for the life of the connection.\n      // Do not manually exit loop, as that would close the socket. The loop\n      // exits naturally when the connection closes.\n      for await (const msg of instream) {\n        if (msg[0] === 'reserve-snapshot') {\n          const {taskID} = msg[1];\n          this.#ensureChangeStreamerStarted('incoming snapshot reservation');\n          reservation =\n            await this.#changeStreamer.startSnapshotReservation(taskID);\n          void this.#forwardReservation(reservation, outbound);\n        } else if (msg[0] === 'start-subscription') {\n          const ctx: SubscriberContext = {\n            ...msg[1],\n            protocolVersion,\n            wsBatched: true,\n          };\n          if (ctx.mode === 'serving') {\n            this.#ensureChangeStreamerStarted('incoming subscription');\n          }\n          // subscribe() registers the subscriber and then releases any\n          // reservation for that task.\n          reservation = undefined;\n          await this.#changeStreamer.subscribe(ctx, outbound);\n        }\n      }\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    } finally {\n      // Covers a reservation-phase disconnect (instream ended before a\n      // subscription started); idempotent with the close handler above.\n      reservation?.cancel();\n    }\n  };\n\n  /**\n   * Forwards a snapshot reservation's single `['status', ...]` message onto\n   * the merged connection's `outbound` sink as a `['reserved', ...]` message.\n   * The loop then blocks on the held-open reservation until it is cancelled,\n   * by `subscribe()` at the subscription handoff, or by the reservation-phase\n   * teardown on disconnect.\n   */\n  async #forwardReservation(\n    reservation: Source<SnapshotMessage>,\n    outbound: Subscription<string | PreSerializedBatch>,\n  ) {\n    try {\n      for await (const [, status] of reservation) {\n        // Translate the /snapshot protocol's ['status', {tag: 'status', ...}]\n        // message to the /subscribe protocol's ['reserved', {tag: 'snapshot', ...}].\n        outbound.push(\n          BigIntJSON.stringify([\n            'reserved',\n            {...status, tag: 'snapshot'},\n          ] satisfies ReservedMessage),\n        );\n      }\n    } catch (e) {\n      this._lc.warn?.(`error forwarding snapshot reservation`, e);\n      outbound.fail(e instanceof Error ? e : new Error(String(e)));\n    }\n  }\n\n  #changeStreamerStarted = false;\n\n  #ensureChangeStreamerStarted(reason?: string) {\n    if (!this.#changeStreamerStarted && this._state.shouldRun()) {\n      this.#lc.info?.(\n        `starting ChangeStreamerService ${reason ? `(${reason})` : ''}`,\n      );\n      void this.#changeStreamer\n        .run()\n        .catch(e =>\n          this.#lc.warn?.(`ChangeStreamerService ended with error`, e),\n        )\n        .finally(() => this.stop());\n\n      this.#changeStreamerStarted = true;\n    }\n  }\n\n  protected override _onStart(): void {\n    const {startupDelayMs, readinessGate = promiseVoid} = this.#opts;\n    if (startupDelayMs === 0) {\n      // In RMv2, there is no need to delay starting the change streamer\n      // because RM startup is non-disruptive.\n      this.#ensureChangeStreamerStarted();\n    } else {\n      // In RMv1, starting the change-streamer forcibly shuts down the\n      // previous change-streamer, causing view-syncers to reconnect.\n      // If this replication-manager has just started, the routing layer may\n      // not have registered it with DNS, as that only happens after it\n      // confirms health checks. To minimize downtime, the takeover is\n      // delayed for the configured startupDelayMs _after_ beginning to\n      // advertise readiness.\n      void readinessGate.then(() => {\n        this.#lc.info?.(\n          `waiting ${startupDelayMs}ms before taking over the change log`,\n        );\n        this._state.setTimeout(\n          () =>\n            this.#ensureChangeStreamerStarted(\n              `startup delay elapsed (${startupDelayMs} ms)`,\n            ),\n          startupDelayMs,\n        );\n      });\n    }\n  }\n\n  protected override async _onStop(): Promise<void> {\n    if (this.#changeStreamerStarted) {\n      await this.#changeStreamer.stop();\n    }\n  }\n}\n\nexport interface SnapshotReserver {\n  reserveSnapshot(taskID: string): Promise<{\n    reserved: SnapshotStatus;\n    followup: ReservationFollowup;\n  }>;\n}\n\nexport interface SubscriptionStarter {\n  subscribe(\n    ctx: SubscriberContext,\n  ): Promise<Source<Sized<SubscribeDownstream>>>;\n}\n\nexport interface ReservationFollowup extends SubscriptionStarter {\n  /**\n   * A signal to indicate that the followup is no longer valid (i.e. the\n   * connection to the reserving task has been severed).\n   */\n  readonly signal: AbortSignal;\n\n  /**\n   * Abandons the reservation without subscribing, closing the connection that\n   * was held open to pin the change-log floor. Used when the caller cannot\n   * proceed to `subscribe()` (e.g. process shutdown while a restore is in\n   * progress) and must not leave the reservation's connection open forever.\n   */\n  cancel(reason?: Error): void;\n}\n\nexport class ChangeStreamerHttpClient\n  implements ChangeStreamer, SnapshotReserver, SubscriptionStarter\n{\n  readonly #lc: LogContext;\n  readonly #shardID: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #changeStreamerURI: string | undefined;\n\n  constructor(\n    lc: LogContext,\n    shardID: ShardID,\n    changeDB: string,\n    changeStreamerURI: string | undefined,\n  ) {\n    this.#lc = lc;\n    this.#shardID = shardID;\n    // Create a pg client with a single short-lived connection for the purpose\n    // of change-streamer discovery (i.e. ChangeDB as DNS).\n    this.#changeDB = pgClient(lc, changeDB, 'change-streamer-discovery', {\n      max: 1,\n      ['idle_timeout']: 15,\n    });\n    this.#changeStreamerURI = changeStreamerURI;\n  }\n\n  async #resolveChangeStreamer(path: string) {\n    let baseURL = this.#changeStreamerURI;\n    if (!baseURL) {\n      const address = await discoverChangeStreamerAddress(\n        this.#shardID,\n        this.#changeDB,\n      );\n      if (!address) {\n        throw new Error(`no change-streamer is running`);\n      }\n      baseURL = address.includes('://') ? `${address}/` : `ws://${address}/`;\n    }\n    const uri = new URL(path, baseURL);\n    this.#lc.info?.(`connecting to change-streamer@${uri}`);\n    return uri;\n  }\n\n  async reserveSnapshot(taskID: string): Promise<{\n    reserved: SnapshotStatus;\n    followup: ReservationFollowup;\n  }> {\n    const {instream, outbound} = await this.connect(taskID);\n    outbound.push(['reserve-snapshot', {taskID}]);\n\n    const {head, rest} = readHead(instream);\n    const item = await head.next();\n    if (!item.value) {\n      throw new Error('stream closed without snapshot reservation');\n    }\n    const {data: msg} = item.value;\n    if (msg[0] !== 'reserved') {\n      throw new Error(`expected \"reserved\" message but got ${msg[0]}`);\n    }\n    const reserved: SnapshotStatus = msg[1];\n    const followup: ReservationFollowup = {\n      subscribe: ctx => {\n        outbound.push(['start-subscription', toStartSubscriptionContext(ctx)]);\n        return Promise.resolve(rest);\n      },\n      signal: outbound.signal,\n      cancel: reason => outbound.cancel(reason),\n    };\n\n    return {reserved, followup};\n  }\n\n  async subscribe(\n    ctx: SubscriberContext,\n  ): Promise<Source<Sized<SubscribeDownstream>>> {\n    const {instream, outbound} = await this.connect(ctx.taskID);\n    outbound.push(['start-subscription', toStartSubscriptionContext(ctx)]);\n    return instream;\n  }\n\n  /**\n   * Opens a merged v7 `/subscribe` connection (see subscribe.ts). Returns the\n   * inbound change/reservation stream and an `outbound` sink into which the\n   * caller pushes the `reserve-snapshot` / `start-subscription` control\n   * messages that drive the connection's phases. The single connection spans\n   * the reservation, backup restore, and subscription, so the reserving\n   * replication-manager is the one that serves the subscription.\n   */\n  // exported for testing\n  async connect(taskID: string): Promise<{\n    instream: Source<Sized<SubscribeDownstream>>;\n    outbound: Subscription<SubscribeUpstream>;\n  }> {\n    const uri = await this.#resolveChangeStreamer(SUBSCRIBE_PATH);\n\n    // taskID is carried in the query for observability/routing; the server\n    // reads the authoritative taskID from the control messages themselves.\n    const params = new URLSearchParams({taskID});\n    // Bound the TCP connect + HTTP upgrade. Without this, a SYN black-holed\n    // by a stale Service endpoint (e.g. a just-terminated replication-manager\n    // pod) is retransmitted by the kernel for ~2 minutes before ETIMEDOUT.\n    // Failing fast lets the caller's retry open a fresh connection that is\n    // routed to a live endpoint.\n    const ws = new WebSocket(uri + `?${params.toString()}`, {\n      handshakeTimeout: CONNECT_TIMEOUT_MS,\n    });\n\n    const outbound = Subscription.create<SubscribeUpstream>();\n    const instream = await streamInternalWithSize(\n      this.#lc,\n      ws,\n      subscribeDownstreamSchema,\n      outbound,\n    );\n    return {instream, outbound};\n  }\n}\n\ntype RequestHeaders = Pick<IncomingMessage, 'url' | 'headers'>;\n\nexport function getSubscriberContext(req: RequestHeaders): SubscriberContext {\n  const url = new URL(req.url ?? '', req.headers.origin ?? 'http://localhost');\n  const protocolVersion = checkProtocolVersion(url.pathname);\n  const params = new URLParams(url);\n\n  return {\n    protocolVersion,\n    id: params.get('id', true),\n    taskID: params.get('taskID', true),\n    mode: params.get('mode', false) === 'backup' ? 'backup' : 'serving',\n    replicaVersion: params.get('replicaVersion', true),\n    watermark: params.get('watermark', true),\n    // Absent for subscribers that predate the parameter, which is the safe\n    // default: the barrier falls back to polling rather than waiting on an\n    // ACK that would never be attributed to a writer.\n    wsBatched: params.getBoolean('wsBatched'),\n  };\n}\n\nfunction checkProtocolVersion(pathname: string): number {\n  const match = PATH_REGEX.exec(pathname);\n  if (!match) {\n    throw new Error(`invalid path: ${pathname}`);\n  }\n  const v = Number(match.groups?.version);\n  if (\n    Number.isNaN(v) ||\n    v > PROTOCOL_VERSION ||\n    v < MIN_SUPPORTED_PROTOCOL_VERSION\n  ) {\n    throw new Error(\n      `Cannot service client at protocol v${v}. ` +\n        `Supported protocols: [v${MIN_SUPPORTED_PROTOCOL_VERSION} ... v${PROTOCOL_VERSION}]`,\n    );\n  }\n  return v;\n}\n\n// Projects a client-side SubscriberContext onto the start-subscription\n// message payload, dropping the fields that are implicit on the merged\n// connection: protocolVersion (from the request path), wsBatched (always on).\nfunction toStartSubscriptionContext(ctx: SubscriberContext): SubscribeContext {\n  const {taskID, id, mode, replicaVersion, watermark, backfills} = ctx;\n  return {taskID, id, mode, replicaVersion, watermark, backfills};\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AA4CA,IAAM,iCAAiC;AAGvC,IAAM,iCAAiC;AAEvC,IAAM,wBAAwB;AAC9B,IAAM,uBAAuB;AAC7B,IAAM,yBAAyB;AAC/B,IAAM,aACJ;AAEF,IAAM,iBAAiB;AAIvB,IAAM,qBAAqB;AAU3B,IAAa,2BAAb,cAA8C,YAAY;CACxD,KAAc;CACd;CACA;CACA;CAEA,YACE,IACA,MACA,QACA,gBACA;EACA,MAAM,+BAA+B,IAAI,MAAM,OAAM,YAAW;GAC9D,MAAM,QAAQ,SAAS,SAAS;GAEhC,QAAQ,IAAI,sBAAsB,EAAC,WAAW,KAAI,GAAG,KAAKG,QAAQ;GAClE,QAAQ,IACN,uBACA,EAAC,WAAW,KAAI,GAChB,KAAKC,gBACP;GACA,QAAQ,IAAI,wBAAwB,EAAC,WAAW,KAAI,GAAG,KAAKC,UAAU;GAEtE,QAAQ,IAAI,WAAW,KAAK,QAC1B,mBACE,IACA,KAAK,UAAU;IACb,eAAe,KAAA;IACf,kBAAkB,KAAA;GACpB,GACA,KACA,KACA,KAAK,kBACL,KAAA,GACA,iBACF,CACF;GAEA,yBACE,IACA,QAAQ,iBACR,KAAKC,mBACL,MACF;EACF,CAAC;EAED,KAAKN,MAAM;EACX,KAAKC,QAAQ;EACb,KAAKC,kBAAkB;CACzB;CAGA,qBACE,IACA,QACA,QACG;EACH,QAAQ,QAAR;GACE,KAAK,YACH,OAAO,KAAKE,iBAAiB,IAAI,GAAG;GACtC,KAAK,WACH,OAAO,KAAKD,SAAS,IAAI,GAAG;GAC9B,KAAK,aACH,OAAO,KAAKE,WAAW,IAAI,GAAG;GAChC;IACE,eACE,KAAK,KACL,IACA,mBAAmB,OAAO,sBAC5B;IACA;EACJ;CACF;CAEA,mBAA4B,OAAO,IAAe,QAAwB;EACxE,KAAKE,6BAA6B,+BAA+B;EACjE,IAAI;GACF,MAAM,MAAM,IAAI,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,kBACxB;GACA,qBAAqB,IAAI,QAAQ;GACjC,MAAM,SAAS,IAAI,aAAa,IAAI,QAAQ;GAC5C,IAAI,CAAC,QACH,MAAM,IAAI,MAAM,oCAAoC;GAEtD,MAAM,aACJ,MAAM,KAAKL,gBAAgB,yBAAyB,MAAM;GAG5D,eAAoB,KAAK,KAAK,IAAI,KAAA,GAAW,UAAU;EACzD,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;CAEA,WAAoB,OAAO,IAAe,QAAwB;EAChE,IAAI;GACF,MAAM,MAAM,qBAAqB,GAAG;GACpC,IAAI,IAAI,SAAS,WACf,KAAKK,6BAA6B,uBAAuB;GAG3D,MAAM,aAAa,MAAM,KAAKL,gBAAgB,UAAU,GAAG;GAG3D,0BAA+B,KAAK,KAAK,IAAI,KAAA,GAAW,YAAY,EAClE,SAAS,IAAI,UACf,CAAC;EACH,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;;;;;;;;CASA,aAAsB,OAAO,IAAe,QAAwB;EAClE,IAAI;EACJ,MAAM,WAAW,aAAa,OAAoC;EAElE,IAAI;GAKF,MAAM,kBAAkB,qBAAqB,IAJ7B,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,kBAEqB,EAAI,QAAQ;GACzD,IAAI,kBAAkB,gCACpB,MAAM,IAAI,MACR,8CAA8C,+BAA+B,kBACzD,gBAAgB,EACtC;GAGF,MAAM,WAAW,MAAM,0BACrB,KAAK,KACL,IACA,yBACA,UACA,EAAC,SAAS,KAAI,CAChB;GAKA,WAAW,MAAM,OAAO,UACtB,IAAI,IAAI,OAAO,oBAAoB;IACjC,MAAM,EAAC,WAAU,IAAI;IACrB,KAAKK,6BAA6B,+BAA+B;IACjE,cACE,MAAM,KAAKL,gBAAgB,yBAAyB,MAAM;IAC5D,KAAUM,oBAAoB,aAAa,QAAQ;GACrD,OAAO,IAAI,IAAI,OAAO,sBAAsB;IAC1C,MAAM,MAAyB;KAC7B,GAAG,IAAI;KACP;KACA,WAAW;IACb;IACA,IAAI,IAAI,SAAS,WACf,KAAKD,6BAA6B,uBAAuB;IAI3D,cAAc,KAAA;IACd,MAAM,KAAKL,gBAAgB,UAAU,KAAK,QAAQ;GACpD;EAEJ,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD,UAAU;GAGR,aAAa,OAAO;EACtB;CACF;;;;;;;;CASA,MAAMM,oBACJ,aACA,UACA;EACA,IAAI;GACF,WAAW,MAAM,GAAG,WAAW,aAG7B,SAAS,KACP,WAAW,UAAU,CACnB,YACA;IAAC,GAAG;IAAQ,KAAK;GAAU,CAC7B,CAA2B,CAC7B;EAEJ,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,yCAAyC,CAAC;GAC1D,SAAS,KAAK,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,CAAC,CAAC,CAAC;EAC7D;CACF;CAEA,yBAAyB;CAEzB,6BAA6B,QAAiB;EAC5C,IAAI,CAAC,KAAKC,0BAA0B,KAAK,OAAO,UAAU,GAAG;GAC3D,KAAKT,IAAI,OACP,kCAAkC,SAAS,IAAI,OAAO,KAAK,IAC7D;GACA,KAAUE,gBACP,IAAI,EACJ,OAAM,MACL,KAAKF,IAAI,OAAO,0CAA0C,CAAC,CAC7D,EACC,cAAc,KAAK,KAAK,CAAC;GAE5B,KAAKS,yBAAyB;EAChC;CACF;CAEA,WAAoC;EAClC,MAAM,EAAC,gBAAgB,gBAAgB,gBAAe,KAAKR;EAC3D,IAAI,mBAAmB,GAGrB,KAAKM,6BAA6B;OASlC,cAAmB,WAAW;GAC5B,KAAKP,IAAI,OACP,WAAW,eAAe,qCAC5B;GACA,KAAK,OAAO,iBAER,KAAKO,6BACH,0BAA0B,eAAe,KAC3C,GACF,cACF;EACF,CAAC;CAEL;CAEA,MAAyB,UAAyB;EAChD,IAAI,KAAKE,wBACP,MAAM,KAAKP,gBAAgB,KAAK;CAEpC;AACF;AA+BA,IAAa,2BAAb,MAEA;CACE;CACA;CACA;CACA;CAEA,YACE,IACA,SACA,UACA,mBACA;EACA,KAAKF,MAAM;EACX,KAAKU,WAAW;EAGhB,KAAKC,YAAY,SAAS,IAAI,UAAU,6BAA6B;GACnE,KAAK;IACJ,iBAAiB;EACpB,CAAC;EACD,KAAKC,qBAAqB;CAC5B;CAEA,MAAMC,uBAAuB,MAAc;EACzC,IAAI,UAAU,KAAKD;EACnB,IAAI,CAAC,SAAS;GACZ,MAAM,UAAU,MAAM,8BACpB,KAAKF,UACL,KAAKC,SACP;GACA,IAAI,CAAC,SACH,MAAM,IAAI,MAAM,+BAA+B;GAEjD,UAAU,QAAQ,SAAS,KAAK,IAAI,GAAG,QAAQ,KAAK,QAAQ,QAAQ;EACtE;EACA,MAAM,MAAM,IAAI,IAAI,MAAM,OAAO;EACjC,KAAKX,IAAI,OAAO,iCAAiC,KAAK;EACtD,OAAO;CACT;CAEA,MAAM,gBAAgB,QAGnB;EACD,MAAM,EAAC,UAAU,aAAY,MAAM,KAAK,QAAQ,MAAM;EACtD,SAAS,KAAK,CAAC,oBAAoB,EAAC,OAAM,CAAC,CAAC;EAE5C,MAAM,EAAC,MAAM,SAAQ,SAAS,QAAQ;EACtC,MAAM,OAAO,MAAM,KAAK,KAAK;EAC7B,IAAI,CAAC,KAAK,OACR,MAAM,IAAI,MAAM,4CAA4C;EAE9D,MAAM,EAAC,MAAM,QAAO,KAAK;EACzB,IAAI,IAAI,OAAO,YACb,MAAM,IAAI,MAAM,uCAAuC,IAAI,IAAI;EAYjE,OAAO;GAAC,UAVyB,IAAI;GAUnB,UAAA;IARhB,YAAW,QAAO;KAChB,SAAS,KAAK,CAAC,sBAAsB,2BAA2B,GAAG,CAAC,CAAC;KACrE,OAAO,QAAQ,QAAQ,IAAI;IAC7B;IACA,QAAQ,SAAS;IACjB,SAAQ,WAAU,SAAS,OAAO,MAAM;GAGxB;EAAQ;CAC5B;CAEA,MAAM,UACJ,KAC6C;EAC7C,MAAM,EAAC,UAAU,aAAY,MAAM,KAAK,QAAQ,IAAI,MAAM;EAC1D,SAAS,KAAK,CAAC,sBAAsB,2BAA2B,GAAG,CAAC,CAAC;EACrE,OAAO;CACT;;;;;;;;;CAWA,MAAM,QAAQ,QAGX;EAWD,MAAM,KAAK,IAAI,YAAU,MAVP,KAAKa,uBAAuB,cAAc,IAU7B,IAAI,IANhB,gBAAgB,EAAC,OAAM,CAMP,EAAO,SAAS,KAAK,EACtD,kBAAkB,mBACpB,CAAC;EAED,MAAM,WAAW,aAAa,OAA0B;EAOxD,OAAO;GAAC,UAAA,MANe,uBACrB,KAAKb,KACL,IACA,2BACA,QACF;GACkB;EAAQ;CAC5B;AACF;AAIA,SAAgB,qBAAqB,KAAwC;CAC3E,MAAM,MAAM,IAAI,IAAI,IAAI,OAAO,IAAI,IAAI,QAAQ,UAAU,kBAAkB;CAC3E,MAAM,kBAAkB,qBAAqB,IAAI,QAAQ;CACzD,MAAM,SAAS,IAAI,UAAU,GAAG;CAEhC,OAAO;EACL;EACA,IAAI,OAAO,IAAI,MAAM,IAAI;EACzB,QAAQ,OAAO,IAAI,UAAU,IAAI;EACjC,MAAM,OAAO,IAAI,QAAQ,KAAK,MAAM,WAAW,WAAW;EAC1D,gBAAgB,OAAO,IAAI,kBAAkB,IAAI;EACjD,WAAW,OAAO,IAAI,aAAa,IAAI;EAIvC,WAAW,OAAO,WAAW,WAAW;CAC1C;AACF;AAEA,SAAS,qBAAqB,UAA0B;CACtD,MAAM,QAAQ,WAAW,KAAK,QAAQ;CACtC,IAAI,CAAC,OACH,MAAM,IAAI,MAAM,iBAAiB,UAAU;CAE7C,MAAM,IAAI,OAAO,MAAM,QAAQ,OAAO;CACtC,IACE,OAAO,MAAM,CAAC,KACd,IAAA,KACA,IAAI,gCAEJ,MAAM,IAAI,MACR,sCAAsC,EAAE,2BACZ,+BAA+B,SAC7D;CAEF,OAAO;AACT;AAKA,SAAS,2BAA2B,KAA0C;CAC5E,MAAM,EAAC,QAAQ,IAAI,MAAM,gBAAgB,WAAW,cAAa;CACjE,OAAO;EAAC;EAAQ;EAAI;EAAM;EAAgB;EAAW;CAAS;AAChE"}
+{"version":3,"file":"change-streamer-http.js","names":["#lc","#opts","#changeStreamer","#changes","#reserveSnapshot","#subscribe","#receiveWebsocket","#ensureChangeStreamerStarted","#forwardReservation","#changeStreamerStarted","#shardID","#changeDB","#changeStreamerURI","#resolveChangeStreamer"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport websocket from '@fastify/websocket';\nimport type {LogContext} from '@rocicorp/logger';\nimport WebSocket from 'ws';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport type {IncomingMessageSubset} from '../../types/http.ts';\nimport {pgClient, type PostgresDB} from '../../types/pg.ts';\nimport {type Worker} from '../../types/processes.ts';\nimport {type ShardID} from '../../types/shards.ts';\nimport {\n  readHead,\n  streamInternal,\n  streamInternalStringified,\n  streamInternalWithSize,\n  type Sized,\n  type Source,\n} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {URLParams} from '../../types/url-params.ts';\nimport {installWebSocketReceiver} from '../../types/websocket-handoff.ts';\nimport {closeWithError, PROTOCOL_ERROR} from '../../types/ws.ts';\nimport {HttpService, type Options as HttpOptions} from '../http-service.ts';\nimport {handleProfzRequest} from '../profz.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport {\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type ChangeStreamerService,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport {discoverChangeStreamerAddress} from './schema/tables.ts';\nimport {type SnapshotMessage} from './snapshot-message.ts';\nimport {\n  type LocalReplica,\n  subscribeDownstreamSchema,\n  subscribeUpstreamSchema,\n  type ReservedMessage,\n  type SnapshotStatus,\n  type SubscribeContext,\n  type SubscribeDownstream,\n  type SubscribeUpstream,\n} from './subscribe.ts';\n\nconst MIN_SUPPORTED_PROTOCOL_VERSION = 4;\n\n// The merged `/subscribe` protocol was introduced at v7.\nconst MIN_SUBSCRIBE_PROTOCOL_VERSION = 7;\n\nconst SNAPSHOT_PATH_PATTERN = '/replication/:version/snapshot';\nconst CHANGES_PATH_PATTERN = '/replication/:version/changes';\nconst SUBSCRIBE_PATH_PATTERN = '/replication/:version/subscribe';\nconst PATH_REGEX =\n  /\\/replication\\/v(?<version>\\d+)\\/(changes|snapshot|subscribe)$/;\n\nconst SUBSCRIBE_PATH = `/replication/v${PROTOCOL_VERSION}/subscribe`;\n\n// Max time for a client to establish the /subscribe websocket (TCP connect\n// plus upgrade) before abandoning the attempt.\nconst CONNECT_TIMEOUT_MS = 10_000;\n\ntype Options = HttpOptions & {\n  startupDelayMs: number;\n  config?:\n    | Pick<NormalizedZeroConfig, 'adminPassword' | 'operatorPassword'>\n    | undefined;\n  getProfileWorker?: (() => Promise<Worker>) | undefined;\n};\n\nexport class ChangeStreamerHttpServer extends HttpService {\n  readonly id = 'change-streamer-http-server';\n  readonly #lc: LogContext;\n  readonly #opts: Options;\n  readonly #changeStreamer: ChangeStreamerService;\n\n  constructor(\n    lc: LogContext,\n    opts: Options,\n    parent: Worker,\n    changeStreamer: ChangeStreamerService,\n  ) {\n    super('change-streamer-http-server', lc, opts, async fastify => {\n      await fastify.register(websocket);\n\n      fastify.get(CHANGES_PATH_PATTERN, {websocket: true}, this.#changes);\n      fastify.get(\n        SNAPSHOT_PATH_PATTERN,\n        {websocket: true},\n        this.#reserveSnapshot,\n      );\n      fastify.get(SUBSCRIBE_PATH_PATTERN, {websocket: true}, this.#subscribe);\n\n      fastify.get('/profz', (req, res) =>\n        handleProfzRequest(\n          lc,\n          opts.config ?? {\n            adminPassword: undefined,\n            operatorPassword: undefined,\n          },\n          req,\n          res,\n          opts.getProfileWorker,\n          undefined,\n          'change-streamer',\n        ),\n      );\n\n      installWebSocketReceiver<'snapshot' | 'changes' | 'subscribe'>(\n        lc,\n        fastify.websocketServer,\n        this.#receiveWebsocket,\n        parent,\n      );\n    });\n\n    this.#lc = lc;\n    this.#opts = opts;\n    this.#changeStreamer = changeStreamer;\n  }\n\n  // Called when receiving a web socket via the main dispatcher handoff.\n  readonly #receiveWebsocket = (\n    ws: WebSocket,\n    action: 'changes' | 'snapshot' | 'subscribe',\n    msg: IncomingMessageSubset,\n  ) => {\n    switch (action) {\n      case 'snapshot':\n        return this.#reserveSnapshot(ws, msg);\n      case 'changes':\n        return this.#changes(ws, msg);\n      case 'subscribe':\n        return this.#subscribe(ws, msg);\n      default:\n        closeWithError(\n          this._lc,\n          ws,\n          `invalid action \"${action}\" received in handoff`,\n        );\n        return;\n    }\n  };\n\n  readonly #reserveSnapshot = async (ws: WebSocket, req: RequestHeaders) => {\n    this.#ensureChangeStreamerStarted('incoming snapshot reservation');\n    try {\n      const url = new URL(\n        req.url ?? '',\n        req.headers.origin ?? 'http://localhost',\n      );\n      checkProtocolVersion(url.pathname);\n      const taskID = url.searchParams.get('taskID');\n      if (!taskID) {\n        throw new Error('Missing taskID in snapshot request');\n      }\n      const downstream =\n        await this.#changeStreamer.startSnapshotReservation(taskID);\n      // Send-only for now: the reservation carries no inbound application\n      // messages (only transport acks, demuxed internally).\n      void streamInternal(this._lc, ws, undefined, downstream);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  readonly #changes = async (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const ctx = getSubscriberContext(req);\n      if (ctx.mode === 'serving') {\n        this.#ensureChangeStreamerStarted('incoming subscription');\n      }\n\n      const downstream = await this.#changeStreamer.subscribe(ctx);\n      // Send-only for now: the subscription carries no inbound application\n      // messages (only transport acks, demuxed internally).\n      void streamInternalStringified(this._lc, ws, undefined, downstream, {\n        batched: ctx.wsBatched,\n      });\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  /**\n   * The merged v7 `/subscribe` handler. A single bidirectional connection\n   * carries, in sequence: an optional reservation phase (`reserve-snapshot` →\n   * `reserved`) and then the subscription phase (`start-subscription` → change\n   * stream). This ensures that the subscriber subscribes to the same task from\n   * which it restored the backup.\n   */\n  readonly #subscribe = async (ws: WebSocket, req: RequestHeaders) => {\n    let reservation: Source<SnapshotMessage> | undefined;\n    const outbound = Subscription.create<string | PreSerializedBatch>();\n\n    try {\n      const url = new URL(\n        req.url ?? '',\n        req.headers.origin ?? 'http://localhost',\n      );\n      const protocolVersion = checkProtocolVersion(url.pathname);\n      if (protocolVersion < MIN_SUBSCRIBE_PROTOCOL_VERSION) {\n        throw new Error(\n          `the /subscribe endpoint requires protocol v${MIN_SUBSCRIBE_PROTOCOL_VERSION} ` +\n            `(client is at v${protocolVersion})`,\n        );\n      }\n\n      const instream = await streamInternalStringified(\n        this._lc,\n        ws,\n        subscribeUpstreamSchema,\n        outbound,\n        {batched: true},\n      );\n\n      // Keep reading upstream control messages for the life of the connection.\n      // Do not manually exit loop, as that would close the socket. The loop\n      // exits naturally when the connection closes.\n      for await (const msg of instream) {\n        if (msg[0] === 'reserve-snapshot') {\n          const {taskID, localReplica} = msg[1];\n          this.#ensureChangeStreamerStarted('incoming snapshot reservation');\n          reservation = await this.#changeStreamer.startSnapshotReservation(\n            taskID,\n            {localReplica},\n          );\n          void this.#forwardReservation(reservation, outbound);\n        } else if (msg[0] === 'start-subscription') {\n          const ctx: SubscriberContext = {\n            ...msg[1],\n            protocolVersion,\n            wsBatched: true,\n          };\n          if (ctx.mode === 'serving') {\n            this.#ensureChangeStreamerStarted('incoming subscription');\n          }\n          // subscribe() registers the subscriber and then releases any\n          // reservation for that task.\n          reservation = undefined;\n          await this.#changeStreamer.subscribe(ctx, outbound);\n        }\n      }\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    } finally {\n      // Covers a reservation-phase disconnect (instream ended before a\n      // subscription started); idempotent with the close handler above.\n      reservation?.cancel();\n    }\n  };\n\n  /**\n   * Forwards a snapshot reservation's single `['status', ...]` message onto\n   * the merged connection's `outbound` sink as a `['reserved', ...]` message.\n   * The loop then blocks on the held-open reservation until it is cancelled,\n   * by `subscribe()` at the subscription handoff, or by the reservation-phase\n   * teardown on disconnect.\n   */\n  async #forwardReservation(\n    reservation: Source<SnapshotMessage>,\n    outbound: Subscription<string | PreSerializedBatch>,\n  ) {\n    try {\n      for await (const [, status] of reservation) {\n        // Translate the /snapshot protocol's ['status', {tag: 'status', ...}]\n        // message to the /subscribe protocol's ['reserved', {tag: 'snapshot', ...}].\n        outbound.push(\n          BigIntJSON.stringify([\n            'reserved',\n            {...status, tag: 'snapshot'},\n          ] satisfies ReservedMessage),\n        );\n      }\n    } catch (e) {\n      this._lc.warn?.(`error forwarding snapshot reservation`, e);\n      outbound.fail(e instanceof Error ? e : new Error(String(e)));\n    }\n  }\n\n  #changeStreamerStarted = false;\n\n  #ensureChangeStreamerStarted(reason?: string) {\n    if (!this.#changeStreamerStarted && this._state.shouldRun()) {\n      this.#lc.info?.(\n        `starting ChangeStreamerService ${reason ? `(${reason})` : ''}`,\n      );\n      void this.#changeStreamer\n        .run()\n        .catch(e =>\n          this.#lc.warn?.(`ChangeStreamerService ended with error`, e),\n        )\n        .finally(() => this.stop());\n\n      this.#changeStreamerStarted = true;\n    }\n  }\n\n  protected override _onStart(): void {\n    const {startupDelayMs, readinessGate = promiseVoid} = this.#opts;\n    if (startupDelayMs === 0) {\n      // In RMv2, there is no need to delay starting the change streamer\n      // because RM startup is non-disruptive.\n      this.#ensureChangeStreamerStarted();\n    } else {\n      // In RMv1, starting the change-streamer forcibly shuts down the\n      // previous change-streamer, causing view-syncers to reconnect.\n      // If this replication-manager has just started, the routing layer may\n      // not have registered it with DNS, as that only happens after it\n      // confirms health checks. To minimize downtime, the takeover is\n      // delayed for the configured startupDelayMs _after_ beginning to\n      // advertise readiness.\n      void readinessGate.then(() => {\n        this.#lc.info?.(\n          `waiting ${startupDelayMs}ms before taking over the change log`,\n        );\n        this._state.setTimeout(\n          () =>\n            this.#ensureChangeStreamerStarted(\n              `startup delay elapsed (${startupDelayMs} ms)`,\n            ),\n          startupDelayMs,\n        );\n      });\n    }\n  }\n\n  /**\n   * Signals (drainable) subscribers to switch over to another\n   * replication-manager, resolving when they have disconnected. Subscribers\n   * continue to be served in the meantime.\n   */\n  async drain(): Promise<void> {\n    if (this.#changeStreamerStarted && this.#changeStreamer.drain) {\n      await this.#changeStreamer.drain();\n    }\n  }\n\n  protected override async _onStop(): Promise<void> {\n    if (this.#changeStreamerStarted) {\n      await this.#changeStreamer.stop();\n    }\n  }\n}\n\nexport interface SnapshotReserver {\n  reserveSnapshot(\n    taskID: string,\n    localReplica?: LocalReplica | undefined,\n  ): Promise<{\n    reserved: SnapshotStatus;\n    followup: ReservationFollowup;\n  }>;\n}\n\nexport interface SubscriptionStarter {\n  subscribe(\n    ctx: SubscriberContext,\n  ): Promise<Source<Sized<SubscribeDownstream>>>;\n}\n\nexport interface ReservationFollowup extends SubscriptionStarter {\n  /**\n   * A signal to indicate that the followup is no longer valid (i.e. the\n   * connection to the reserving task has been severed).\n   */\n  readonly signal: AbortSignal;\n\n  /**\n   * Abandons the reservation without subscribing, closing the connection that\n   * was held open to pin the change-log floor. Used when the caller cannot\n   * proceed to `subscribe()` (e.g. process shutdown while a restore is in\n   * progress) and must not leave the reservation's connection open forever.\n   */\n  cancel(reason?: Error): void;\n}\n\nexport class ChangeStreamerHttpClient\n  implements ChangeStreamer, SnapshotReserver, SubscriptionStarter\n{\n  readonly #lc: LogContext;\n  readonly #shardID: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #changeStreamerURI: string | undefined;\n\n  constructor(\n    lc: LogContext,\n    shardID: ShardID,\n    changeDB: string,\n    changeStreamerURI: string | undefined,\n  ) {\n    this.#lc = lc;\n    this.#shardID = shardID;\n    // Create a pg client with a single short-lived connection for the purpose\n    // of change-streamer discovery (i.e. ChangeDB as DNS).\n    this.#changeDB = pgClient(lc, changeDB, 'change-streamer-discovery', {\n      max: 1,\n      ['idle_timeout']: 15,\n    });\n    this.#changeStreamerURI = changeStreamerURI;\n  }\n\n  async #resolveChangeStreamer(path: string) {\n    let baseURL = this.#changeStreamerURI;\n    if (!baseURL) {\n      const address = await discoverChangeStreamerAddress(\n        this.#shardID,\n        this.#changeDB,\n      );\n      if (!address) {\n        throw new Error(`no change-streamer is running`);\n      }\n      baseURL = address.includes('://') ? `${address}/` : `ws://${address}/`;\n    }\n    const uri = new URL(path, baseURL);\n    this.#lc.info?.(`connecting to change-streamer@${uri}`);\n    return uri;\n  }\n\n  async reserveSnapshot(\n    taskID: string,\n    localReplica?: LocalReplica | undefined,\n  ): Promise<{\n    reserved: SnapshotStatus;\n    followup: ReservationFollowup;\n  }> {\n    const {instream, outbound} = await this.connect(taskID);\n    outbound.push([\n      'reserve-snapshot',\n      {taskID, ...(localReplica ? {localReplica} : {})},\n    ]);\n\n    const {head, rest} = readHead(instream);\n    const item = await head.next();\n    if (!item.value) {\n      throw new Error('stream closed without snapshot reservation');\n    }\n    const {data: msg} = item.value;\n    if (msg[0] !== 'reserved') {\n      throw new Error(`expected \"reserved\" message but got ${msg[0]}`);\n    }\n    const reserved: SnapshotStatus = msg[1];\n    const followup: ReservationFollowup = {\n      subscribe: ctx => {\n        outbound.push(['start-subscription', toStartSubscriptionContext(ctx)]);\n        return Promise.resolve(rest);\n      },\n      signal: outbound.signal,\n      cancel: reason => outbound.cancel(reason),\n    };\n\n    return {reserved, followup};\n  }\n\n  async subscribe(\n    ctx: SubscriberContext,\n  ): Promise<Source<Sized<SubscribeDownstream>>> {\n    const {instream, outbound} = await this.connect(ctx.taskID);\n    outbound.push(['start-subscription', toStartSubscriptionContext(ctx)]);\n    return instream;\n  }\n\n  /**\n   * Opens a merged v7 `/subscribe` connection (see subscribe.ts). Returns the\n   * inbound change/reservation stream and an `outbound` sink into which the\n   * caller pushes the `reserve-snapshot` / `start-subscription` control\n   * messages that drive the connection's phases. The single connection spans\n   * the reservation, backup restore, and subscription, so the reserving\n   * replication-manager is the one that serves the subscription.\n   */\n  // exported for testing\n  async connect(taskID: string): Promise<{\n    instream: Source<Sized<SubscribeDownstream>>;\n    outbound: Subscription<SubscribeUpstream>;\n  }> {\n    const uri = await this.#resolveChangeStreamer(SUBSCRIBE_PATH);\n\n    // taskID is carried in the query for observability/routing; the server\n    // reads the authoritative taskID from the control messages themselves.\n    const params = new URLSearchParams({taskID});\n    // Bound the TCP connect + HTTP upgrade. Without this, a SYN black-holed\n    // by a stale Service endpoint (e.g. a just-terminated replication-manager\n    // pod) is retransmitted by the kernel for ~2 minutes before ETIMEDOUT.\n    // Failing fast lets the caller's retry open a fresh connection that is\n    // routed to a live endpoint.\n    const ws = new WebSocket(uri + `?${params.toString()}`, {\n      handshakeTimeout: CONNECT_TIMEOUT_MS,\n    });\n\n    const outbound = Subscription.create<SubscribeUpstream>();\n    const instream = await streamInternalWithSize(\n      this.#lc,\n      ws,\n      subscribeDownstreamSchema,\n      outbound,\n    );\n    return {instream, outbound};\n  }\n}\n\ntype RequestHeaders = Pick<IncomingMessage, 'url' | 'headers'>;\n\nexport function getSubscriberContext(req: RequestHeaders): SubscriberContext {\n  const url = new URL(req.url ?? '', req.headers.origin ?? 'http://localhost');\n  const protocolVersion = checkProtocolVersion(url.pathname);\n  const params = new URLParams(url);\n\n  return {\n    protocolVersion,\n    id: params.get('id', true),\n    taskID: params.get('taskID', true),\n    mode: params.get('mode', false) === 'backup' ? 'backup' : 'serving',\n    replicaVersion: params.get('replicaVersion', true),\n    watermark: params.get('watermark', true),\n    // Absent for subscribers that predate the parameter, which is the safe\n    // default: the barrier falls back to polling rather than waiting on an\n    // ACK that would never be attributed to a writer.\n    wsBatched: params.getBoolean('wsBatched'),\n  };\n}\n\nfunction checkProtocolVersion(pathname: string): number {\n  const match = PATH_REGEX.exec(pathname);\n  if (!match) {\n    throw new Error(`invalid path: ${pathname}`);\n  }\n  const v = Number(match.groups?.version);\n  if (\n    Number.isNaN(v) ||\n    v > PROTOCOL_VERSION ||\n    v < MIN_SUPPORTED_PROTOCOL_VERSION\n  ) {\n    throw new Error(\n      `Cannot service client at protocol v${v}. ` +\n        `Supported protocols: [v${MIN_SUPPORTED_PROTOCOL_VERSION} ... v${PROTOCOL_VERSION}]`,\n    );\n  }\n  return v;\n}\n\n// Projects a client-side SubscriberContext onto the start-subscription\n// message payload, dropping the fields that are implicit on the merged\n// connection: protocolVersion (from the request path), wsBatched (always on).\nfunction toStartSubscriptionContext(ctx: SubscriberContext): SubscribeContext {\n  const {\n    taskID,\n    id,\n    mode,\n    replicaVersion,\n    watermark,\n    backfills,\n    drainable,\n    switchover,\n  } = ctx;\n  return {\n    taskID,\n    id,\n    mode,\n    replicaVersion,\n    watermark,\n    backfills,\n    ...(drainable !== undefined ? {drainable} : {}),\n    ...(switchover !== undefined ? {switchover} : {}),\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AA6CA,IAAM,iCAAiC;AAGvC,IAAM,iCAAiC;AAEvC,IAAM,wBAAwB;AAC9B,IAAM,uBAAuB;AAC7B,IAAM,yBAAyB;AAC/B,IAAM,aACJ;AAEF,IAAM,iBAAiB;AAIvB,IAAM,qBAAqB;AAU3B,IAAa,2BAAb,cAA8C,YAAY;CACxD,KAAc;CACd;CACA;CACA;CAEA,YACE,IACA,MACA,QACA,gBACA;EACA,MAAM,+BAA+B,IAAI,MAAM,OAAM,YAAW;GAC9D,MAAM,QAAQ,SAAS,SAAS;GAEhC,QAAQ,IAAI,sBAAsB,EAAC,WAAW,KAAI,GAAG,KAAKG,QAAQ;GAClE,QAAQ,IACN,uBACA,EAAC,WAAW,KAAI,GAChB,KAAKC,gBACP;GACA,QAAQ,IAAI,wBAAwB,EAAC,WAAW,KAAI,GAAG,KAAKC,UAAU;GAEtE,QAAQ,IAAI,WAAW,KAAK,QAC1B,mBACE,IACA,KAAK,UAAU;IACb,eAAe,KAAA;IACf,kBAAkB,KAAA;GACpB,GACA,KACA,KACA,KAAK,kBACL,KAAA,GACA,iBACF,CACF;GAEA,yBACE,IACA,QAAQ,iBACR,KAAKC,mBACL,MACF;EACF,CAAC;EAED,KAAKN,MAAM;EACX,KAAKC,QAAQ;EACb,KAAKC,kBAAkB;CACzB;CAGA,qBACE,IACA,QACA,QACG;EACH,QAAQ,QAAR;GACE,KAAK,YACH,OAAO,KAAKE,iBAAiB,IAAI,GAAG;GACtC,KAAK,WACH,OAAO,KAAKD,SAAS,IAAI,GAAG;GAC9B,KAAK,aACH,OAAO,KAAKE,WAAW,IAAI,GAAG;GAChC;IACE,eACE,KAAK,KACL,IACA,mBAAmB,OAAO,sBAC5B;IACA;EACJ;CACF;CAEA,mBAA4B,OAAO,IAAe,QAAwB;EACxE,KAAKE,6BAA6B,+BAA+B;EACjE,IAAI;GACF,MAAM,MAAM,IAAI,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,kBACxB;GACA,qBAAqB,IAAI,QAAQ;GACjC,MAAM,SAAS,IAAI,aAAa,IAAI,QAAQ;GAC5C,IAAI,CAAC,QACH,MAAM,IAAI,MAAM,oCAAoC;GAEtD,MAAM,aACJ,MAAM,KAAKL,gBAAgB,yBAAyB,MAAM;GAG5D,eAAoB,KAAK,KAAK,IAAI,KAAA,GAAW,UAAU;EACzD,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;CAEA,WAAoB,OAAO,IAAe,QAAwB;EAChE,IAAI;GACF,MAAM,MAAM,qBAAqB,GAAG;GACpC,IAAI,IAAI,SAAS,WACf,KAAKK,6BAA6B,uBAAuB;GAG3D,MAAM,aAAa,MAAM,KAAKL,gBAAgB,UAAU,GAAG;GAG3D,0BAA+B,KAAK,KAAK,IAAI,KAAA,GAAW,YAAY,EAClE,SAAS,IAAI,UACf,CAAC;EACH,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;;;;;;;;CASA,aAAsB,OAAO,IAAe,QAAwB;EAClE,IAAI;EACJ,MAAM,WAAW,aAAa,OAAoC;EAElE,IAAI;GAKF,MAAM,kBAAkB,qBAAqB,IAJ7B,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,kBAEqB,EAAI,QAAQ;GACzD,IAAI,kBAAkB,gCACpB,MAAM,IAAI,MACR,8CAA8C,+BAA+B,kBACzD,gBAAgB,EACtC;GAGF,MAAM,WAAW,MAAM,0BACrB,KAAK,KACL,IACA,yBACA,UACA,EAAC,SAAS,KAAI,CAChB;GAKA,WAAW,MAAM,OAAO,UACtB,IAAI,IAAI,OAAO,oBAAoB;IACjC,MAAM,EAAC,QAAQ,iBAAgB,IAAI;IACnC,KAAKK,6BAA6B,+BAA+B;IACjE,cAAc,MAAM,KAAKL,gBAAgB,yBACvC,QACA,EAAC,aAAY,CACf;IACA,KAAUM,oBAAoB,aAAa,QAAQ;GACrD,OAAO,IAAI,IAAI,OAAO,sBAAsB;IAC1C,MAAM,MAAyB;KAC7B,GAAG,IAAI;KACP;KACA,WAAW;IACb;IACA,IAAI,IAAI,SAAS,WACf,KAAKD,6BAA6B,uBAAuB;IAI3D,cAAc,KAAA;IACd,MAAM,KAAKL,gBAAgB,UAAU,KAAK,QAAQ;GACpD;EAEJ,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD,UAAU;GAGR,aAAa,OAAO;EACtB;CACF;;;;;;;;CASA,MAAMM,oBACJ,aACA,UACA;EACA,IAAI;GACF,WAAW,MAAM,GAAG,WAAW,aAG7B,SAAS,KACP,WAAW,UAAU,CACnB,YACA;IAAC,GAAG;IAAQ,KAAK;GAAU,CAC7B,CAA2B,CAC7B;EAEJ,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,yCAAyC,CAAC;GAC1D,SAAS,KAAK,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,CAAC,CAAC,CAAC;EAC7D;CACF;CAEA,yBAAyB;CAEzB,6BAA6B,QAAiB;EAC5C,IAAI,CAAC,KAAKC,0BAA0B,KAAK,OAAO,UAAU,GAAG;GAC3D,KAAKT,IAAI,OACP,kCAAkC,SAAS,IAAI,OAAO,KAAK,IAC7D;GACA,KAAUE,gBACP,IAAI,EACJ,OAAM,MACL,KAAKF,IAAI,OAAO,0CAA0C,CAAC,CAC7D,EACC,cAAc,KAAK,KAAK,CAAC;GAE5B,KAAKS,yBAAyB;EAChC;CACF;CAEA,WAAoC;EAClC,MAAM,EAAC,gBAAgB,gBAAgB,gBAAe,KAAKR;EAC3D,IAAI,mBAAmB,GAGrB,KAAKM,6BAA6B;OASlC,cAAmB,WAAW;GAC5B,KAAKP,IAAI,OACP,WAAW,eAAe,qCAC5B;GACA,KAAK,OAAO,iBAER,KAAKO,6BACH,0BAA0B,eAAe,KAC3C,GACF,cACF;EACF,CAAC;CAEL;;;;;;CAOA,MAAM,QAAuB;EAC3B,IAAI,KAAKE,0BAA0B,KAAKP,gBAAgB,OACtD,MAAM,KAAKA,gBAAgB,MAAM;CAErC;CAEA,MAAyB,UAAyB;EAChD,IAAI,KAAKO,wBACP,MAAM,KAAKP,gBAAgB,KAAK;CAEpC;AACF;AAkCA,IAAa,2BAAb,MAEA;CACE;CACA;CACA;CACA;CAEA,YACE,IACA,SACA,UACA,mBACA;EACA,KAAKF,MAAM;EACX,KAAKU,WAAW;EAGhB,KAAKC,YAAY,SAAS,IAAI,UAAU,6BAA6B;GACnE,KAAK;IACJ,iBAAiB;EACpB,CAAC;EACD,KAAKC,qBAAqB;CAC5B;CAEA,MAAMC,uBAAuB,MAAc;EACzC,IAAI,UAAU,KAAKD;EACnB,IAAI,CAAC,SAAS;GACZ,MAAM,UAAU,MAAM,8BACpB,KAAKF,UACL,KAAKC,SACP;GACA,IAAI,CAAC,SACH,MAAM,IAAI,MAAM,+BAA+B;GAEjD,UAAU,QAAQ,SAAS,KAAK,IAAI,GAAG,QAAQ,KAAK,QAAQ,QAAQ;EACtE;EACA,MAAM,MAAM,IAAI,IAAI,MAAM,OAAO;EACjC,KAAKX,IAAI,OAAO,iCAAiC,KAAK;EACtD,OAAO;CACT;CAEA,MAAM,gBACJ,QACA,cAIC;EACD,MAAM,EAAC,UAAU,aAAY,MAAM,KAAK,QAAQ,MAAM;EACtD,SAAS,KAAK,CACZ,oBACA;GAAC;GAAQ,GAAI,eAAe,EAAC,aAAY,IAAI,CAAC;EAAE,CAClD,CAAC;EAED,MAAM,EAAC,MAAM,SAAQ,SAAS,QAAQ;EACtC,MAAM,OAAO,MAAM,KAAK,KAAK;EAC7B,IAAI,CAAC,KAAK,OACR,MAAM,IAAI,MAAM,4CAA4C;EAE9D,MAAM,EAAC,MAAM,QAAO,KAAK;EACzB,IAAI,IAAI,OAAO,YACb,MAAM,IAAI,MAAM,uCAAuC,IAAI,IAAI;EAYjE,OAAO;GAAC,UAVyB,IAAI;GAUnB,UAAA;IARhB,YAAW,QAAO;KAChB,SAAS,KAAK,CAAC,sBAAsB,2BAA2B,GAAG,CAAC,CAAC;KACrE,OAAO,QAAQ,QAAQ,IAAI;IAC7B;IACA,QAAQ,SAAS;IACjB,SAAQ,WAAU,SAAS,OAAO,MAAM;GAGxB;EAAQ;CAC5B;CAEA,MAAM,UACJ,KAC6C;EAC7C,MAAM,EAAC,UAAU,aAAY,MAAM,KAAK,QAAQ,IAAI,MAAM;EAC1D,SAAS,KAAK,CAAC,sBAAsB,2BAA2B,GAAG,CAAC,CAAC;EACrE,OAAO;CACT;;;;;;;;;CAWA,MAAM,QAAQ,QAGX;EAWD,MAAM,KAAK,IAAI,YAAU,MAVP,KAAKa,uBAAuB,cAAc,IAU7B,IAAI,IANhB,gBAAgB,EAAC,OAAM,CAMP,EAAO,SAAS,KAAK,EACtD,kBAAkB,mBACpB,CAAC;EAED,MAAM,WAAW,aAAa,OAA0B;EAOxD,OAAO;GAAC,UAAA,MANe,uBACrB,KAAKb,KACL,IACA,2BACA,QACF;GACkB;EAAQ;CAC5B;AACF;AAIA,SAAgB,qBAAqB,KAAwC;CAC3E,MAAM,MAAM,IAAI,IAAI,IAAI,OAAO,IAAI,IAAI,QAAQ,UAAU,kBAAkB;CAC3E,MAAM,kBAAkB,qBAAqB,IAAI,QAAQ;CACzD,MAAM,SAAS,IAAI,UAAU,GAAG;CAEhC,OAAO;EACL;EACA,IAAI,OAAO,IAAI,MAAM,IAAI;EACzB,QAAQ,OAAO,IAAI,UAAU,IAAI;EACjC,MAAM,OAAO,IAAI,QAAQ,KAAK,MAAM,WAAW,WAAW;EAC1D,gBAAgB,OAAO,IAAI,kBAAkB,IAAI;EACjD,WAAW,OAAO,IAAI,aAAa,IAAI;EAIvC,WAAW,OAAO,WAAW,WAAW;CAC1C;AACF;AAEA,SAAS,qBAAqB,UAA0B;CACtD,MAAM,QAAQ,WAAW,KAAK,QAAQ;CACtC,IAAI,CAAC,OACH,MAAM,IAAI,MAAM,iBAAiB,UAAU;CAE7C,MAAM,IAAI,OAAO,MAAM,QAAQ,OAAO;CACtC,IACE,OAAO,MAAM,CAAC,KACd,IAAA,KACA,IAAI,gCAEJ,MAAM,IAAI,MACR,sCAAsC,EAAE,2BACZ,+BAA+B,SAC7D;CAEF,OAAO;AACT;AAKA,SAAS,2BAA2B,KAA0C;CAC5E,MAAM,EACJ,QACA,IACA,MACA,gBACA,WACA,WACA,WACA,eACE;CACJ,OAAO;EACL;EACA;EACA;EACA;EACA;EACA;EACA,GAAI,cAAc,KAAA,IAAY,EAAC,UAAS,IAAI,CAAC;EAC7C,GAAI,eAAe,KAAA,IAAY,EAAC,WAAU,IAAI,CAAC;CACjD;AACF"}

package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts

@@ -90,6 +90,14 @@ export type TuningOptions = StorerOptions & {
     pgChangeLogBehindSlot: boolean;
     flowControlConsensusTimeoutProportion: number;
     flowControlSlowSubscriberGracePeriodMs?: number | undefined;
+    /**
+     * The target number of subscribers that may be replacing their replica at
+     * the same time, which limits when subscribers switching over from an
+     * incompatible replication-manager are granted permission to restart (see
+     * {@link RestoreAdmission}). Defaults to 1. Only applicable when backups
+     * are configured.
+     */
+    maxConcurrentSwitchoverRestores?: number | undefined;
     sqliteCatchup?: SQLiteCatchupOptions | undefined;
     /**
      * Supplied when `sqliteChangeLogMode != off`, i.e. this is the gate on the

package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-service.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAcjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAClD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAGnD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,mCAAmC,CAAC;AAM3C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAGL,KAAK,0BAA0B,EAChC,MAAM,qCAAqC,CAAC;AAC7C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,2CAA2C,CAAC;AAajF,OAAO,EACL,KAAK,qBAAqB,EAC1B,KAAK,iBAAiB,EAEvB,MAAM,sBAAsB,CAAC;AAU9B,OAAO,EAEL,KAAK,2BAA2B,EACjC,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAEL,KAAK,6BAA6B,EACnC,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAGL,KAAK,oCAAoC,EAC1C,MAAM,wCAAwC,CAAC;AAChD,OAAO,EAIL,KAAK,uBAAuB,EAC7B,MAAM,oCAAoC,CAAC;AAE5C,OAAO,EAEL,KAAK,4BAA4B,EAClC,MAAM,+BAA+B,CAAC;AAEvC,OAAO,EAEL,KAAK,SAAS,EACd,KAAK,aAAa,IAAI,aAAa,EACpC,MAAM,aAAa,CAAC;AAIrB,MAAM,MAAM,YAAY,GAAG;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,iBAAiB,CAAC;IACrC;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAClC,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,sEAAsE;IACtE,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3C;;;;OAIG;IACH,SAAS,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,iBAAiB,KAAK,OAAO,CAAC,GAAG,SAAS,CAAC;IAC9D;;;;OAIG;IACH,YAAY,CAAC,EAAE,2BAA2B,GAAG,SAAS,CAAC;IACvD;;;;OAIG;IACH,uBAAuB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC9C,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG;IACxC,oEAAoE;IACpE,WAAW,EAAE,MAAM,CAAC;IACpB;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,sDAAsD;IACtD,WAAW,EAAE,MAAM,CAAC;IACpB,kDAAkD;IAClD,iBAAiB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,8DAA8D;IAC9D,GAAG,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,GAAG,SAAS,CAAC;IACjC,+DAA+D;IAC/D,OAAO,CAAC,EAAE,CAAC,MAAM,uBAAuB,GAAG,SAAS,CAAC,GAAG,SAAS,CAAC;CACnE,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG,aAAa,GAAG;IAC1C;;;;OAIG;IACH,kBAAkB,EAAE,OAAO,CAAC;IAC5B;;;;;;;;;OASG;IACH,qBAAqB,EAAE,OAAO,CAAC;IAC/B,qCAAqC,EAAE,MAAM,CAAC;IAC9C,sCAAsC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5D,aAAa,CAAC,EAAE,oBAAoB,GAAG,SAAS,CAAC;IACjD;;;OAGG;IACH,qBAAqB,CAAC,EAClB,IAAI,CACF,4BAA4B,EAC5B,UAAU,GAAG,YAAY,GAAG,WAAW,CACxC,GACD,SAAS,CAAC;IACd;;;;;;;OAOG;IACH,oBAAoB,CAAC,EAAE,oCAAoC,GAAG,SAAS,CAAC;IACxE;;;;;;OAMG;IACH,sBAAsB,CAAC,EACnB,CAAC,6BAA6B,GAAG;QAAC,WAAW,EAAE,MAAM,CAAA;KAAC,CAAC,GACvD,SAAS,CAAC;IACd,+EAA+E;IAC/E,oBAAoB,CAAC,EAAE,2BAA2B,GAAG,SAAS,CAAC;CAChE,CAAC;AAEF;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,QAAQ,EAAE,UAAU,EACpB,YAAY,EAAE,YAAY,EAC1B,0BAA0B,EAAE,0BAA0B,EACtD,iBAAiB,EAAE,iBAAiB,EACpC,YAAY,EAAE,YAAY,GAAG,IAAI,EACjC,SAAS,EAAE,SAAS,GAAG,IAAI,EAC3B,SAAS,EAAE,OAAO,EAClB,IAAI,EAAE,aAAa,EACnB,YAAY,oBAAa,GACxB,OAAO,CAAC,qBAAqB,CAAC,CAiDhC"}
+{"version":3,"file":"change-streamer-service.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAcjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAClD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAGnD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,mCAAmC,CAAC;AAM3C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAGL,KAAK,0BAA0B,EAChC,MAAM,qCAAqC,CAAC;AAC7C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,2CAA2C,CAAC;AAajF,OAAO,EACL,KAAK,qBAAqB,EAC1B,KAAK,iBAAiB,EAEvB,MAAM,sBAAsB,CAAC;AAY9B,OAAO,EAEL,KAAK,2BAA2B,EACjC,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAEL,KAAK,6BAA6B,EACnC,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAGL,KAAK,oCAAoC,EAC1C,MAAM,wCAAwC,CAAC;AAChD,OAAO,EAIL,KAAK,uBAAuB,EAC7B,MAAM,oCAAoC,CAAC;AAE5C,OAAO,EAEL,KAAK,4BAA4B,EAClC,MAAM,+BAA+B,CAAC;AAEvC,OAAO,EAEL,KAAK,SAAS,EACd,KAAK,aAAa,IAAI,aAAa,EACpC,MAAM,aAAa,CAAC;AAKrB,MAAM,MAAM,YAAY,GAAG;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,iBAAiB,CAAC;IACrC;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAClC,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,sEAAsE;IACtE,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3C;;;;OAIG;IACH,SAAS,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,iBAAiB,KAAK,OAAO,CAAC,GAAG,SAAS,CAAC;IAC9D;;;;OAIG;IACH,YAAY,CAAC,EAAE,2BAA2B,GAAG,SAAS,CAAC;IACvD;;;;OAIG;IACH,uBAAuB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC9C,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG;IACxC,oEAAoE;IACpE,WAAW,EAAE,MAAM,CAAC;IACpB;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,sDAAsD;IACtD,WAAW,EAAE,MAAM,CAAC;IACpB,kDAAkD;IAClD,iBAAiB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,8DAA8D;IAC9D,GAAG,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,GAAG,SAAS,CAAC;IACjC,+DAA+D;IAC/D,OAAO,CAAC,EAAE,CAAC,MAAM,uBAAuB,GAAG,SAAS,CAAC,GAAG,SAAS,CAAC;CACnE,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG,aAAa,GAAG;IAC1C;;;;OAIG;IACH,kBAAkB,EAAE,OAAO,CAAC;IAC5B;;;;;;;;;OASG;IACH,qBAAqB,EAAE,OAAO,CAAC;IAC/B,qCAAqC,EAAE,MAAM,CAAC;IAC9C,sCAAsC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5D;;;;;;OAMG;IACH,+BAA+B,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACrD,aAAa,CAAC,EAAE,oBAAoB,GAAG,SAAS,CAAC;IACjD;;;OAGG;IACH,qBAAqB,CAAC,EAClB,IAAI,CACF,4BAA4B,EAC5B,UAAU,GAAG,YAAY,GAAG,WAAW,CACxC,GACD,SAAS,CAAC;IACd;;;;;;;OAOG;IACH,oBAAoB,CAAC,EAAE,oCAAoC,GAAG,SAAS,CAAC;IACxE;;;;;;OAMG;IACH,sBAAsB,CAAC,EACnB,CAAC,6BAA6B,GAAG;QAAC,WAAW,EAAE,MAAM,CAAA;KAAC,CAAC,GACvD,SAAS,CAAC;IACd,+EAA+E;IAC/E,oBAAoB,CAAC,EAAE,2BAA2B,GAAG,SAAS,CAAC;CAChE,CAAC;AAEF;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,QAAQ,EAAE,UAAU,EACpB,YAAY,EAAE,YAAY,EAC1B,0BAA0B,EAAE,0BAA0B,EACtD,iBAAiB,EAAE,iBAAiB,EACpC,YAAY,EAAE,YAAY,GAAG,IAAI,EACjC,SAAS,EAAE,SAAS,GAAG,IAAI,EAC3B,SAAS,EAAE,OAAO,EAClB,IAAI,EAAE,aAAa,EACnB,YAAY,oBAAa,GACxB,OAAO,CAAC,qBAAqB,CAAC,CAiDhC"}

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js

@@ -15,7 +15,9 @@ import { BackfillStateTracker } from "./backfill-state-tracker.js";
 import { serializeChangeStreamDataWithChange } from "./change-log-codec.js";
 import { SQLiteChangeLogReader } from "./sqlite-change-log-reader.js";
 import { ChangeLogInitializer, replicaInitializationSource } from "./change-log-initializer.js";
+import { DrainTracker } from "./drain-tracker.js";
 import { Forwarder } from "./forwarder.js";
+import { RestoreAdmission } from "./restore-admission.js";
 import { SnapshotReservations } from "./snapshot-reservations.js";
 import { SQLiteChangeLogCatchup } from "./sqlite-change-log-catchup.js";
 import { SQLiteChangeLogComparator } from "./sqlite-change-log-comparator.js";
@@ -193,6 +195,8 @@ var ChangeStreamerImpl = class {
 	#pgChangeLogEnabled;
 	#forwarder;
 	#reservations;
+	#restoreAdmission;
+	#drainTracker;
 	#replicationStatusPublisher;
 	#sqliteCatchupOptions;
 	#changeLogWriter;
@@ -266,10 +270,18 @@ var ChangeStreamerImpl = class {
 			now: serveOptions.now,
 			inspect: serveOptions.inspect ?? (() => inspectSQLiteChangeLog(lc, catchupOptions.changeLogFile, writerOptions.identity))
 		}) : void 0;
-		this.#reservations = backupConfig ? new SnapshotReservations(lc, backupConfig, (taskID) => {
-			this.#readRouter?.release(taskID);
-			this.#purgeScheduler?.resume(taskID);
+		this.#reservations = backupConfig ? new SnapshotReservations(lc, backupConfig, {
+			onClose: (taskID) => {
+				this.#readRouter?.release(taskID);
+				this.#purgeScheduler?.resume(taskID);
+			},
+			onEnd: (taskID) => this.#restoreAdmission?.reservationClosed(taskID)
+		}) : void 0;
+		this.#restoreAdmission = backupConfig ? new RestoreAdmission(lc, {
+			replicaVersion,
+			maxConcurrentRestores: opts.maxConcurrentSwitchoverRestores ?? 1
 		}) : void 0;
+		this.#drainTracker = new DrainTracker(lc);
 		this.#replicationStatusPublisher = replicationStatusPublisher;
 		this.#changeLogWriter = opts.sqliteChangeLogWriter ? new SQLiteChangeLogWriter(lc, {
 			...opts.sqliteChangeLogWriter,
@@ -453,77 +465,128 @@ var ChangeStreamerImpl = class {
 		const subscriber = new Subscriber(protocolVersion, id, mode, watermark, downstream, {
 			wsBatched,
 			lagReportingDisabled: this.#lagReportingDisabled.promise,
+			drainable: mode === "serving" && ctx.drainable,
 			...this.#backfills.subscriberOptions(lc, ctx)
 		});
 		this.#backfills.register(subscriber, downstream);
+		try {
+			if (ctx.switchover && this.#drainTracker.draining) {
+				lc.warn?.(`rejecting switchover subscriber ${id}: this replication-manager is draining`);
+				subscriber.close();
+			} else if (replicaVersion !== this.#replicaVersion) if (ctx.switchover && mode === "serving" && this.#restoreAdmission && !this.#drainTracker.draining && replicaVersion < this.#replicaVersion) {
+				lc.info?.(`holding switchover subscriber ${subscriber.id} at replica version ${replicaVersion} for permission to restore ${this.#replicaVersion}`);
+				const ticket = this.#restoreAdmission.request({
+					onQueued: (position) => subscriber.sendControlStatus({
+						tag: "status",
+						restore: {
+							state: "queued",
+							position
+						}
+					}),
+					onGranted: () => subscriber.sendControlStatus({
+						tag: "status",
+						restore: { state: "granted" }
+					})
+				});
+				cleanupSubscriber = () => ticket.release();
+			} else {
+				lc.warn?.(`rejecting subscriber at replica version ${replicaVersion}`);
+				subscriber.close(1, `current replica version is ${this.#replicaVersion} (requested ${replicaVersion})`, { replicaVersion: this.#replicaVersion });
+			}
+			else {
+				lc.info?.(`adding subscriber ${subscriber.id}`);
+				this.#drainTracker.register(subscriber, downstream);
+				const release = this.#restoreAdmission?.reservationSubscribed(ctx.taskID);
+				if (release) {
+					subscriber.caughtUp.then(release);
+					downstream.addCloseHandler(release);
+				}
+				cleanupSubscriber = await this.#startCatchup(lc, ctx, subscriber);
+			}
+		} catch (e) {
+			downstream.fail(e instanceof Error ? e : new Error(String(e)));
+			throw e;
+		} finally {
+			downstream.addCloseHandler(() => cleanupSubscriber());
+			this.#reservations?.close(ctx.taskID);
+		}
+		return downstream;
+	}
+	/**
+	* Routes the catchup of an accepted `subscriber` to the SQLite or PG change
+	* log (or rejects it), and returns the function that removes the subscriber
+	* from wherever it was registered. If routing fails partway, any partial
+	* registration is removed before the error is rethrown.
+	*/
+	async #startCatchup(lc, ctx, subscriber) {
 		const removeFromForwarder = () => {
 			lc.info?.(`removing subscriber ${subscriber.id}`);
 			this.#forwarder.remove(subscriber);
 		};
-		cleanupSubscriber = removeFromForwarder;
-		if (replicaVersion !== this.#replicaVersion) {
-			lc.warn?.(`rejecting subscriber at replica version ${replicaVersion}`);
-			subscriber.close(1, `current replica version is ${this.#replicaVersion} (requested ${replicaVersion})`);
-		} else {
-			lc.info?.(`adding subscriber ${subscriber.id}`);
-			const catchupFromPG = () => {
-				assert(this.#pgChangeLogEnabled, "cannot catch up from a disabled PG change log");
-				cleanupSubscriber = removeFromForwarder;
+		const catchupFromPG = () => {
+			assert(this.#pgChangeLogEnabled, "cannot catch up from a disabled PG change log");
+			try {
 				this.#forwarder.add(subscriber);
-				this.#storer.catchup(subscriber, mode);
-			};
-			const sqliteDecision = this.#selectSQLiteCatchup(lc, ctx);
-			if (!sqliteDecision) catchupFromPG();
-			else if (sqliteDecision.kind === "rejected") {
-				lc.warn?.(`${sqliteDecision.terminal ? "rejecting" : "ending"} subscription for ${ctx.id}: ${sqliteDecision.message}`);
-				if (sqliteDecision.terminal) subscriber.close(2, sqliteDecision.message);
-				else subscriber.close();
+				this.#storer.catchup(subscriber, ctx.mode);
+			} catch (e) {
+				removeFromForwarder();
+				throw e;
+			}
+			return removeFromForwarder;
+		};
+		const sqliteDecision = this.#selectSQLiteCatchup(lc, ctx);
+		if (!sqliteDecision) return catchupFromPG();
+		if (sqliteDecision.kind === "rejected") {
+			lc.warn?.(`${sqliteDecision.terminal ? "rejecting" : "ending"} subscription for ${ctx.id}: ${sqliteDecision.message}`);
+			if (sqliteDecision.terminal) subscriber.close(2, sqliteDecision.message);
+			else subscriber.close();
+			return () => {};
+		}
+		const { catchup, reason, coverage, logWarm } = sqliteDecision;
+		const removeFromSQLite = () => catchup.remove(subscriber);
+		let registration;
+		try {
+			registration = await catchup.catchup(subscriber, () => this.#captureRequiredHead(), { logWarm });
+		} catch (e) {
+			removeFromSQLite();
+			throw e;
+		}
+		switch (registration.kind) {
+			case "registered":
+				lc.debug?.(`serving ${ctx.id} from SQLite catchup`, ...coverage ? [{ sqliteChangeLogCoverage: coverage }] : []);
+				this.#recordCatchupRoute("sqlite", reason);
+				return removeFromSQLite;
+			case "uncovered": if (this.#pgChangeLogEnabled) {
+				lc.info?.(`serving ${ctx.id} from PG catchup: subscriber watermark ${ctx.watermark} is below the SQLite change-log minimum ` + registration.minWatermark);
+				this.#recordCatchupRoute("pg", "watermark-uncovered");
+				return catchupFromPG();
 			} else {
-				const { catchup, reason, coverage, logWarm } = sqliteDecision;
-				cleanupSubscriber = () => catchup.remove(subscriber);
-				const registration = await catchup.catchup(subscriber, () => this.#captureRequiredHead(), { logWarm });
-				switch (registration.kind) {
-					case "registered":
-						lc.debug?.(`serving ${ctx.id} from SQLite catchup`, ...coverage ? [{ sqliteChangeLogCoverage: coverage }] : []);
-						this.#recordCatchupRoute("sqlite", reason);
-						break;
-					case "uncovered":
-						if (this.#pgChangeLogEnabled) {
-							lc.info?.(`serving ${ctx.id} from PG catchup: subscriber watermark ${ctx.watermark} is below the SQLite change-log minimum ` + registration.minWatermark);
-							this.#recordCatchupRoute("pg", "watermark-uncovered");
-							catchupFromPG();
-						} else {
-							const message = `earliest supported watermark is ${registration.minWatermark} (requested ${ctx.watermark})`;
-							lc.warn?.(`rejecting subscriber at watermark ${ctx.watermark}: the SQLite change log starts at ${registration.minWatermark} and the PG change log is disabled`);
-							this.#recordCatchupRoute("none", "watermark-uncovered");
-							subscriber.close(2, message);
-						}
-						break;
-					case "declined":
-						if (this.#pgChangeLogEnabled) {
-							lc.error?.(`serving ${ctx.id} from PG catchup: SQLite catchup registration failed`, registration.error);
-							this.#recordCatchupRoute("pg", "registration-failed");
-							catchupFromPG();
-						} else {
-							lc.error?.(`ending subscription for ${ctx.id} to retry SQLite catchup: registration failed and the PG change log is disabled`, registration.error);
-							this.#recordCatchupRoute("none", "registration-failed");
-							subscriber.fail(registration.error);
-						}
-						break;
-					case "handled":
-						this.#recordCatchupRoute("sqlite", "registration-handled");
-						break;
-					default: unreachable(registration);
-				}
+				const message = `earliest supported watermark is ${registration.minWatermark} (requested ${ctx.watermark})`;
+				lc.warn?.(`rejecting subscriber at watermark ${ctx.watermark}: the SQLite change log starts at ${registration.minWatermark} and the PG change log is disabled`);
+				this.#recordCatchupRoute("none", "watermark-uncovered");
+				subscriber.close(2, message);
+				return removeFromSQLite;
+			}
+			case "declined": if (this.#pgChangeLogEnabled) {
+				lc.error?.(`serving ${ctx.id} from PG catchup: SQLite catchup registration failed`, registration.error);
+				this.#recordCatchupRoute("pg", "registration-failed");
+				return catchupFromPG();
+			} else {
+				lc.error?.(`ending subscription for ${ctx.id} to retry SQLite catchup: registration failed and the PG change log is disabled`, registration.error);
+				this.#recordCatchupRoute("none", "registration-failed");
+				subscriber.fail(registration.error);
+				return removeFromSQLite;
 			}
+			case "handled":
+				this.#recordCatchupRoute("sqlite", "registration-handled");
+				return removeFromSQLite;
+			default: unreachable(registration);
 		}
-		downstream.addCloseHandler(() => cleanupSubscriber());
-		this.#reservations?.close(ctx.taskID);
-		return downstream;
 	}
-	async startSnapshotReservation(taskID) {
+	async startSnapshotReservation(taskID, { localReplica } = {}) {
 		if (!this.#reservations) throw new Error("backups are not configured");
 		const downstream = this.#reservations.open(taskID);
+		this.#restoreAdmission?.reservationOpened(taskID, localReplica);
 		try {
 			await (this.#purgeScheduler?.pause(taskID) ?? promiseVoid);
 			if (!this.#reservations.isCurrent(taskID, downstream)) return downstream;
@@ -535,6 +598,9 @@ var ChangeStreamerImpl = class {
 			throw e;
 		}
 	}
+	drain() {
+		return this.#drainTracker.drain();
+	}
 	trackBackupWatermark(watermark) {
 		this.#backupWatermark = watermark;
 		this.#acker.trackBackup(watermark);
@@ -572,8 +638,10 @@ var ChangeStreamerImpl = class {
 			let minWatermark;
 			if (route?.source === "sqlite") minWatermark = must(route.coverage, "a pinned SQLite route must carry its covered range").minWatermark;
 			else minWatermark = must(pgState, "a PG reservation route requires the PG change log").minWatermark;
-			if (minWatermark <= backupWatermark) reservations.confirmFor(taskID, this.#replicaVersion, backupWatermark, source);
-			else {
+			if (minWatermark <= backupWatermark) {
+				reservations.confirmFor(taskID, this.#replicaVersion, backupWatermark, source);
+				this.#restoreAdmission?.reservationConfirmed(taskID, backupWatermark);
+			} else {
 				if (reservations.noteConfirmationDelayed(taskID)) this.#reservationConfirmDelays.add(1);
 				this.#lc.error?.(`${source} change-log minWatermark ${minWatermark} is later than backupWatermark ${backupWatermark}. Delaying confirmation of snapshot reservation until next backup.`);
 			}
@@ -674,6 +742,7 @@ var ChangeStreamerImpl = class {
 	}
 	async stop(err) {
 		this.#state.stop(this.#lc, err);
+		this.#drainTracker.stop();
 		this.#lagReportingDisabled.resolve(false);
 		this.#stream?.changes.cancel();
 		this.#purgeScheduler?.stop();