@rocicorp/zero 1.11.0-canary.32 → 1.11.0-canary.34

package/out/zero-cache/src/services/change-source/pg/change-source-init.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-source-init.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source-init.ts"],"sourcesContent":["import type {Readable} from 'node:stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../../shared/src/resolved-promises.ts';\nimport {sleep} from '../../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport {connectPgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {type ShardConfig, type ShardID} from '../../../types/shards.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport type {ReplicaConstraints} from '../../litestream/commands.ts';\nimport {\n  getSubscriptionStateAndContext,\n  type SubscriptionStateAndContext,\n} from '../../replicator/schema/replication-state.ts';\nimport {\n  restoreReplica,\n  type ConstrainingPurgeLock,\n  type InitializeResult,\n  type RestoreOptions,\n  type ResumableBackup,\n} from '../common/replica-restore.ts';\nimport {initReplica} from '../common/replica-schema.ts';\nimport {PostgresChangeSource} from './change-source.ts';\nimport {\n  initialSync,\n  type InitialSyncOptions,\n  type ReplicaOptions,\n  type ServerContext,\n} from './initial-sync.ts';\nimport {endSession} from './logical-replication/stream.ts';\nimport {toBigInt, toStateVersionString, type LSN} from './lsn.ts';\nimport {\n  claimSlotForResumption,\n  createReplicaAndSlot,\n  type ReplicationSlotResult,\n} from './replication-slots.ts';\nimport {ensureShardSchema} from './schema/init.ts';\nimport * as ReplicaStage from './schema/replica-stage-enum.ts';\nimport {InitialSync, Replicate} from './schema/replica-stage-enum.ts';\nimport {\n  dropShard,\n  getActiveReplicas,\n  getReplicaAtVersion,\n  getRestoreCandidates,\n  initRestoreReplica,\n  internalPublicationPrefix,\n  type ReplicaState,\n} from './schema/shard.ts';\n\nexport type InitializeOptions = ReplicaOptions & {\n  // Create a new slot for the replica rather than taking over\n  // an existing one (i.e. high-availability mode).\n  slotPerReplica: boolean | undefined;\n\n  inactiveReplicaGracePeriodMs: number;\n\n  /**\n   * Whether this replication-manager maintains the PG change log, i.e. acks\n   * the replication slot only after transactions are stored in it. Recorded\n   * with the replica (see `pgChangeLog` in `ReplicaState`) when its slot is\n   * taken over.\n   */\n  pgChangeLog: boolean;\n};\n\n/**\n * Initializes a Postgres change source, including the initial sync of the\n * replica, before streaming changes from the corresponding logical replication\n * stream.\n */\nexport async function initializePostgresChangeSource(\n  lc: LogContext,\n  upstreamURI: string,\n  shard: ShardConfig,\n  replicaDbFile: string,\n  syncOptions: InitialSyncOptions,\n  context: ServerContext,\n  lagReportIntervalMs = 0,\n  restoreOptions: RestoreOptions = {},\n  {\n    epoch,\n    slotPerReplica,\n    backupV5,\n    inactiveReplicaGracePeriodMs,\n    pgChangeLog,\n  }: InitializeOptions = {\n    epoch: 0,\n    slotPerReplica: false,\n    backupV5: true,\n    inactiveReplicaGracePeriodMs: DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS,\n    pgChangeLog: false,\n  },\n  streamInboundTimeoutMs?: number | undefined,\n): Promise<InitializeResult> {\n  const db = await connectPgClient(lc, upstreamURI, 'change-source-init');\n  try {\n    await ensureShardSchema(\n      lc,\n      db,\n      shard,\n      syncOptions.installPartialIndexTriggers,\n    );\n\n    const {\n      replica: restoredReplica,\n      purgeLock,\n      pgChangeLogBehindSlot,\n      resumableBackup,\n      slotWatermark,\n    } = slotPerReplica\n      ? await forkOrResumeReplica(\n          lc,\n          db,\n          shard,\n          epoch,\n          syncOptions.replicationSlotFailover ?? false,\n          replicaDbFile,\n          restoreOptions,\n          inactiveReplicaGracePeriodMs,\n        )\n      : await selectAndRestoreReplica(\n          lc,\n          db,\n          shard,\n          replicaDbFile,\n          restoreOptions,\n        );\n\n    let initialSyncedReplica: ReplicaState | undefined;\n    await initReplica(\n      lc,\n      `replica-${shard.appID}-${shard.shardNum}`,\n      replicaDbFile,\n      async (log, tx) => {\n        // Falling back to initial sync abandons the restore, so release the\n        // resources acquired for it, e.g. a replication slot claimed (or\n        // created) for a resumed (or forked) replica, which would otherwise\n        // remain active indefinitely.\n        await restoreOptions.cleanup?.release();\n        // The purge lock on the change-db must be released before performing\n        // initial sync; if the change-db and upstream are the same db, a lock-holding\n        // transaction will prevent a replication slot from being created. This awkward\n        // dependency can go away with RMv2. (The purge lock is also released by\n        // the cleanup when it is registered with it; releasing it is idempotent.)\n        await purgeLock?.release();\n        initialSyncedReplica = await initialSync(\n          log,\n          shard,\n          tx,\n          upstreamURI,\n          syncOptions,\n          context,\n          {epoch, backupV5, pgChangeLog},\n          restoreOptions.cleanup,\n        );\n      },\n    );\n\n    const replica = new Database(lc, replicaDbFile);\n    const subscriptionState = getSubscriptionStateAndContext(\n      new StatementRunner(replica),\n    );\n    replica.close();\n\n    // Check that upstream is properly setup, and throw an AutoReset to re-run\n    // initial sync if not.\n    const {upstreamReplica, pgVersion} = await checkAndUpdateUpstream(\n      lc,\n      db,\n      shard,\n      subscriptionState,\n      (initialSyncedReplica ?? restoredReplica)?.id,\n    );\n\n    // With litestream v5, every replication-manager backs up to its own\n    // backup lineage.\n    const newBackupLineage = backupV5 && !resumableBackup;\n    const backupPath = initialSyncedReplica\n      ? // If initial sync was performed, use that initial backupPath.\n        initialSyncedReplica.backupPath\n      : // Otherwise, use the lineage prepared by the restore, or a new, unique\n        // path when backing up with litestream v5. This will be recorded in the\n        // replicas table by the PostgresChangeSource.\n        backupV5\n        ? (resumableBackup?.backupPath ?? String(Date.now()))\n        : (restoredReplica?.backupPath ?? null);\n\n    const changeSource = new PostgresChangeSource(\n      lc,\n      upstreamURI,\n      shard,\n      upstreamReplica,\n      pgVersion,\n      {backupPath, backupV5, pgChangeLog},\n      context,\n      lagReportIntervalMs,\n      syncOptions.textCopy,\n      streamInboundTimeoutMs,\n      minStartWatermark(\n        subscriptionState.watermark,\n        initialSyncedReplica !== undefined,\n        slotWatermark,\n        purgeLock,\n      ),\n    );\n\n    const destinationBackupURL =\n      backupPath && restoreOptions.litestream?.backupURL\n        ? new URL(backupPath, restoreOptions.litestream.backupURL).toString()\n        : // For legacy RMv1 replicas (on litestream-v3), backup to the same location\n          restoreOptions.litestream?.backupURL;\n\n    return {\n      subscriptionState,\n      changeSource,\n      destinationBackupURL,\n      // The replica this change stream belongs to. It is part of the identity\n      // the SQLite change log records, because a generation (i.e.\n      // `replicaVersion`) is shared by every sibling of a forked replica and so\n      // cannot distinguish two siblings' logs.\n      replicaID: upstreamReplica.id,\n      waitForBackupBeforeServing:\n        // Wait for the first backup if there was an initial sync,\n        initialSyncedReplica !== undefined ||\n        // or if the destination differs from where it was restored\n        // (i.e. backupV5).\n        backupPath !== (restoredReplica?.backupPath ?? null),\n      pgChangeLogBehindSlot,\n      newBackupLineage,\n      walKeeper: resumableBackup?.walKeeper,\n    };\n  } finally {\n    await db.end();\n  }\n}\n\ntype RestoredReplica = {\n  replica: ReplicaState | undefined;\n  purgeLock: ConstrainingPurgeLock | null;\n  pgChangeLogBehindSlot?: boolean | undefined;\n  resumableBackup?: ResumableBackup | undefined;\n  /**\n   * The position (as a watermark) of the replication slot from which\n   * replication resumes, when it was created (for a fork) or claimed (for a\n   * resumption), i.e. while it is held by this process. Absent for a shared\n   * slot (RMv1), whose position is advanced by its current owner until it is\n   * taken over, and for a resumed slot that was acked only once transactions\n   * were stored in the PG change-log (RMv1.5), whose position may be past the\n   * change-log's head for acked keepalives.\n   */\n  slotWatermark?: string | undefined;\n};\n\n// RMv1: Selects a replica to restore from and returns it, with the\n//       intention of taking over the slot (in the ChangeSource).\nasync function selectAndRestoreReplica(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  replicaFile: string,\n  {litestream, acquirePurgeLock, cleanup}: RestoreOptions,\n): Promise<RestoredReplica> {\n  // The purge lock constrains the generation of the replica to restore. With\n  // a shared replication slot, the change-log is resumed from its head\n  // without being checked against the slot's position: the slot is still\n  // being acked by the previous replication-manager, on an idle upstream past\n  // the change-log's head (see PgChangeLogPurgeLocker).\n  const lock = await acquirePurgeLock?.();\n  assert(lock !== 'behind-slot', 'unexpected purge-lock result');\n  const purgeLock = lock ?? null;\n  const replicas = (await getActiveReplicas(lc, sql, shard)).filter(\n    // filter to the generation specified by the purge lock, if present\n    ({generation}) => generation === (purgeLock?.replicaVersion ?? generation),\n  );\n  if (replicas.length === 0) {\n    lc.info?.(`no suitable replicas to restore from`, {replicas});\n    return {replica: undefined, purgeLock};\n  }\n  const [replica] = replicas;\n\n  if (litestream?.backupURL) {\n    const constraints = restoreConstraints(purgeLock, replica);\n    const {backupURL: backupBaseURL} = litestream;\n    const {slot, backupPath, confirmedFlushLsn} = replica;\n    const backupURL = new URL(backupPath ?? '', backupBaseURL).toString();\n    lc.info?.(\n      `restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`,\n      {replicas},\n    );\n    const resumableBackup = await restoreReplica(\n      lc,\n      {...litestream, backupURL}, // includes the replica's backup sub-path\n      replicaFile,\n      constraints,\n      litestream.forkBackup\n        ? {type: 'fork', baseURL: backupBaseURL}\n        : undefined,\n      cleanup,\n    );\n    return {replica, purgeLock, resumableBackup};\n  }\n  return {replica, purgeLock};\n}\n\n// RMv2: Restores from an active replica and creates a new replica / slot\n// to continue replication (i.e. \"fork\"). If only orphaned replicas remain,\n// claims a slot, restores the backup, and \"resumes\" replication for that\n// slot.\nasync function forkOrResumeReplica(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  epoch: number,\n  slotFailover: boolean,\n  replicaFile: string,\n  {litestream, acquirePurgeLock, cleanup}: RestoreOptions,\n  gracePeriodMs: number,\n): Promise<RestoredReplica> {\n  const result = await getSourceAndDestinationReplicas(\n    lc,\n    sql,\n    shard,\n    epoch,\n    slotFailover,\n    gracePeriodMs,\n  );\n  if (!result) {\n    // can't restore, must initial-sync\n    return {replica: undefined, purgeLock: null};\n  }\n  const {restoreFrom, replicateTo, slotLSN, slotSession} = result;\n  // A resumed slot (i.e. of the same replica) that was acked only once\n  // transactions were stored in the PG change-log (RMv1.5) cannot have been\n  // acked past a transaction missing from the change-log. Its position past\n  // the change-log's head only reflects acked keepalives, so it is not\n  // compared with the head (see below).\n  const resumedAfterPgChangeLog =\n    restoreFrom.id === replicateTo.id && restoreFrom.pgChangeLog;\n  const slotWatermark = resumedAfterPgChangeLog\n    ? undefined\n    : toStateVersionString(slotLSN);\n  // The slot is held active by a session in this process until it is taken\n  // over by the PostgresChangeSource. If initialization fails before then,\n  // the session must be released, or the slot remains active indefinitely,\n  // preventing the replica from being resumed (or forked) by a retry.\n  cleanup?.onFailure(`replication slot ${replicateTo.slot}`, () =>\n    slotSession.destroy(),\n  );\n\n  // Acquired only after the replication slot has been created (or claimed),\n  // and only if the change-log can be resumed from the slot's position.\n  //\n  // For a fork, this is the case whenever the change-log is maintained (i.e.\n  // by a replication-manager that acks the slot only once a transaction has\n  // been stored in the change-log): the fork waits for the source replica's\n  // slot to pass the position after the creation of the new replica, which is\n  // a transaction (in the publication) after the new slot's position.\n  //\n  // For a resumed slot, the slot can also have been acked past the head of a\n  // maintained change-log, for keepalives (e.g. for changes outside of the\n  // publication, or on an idle upstream). If the slot was acked by a\n  // replication-manager that maintains the change-log, it is therefore not\n  // checked against the slot's position, which would otherwise re-initialize\n  // the change-log unnecessarily, resetting subscribers behind the replica.\n  // Otherwise (e.g. RMv2, or unknown), it is: the change-log is then\n  // re-initialized from the replica if it does not contain the replica's\n  // changes.\n  const lock = acquirePurgeLock ? await acquirePurgeLock(slotWatermark) : null;\n  const pgChangeLogBehindSlot = lock === 'behind-slot';\n  const purgeLock = lock === 'behind-slot' ? null : lock;\n  // Absent a purge lock, still constrain the restore to the source's\n  // generation, so that a local replica left behind by a different\n  // generation (e.g. a failed initial sync) is not mistaken for it.\n  const constraints = restoreConstraints(purgeLock, restoreFrom);\n\n  if (litestream?.backupURL) {\n    const {backupURL: backupBaseURL} = litestream;\n    const {slot, backupPath, confirmedFlushLsn} = restoreFrom;\n    const backupURL = new URL(backupPath ?? '', backupBaseURL).toString();\n    lc.info?.(\n      `restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`,\n      {restoreFrom, replicateTo},\n    );\n    const resumableBackup = await restoreReplica(\n      lc,\n      {...litestream, backupURL}, // includes the replica's backup sub-path\n      replicaFile,\n      constraints,\n      litestream.forkBackup\n        ? {type: 'fork', baseURL: backupBaseURL}\n        : undefined,\n      cleanup,\n    );\n    return {\n      replica: replicateTo,\n      purgeLock,\n      pgChangeLogBehindSlot,\n      resumableBackup,\n      slotWatermark,\n    };\n  }\n  return {\n    replica: replicateTo,\n    purgeLock,\n    pgChangeLogBehindSlot,\n    slotWatermark,\n  };\n}\n\n/**\n * The watermark below which the replication stream cannot (correctly) be\n * started, i.e. from which it would skip transactions that the change log\n * does not contain. It reflects the order in which each mode stores a\n * transaction and then acks it to the slot, such that a correctly initialized\n * stream always starts at or past it:\n *\n * * After an initial sync, the replica's watermark, which is the position\n *   at which the new slot was created, and the change log is (re)initialized\n *   at it.\n * * With a slot per replica (RMv1.5 and RMv2), the minimum of the restored\n *   replica's watermark and the slot's position. Transactions are only acked\n *   once backed up (so a skipped interval ends at or before the replica), and\n *   if the change log is behind the slot it is re-initialized from the\n *   replica. Note that the slot can be past the replica (acked for keepalives,\n *   with nothing skipped), and the replica past the change log (forwarded\n *   before being stored, and streamed again from the slot), but the stream\n *   never starts below both. (With RMv1.5's purge lock, the change log's head\n *   is at or past the slot, so it does not lower the minimum.)\n * * With a shared slot (RMv1), or a resumed slot that was acked only once\n *   transactions were stored in the change log (RMv1.5), the change log's\n *   head when it was purge locked, as the stream resumes from the head, which\n *   only advances. The slot is not a reference: a shared slot is acked by its\n *   owner until it is taken over, and a resumed RMv1.5 slot can be past the\n *   head for acked keepalives. Without a purge lock (i.e. an empty change\n *   log), there is no floor.\n */\n// Exported for testing.\nexport function minStartWatermark(\n  replicaWatermark: string,\n  initialSynced: boolean,\n  slotWatermark: string | undefined,\n  purgeLock: ConstrainingPurgeLock | null,\n): string | undefined {\n  if (initialSynced) {\n    return replicaWatermark;\n  }\n  if (slotWatermark !== undefined) {\n    return replicaWatermark < slotWatermark ? replicaWatermark : slotWatermark;\n  }\n  return purgeLock?.headWatermark;\n}\n\nfunction restoreConstraints(\n  purgeLock: ConstrainingPurgeLock | null,\n  {generation}: ReplicaState,\n): ReplicaConstraints {\n  return purgeLock ?? {replicaVersion: generation, minWatermark: ''};\n}\n\nasync function currentWalLSN(sql: PostgresDB): Promise<LSN> {\n  const [{lsn}] = await sql<{lsn: LSN}[]>`SELECT pg_current_wal_lsn() AS lsn`;\n  return lsn;\n}\n\nconst REPLICA_POLL_INTERVAL_MS = 500;\n// Polling is frequent so that a fork proceeds as soon as the source replica\n// passes the fork LSN; logging is throttled by time rather than per poll.\nconst PROGRESS_LOG_INTERVAL_MS = 5_000;\nconst SLOW_PROGRESS_LOG_INTERVAL_MS = 60_000;\nconst DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS = 20_000;\n\n// Exported for testing.\nexport async function getSourceAndDestinationReplicas(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  epoch: number,\n  slotFailover: boolean,\n  gracePeriodMs: number,\n  pollIntervalMs = REPLICA_POLL_INTERVAL_MS,\n): Promise<\n  | {\n      restoreFrom: ReplicaState;\n      replicateTo: ReplicaState;\n      /** The position of the slot from which replication resumes. */\n      slotLSN: LSN;\n      /**\n       * The session holding the destination slot active until it is taken\n       * over by the PostgresChangeSource. Destroying it releases the slot.\n       */\n      slotSession: Readable;\n    }\n  | undefined\n> {\n  const inactiveSince = new Map<string, number>(); // tracks replica inactivity\n  let destination: ReplicationSlotResult<void> | undefined;\n  // The position after the creation of the destination replica.\n  let forkLSN: LSN | undefined;\n  const lastLogged = new Map<string, number>();\n  const shouldLog = (key: string, intervalMs: number) => {\n    const now = Date.now();\n    const last = lastLogged.get(key);\n    if (last !== undefined && now - last < intervalMs) {\n      return false;\n    }\n    lastLogged.set(key, now);\n    return true;\n  };\n\n  try {\n    for (let i = 0; ; i++) {\n      if (i > 0) {\n        await sleep(pollIntervalMs);\n      }\n      const replicas = await getRestoreCandidates(sql, shard, epoch);\n      if (shouldLog('replicas', PROGRESS_LOG_INTERVAL_MS)) {\n        lc.info?.(`current replicas at epoch ${epoch}`, {replicas});\n      }\n      if (replicas.length === 0) {\n        lc.info?.(`no suitable replicas to restore from`, {replicas});\n        await endSession(destination?.initialSession);\n        return undefined;\n      }\n      for (const replica of replicas) {\n        // Track inactivity to resume replicas after a grace period.\n        if (replica.active) {\n          inactiveSince.delete(replica.id);\n        } else if (!inactiveSince.has(replica.id)) {\n          inactiveSince.set(replica.id, Date.now());\n        }\n\n        // Note: Only `active` InitialSync replicas are returned from\n        // getRestoreCandidates().\n        if (replica.stage === InitialSync) {\n          // Log periodically; initial-sync can be long\n          if (\n            shouldLog(\n              `initial-sync:${replica.id}`,\n              SLOW_PROGRESS_LOG_INTERVAL_MS,\n            )\n          ) {\n            lc.info?.(`waiting for initial sync of ${replica.id}`, {replica});\n          }\n          break;\n        }\n\n        if (replica.stage === Replicate) {\n          const {active, confirmedFlushLsn} = replica;\n          if (active) {\n            // Create a replication slot to fork the active replica.\n            destination ??= await createReplicaAndSlot(\n              lc,\n              sql,\n              'fork-replica-session',\n              shard,\n              epoch,\n              Date.now().toString(), // replicaID\n              slotFailover,\n              {\n                backupPath: null, // set only after the backup has been confirmed\n                backupV5: true, // RMv2 requires backupV5\n                // The slot is not acked until it is taken over, which sets\n                // this. Until then, it is unknown.\n                pgChangeLog: false,\n              },\n              () => promiseVoid,\n              ReplicaStage.Restore,\n            );\n            // The replica row of the destination is inserted in a transaction\n            // (in the publication) after the destination slot's position.\n            // Waiting for the source replica's slot to pass it (rather than\n            // just the destination slot's position) ensures that a source\n            // replica that maintains the PG change-log has stored a\n            // transaction past the destination slot's position, i.e. that the\n            // change-log can be resumed from the destination slot.\n            forkLSN ??= await currentWalLSN(sql);\n            if (toBigInt(confirmedFlushLsn) < toBigInt(forkLSN)) {\n              if (shouldLog(`fork:${replica.id}`, PROGRESS_LOG_INTERVAL_MS)) {\n                lc.info?.(\n                  `waiting for ${replica.id}@${confirmedFlushLsn} to reach ${forkLSN} (after ${destination.slot.slot_name}@${destination.slot.consistent_point})`,\n                  {replica},\n                );\n              }\n              break;\n            }\n          }\n\n          // If the replica is past the destination LSN, fork it, regardless of\n          // whether its slot is still active. Forking is preferable to resuming,\n          // as resuming removes the replica from being a candidate for\n          // subsequent forks, and carries the risk of stealing the slot from a\n          // task attempting to reconnect.\n          if (\n            destination &&\n            forkLSN &&\n            toBigInt(confirmedFlushLsn) >= toBigInt(forkLSN)\n          ) {\n            lc.info?.(`forking replica ${replica.id}@${replica.slot}`, {\n              replica,\n            });\n            const replicateTo = must(\n              await initRestoreReplica(sql, shard, {\n                sourceID: replica.id,\n                destID: destination.replica.id,\n              }),\n              `replica ${destination.replica.id} was deleted`,\n            );\n            return {\n              restoreFrom: replica,\n              replicateTo,\n              slotLSN: destination.slot.consistent_point,\n              slotSession: destination.initialSession,\n            };\n          }\n        }\n\n        if (\n          replica.stage === ReplicaStage.Restore &&\n          replica.active &&\n          replica.id !== destination?.replica.id\n        ) {\n          // Another task is restoring (e.g. resuming) this replica.\n          if (\n            shouldLog(`restoring:${replica.id}`, SLOW_PROGRESS_LOG_INTERVAL_MS)\n          ) {\n            lc.info?.(\n              `waiting for replica ${replica.id}@${replica.slot} being restored by another task`,\n              {replica},\n            );\n          }\n        }\n\n        if (replica.backupPath && !replica.active) {\n          // An inactive replica that has a backupPath may be orphaned, or it\n          // may be an active task that was temporarily disconnected and\n          // attempting to reestablish a session.\n          //\n          // Resume the replica after a grace period to avoid stealing the slot\n          // from an active task that was temporarily disconnected. Resumption\n          // is a last resort only taken in the absence of alternatives.\n          const now = Date.now();\n          const inactiveMs = now - (inactiveSince.get(replica.id) ?? now);\n          if (\n            inactiveMs >= gracePeriodMs ||\n            shouldLog(`inactive:${replica.id}`, PROGRESS_LOG_INTERVAL_MS)\n          ) {\n            lc.info?.(\n              `replica ${replica.id}@${replica.slot} has been inactive for ${inactiveMs}ms`,\n              {replica},\n            );\n          }\n          if (inactiveMs >= gracePeriodMs) {\n            const reserved = await claimSlotForResumption(\n              lc,\n              sql,\n              shard,\n              'resume-replica',\n              replica,\n            );\n            if (!reserved) {\n              lc.warn?.(\n                `unable to resume replica ${replica.id}@${replica.slot}`,\n              );\n            } else {\n              lc.info?.(`resuming replica ${replica.id}@${replica.slot}`);\n              // If a new replication slot was created in anticipation of forking,\n              // cancel it\n              await endSession(destination?.initialSession);\n              return {\n                restoreFrom: replica,\n                replicateTo: reserved.replica,\n                // The slot is inactive, so its position is fixed.\n                slotLSN: replica.confirmedFlushLsn,\n                slotSession: reserved.reservation,\n              };\n            }\n          }\n        }\n      }\n    }\n  } catch (e) {\n    await endSession(destination?.initialSession);\n    throw e;\n  }\n}\n\nasync function checkAndUpdateUpstream(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardConfig,\n  {\n    replicaVersion,\n    publications: subscribed,\n    initialSyncContext,\n  }: SubscriptionStateAndContext,\n  replicaID: string | undefined,\n) {\n  const upstreamReplica = await getReplicaAtVersion(\n    lc,\n    sql,\n    shard,\n    replicaVersion,\n    replicaID,\n    initialSyncContext,\n  );\n  if (!upstreamReplica) {\n    throw new AutoResetSignal(\n      `No replication slot for replica at version ${replicaVersion} and id ${replicaID}}`,\n    );\n  }\n\n  // Verify that the publications match what is being replicated.\n  const requested = shard.publications.toSorted();\n  const replicated = upstreamReplica.publications\n    .filter(p => !p.startsWith(internalPublicationPrefix(shard)))\n    .sort();\n  if (!deepEqual(requested, replicated)) {\n    lc.warn?.(`Dropping shard to change publications to: [${requested}]`);\n    await sql.unsafe(dropShard(shard.appID, shard.shardNum));\n    throw new AutoResetSignal(\n      `Requested publications [${requested}] do not match configured ` +\n        `publications: [${replicated}]`,\n    );\n  }\n\n  // Sanity check: The subscription state on the replica should have the\n  // same publications. This should be guaranteed by the equivalence of the\n  // replicaVersion, but it doesn't hurt to verify.\n  if (!deepEqual(upstreamReplica.publications, subscribed)) {\n    throw new AutoResetSignal(\n      `Upstream publications [${upstreamReplica.publications}] do not ` +\n        `match subscribed publications [${subscribed}]`,\n    );\n  }\n\n  // Verify that the publications exist.\n  const exists = await sql`\n    SELECT pubname FROM pg_publication WHERE pubname IN ${sql(subscribed)};\n  `.values();\n  if (exists.length !== subscribed.length) {\n    throw new AutoResetSignal(\n      `Upstream publications [${exists.flat()}] do not contain ` +\n        `all subscribed publications [${subscribed}]`,\n    );\n  }\n\n  const {slot} = upstreamReplica;\n  const result = await sql<{restartLSN: LSN | null; walStatus: string | null}[]>\n  /*sql*/ `\n    SELECT restart_lsn as \"restartLSN\", wal_status as \"walStatus\" FROM pg_replication_slots\n      WHERE slot_name = ${slot}`;\n  if (result.length === 0) {\n    throw new AutoResetSignal(`replication slot ${slot} is missing`);\n  }\n  const [{restartLSN, walStatus}] = result;\n  if (restartLSN === null || walStatus === 'lost') {\n    throw new AutoResetSignal(\n      `replication slot ${slot} has been invalidated for exceeding the max_slot_wal_keep_size`,\n    );\n  }\n  const [{pgVersion}] = await sql<{pgVersion: number}[]> /*sql*/ `\n    SELECT current_setting('server_version_num')::int as \"pgVersion\"`;\n  return {upstreamReplica, pgVersion};\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAyEA,eAAsB,+BACpB,IACA,aACA,OACA,eACA,aACA,SACA,sBAAsB,GACtB,iBAAiC,CAAC,GAClC,EACE,OACA,gBACA,UACA,8BACA,gBACqB;CACrB,OAAO;CACP,gBAAgB;CAChB,UAAU;CACV,8BAA8B;CAC9B,aAAa;AACf,GACA,wBAC2B;CAC3B,MAAM,KAAK,MAAM,gBAAgB,IAAI,aAAa,oBAAoB;CACtE,IAAI;EACF,MAAM,kBACJ,IACA,IACA,OACA,YAAY,2BACd;EAEA,MAAM,EACJ,SAAS,iBACT,WACA,uBACA,iBACA,kBACE,iBACA,MAAM,oBACJ,IACA,IACA,OACA,OACA,YAAY,2BAA2B,OACvC,eACA,gBACA,4BACF,IACA,MAAM,wBACJ,IACA,IACA,OACA,eACA,cACF;EAEJ,IAAI;EACJ,MAAM,YACJ,IACA,WAAW,MAAM,MAAM,GAAG,MAAM,YAChC,eACA,OAAO,KAAK,OAAO;GAKjB,MAAM,eAAe,SAAS,QAAQ;GAMtC,MAAM,WAAW,QAAQ;GACzB,uBAAuB,MAAM,YAC3B,KACA,OACA,IACA,aACA,aACA,SACA;IAAC;IAAO;IAAU;GAAW,GAC7B,eAAe,OACjB;EACF,CACF;EAEA,MAAM,UAAU,IAAI,SAAS,IAAI,aAAa;EAC9C,MAAM,oBAAoB,+BACxB,IAAI,gBAAgB,OAAO,CAC7B;EACA,QAAQ,MAAM;EAId,MAAM,EAAC,iBAAiB,cAAa,MAAM,uBACzC,IACA,IACA,OACA,oBACC,wBAAwB,kBAAkB,EAC7C;EAIA,MAAM,mBAAmB,YAAY,CAAC;EACtC,MAAM,aAAa,uBAEf,qBAAqB,aAIrB,WACG,iBAAiB,cAAc,OAAO,KAAK,IAAI,CAAC,IAChD,iBAAiB,cAAc;EA2BtC,OAAO;GACL;GACA,cAAA,IA3BuB,qBACvB,IACA,aACA,OACA,iBACA,WACA;IAAC;IAAY;IAAU;GAAW,GAClC,SACA,qBACA,YAAY,UACZ,wBACA,kBACE,kBAAkB,WAClB,yBAAyB,KAAA,GACzB,eACA,SACF,CAWA;GACA,sBARA,cAAc,eAAe,YAAY,YACrC,IAAI,IAAI,YAAY,eAAe,WAAW,SAAS,EAAE,SAAS,IAElE,eAAe,YAAY;GAU/B,WAAW,gBAAgB;GAC3B,4BAEE,yBAAyB,KAAA,KAGzB,gBAAgB,iBAAiB,cAAc;GACjD;GACA;GACA,WAAW,iBAAiB;EAC9B;CACF,UAAU;EACR,MAAM,GAAG,IAAI;CACf;AACF;AAqBA,eAAe,wBACb,IACA,KACA,OACA,aACA,EAAC,YAAY,kBAAkB,WACL;CAM1B,MAAM,OAAO,MAAM,mBAAmB;CACtC,OAAO,SAAS,eAAe,8BAA8B;CAC7D,MAAM,YAAY,QAAQ;CAC1B,MAAM,YAAY,MAAM,kBAAkB,IAAI,KAAK,KAAK,GAAG,QAExD,EAAC,iBAAgB,gBAAgB,WAAW,kBAAkB,WACjE;CACA,IAAI,SAAS,WAAW,GAAG;EACzB,GAAG,OAAO,wCAAwC,EAAC,SAAQ,CAAC;EAC5D,OAAO;GAAC,SAAS,KAAA;GAAW;EAAS;CACvC;CACA,MAAM,CAAC,WAAW;CAElB,IAAI,YAAY,WAAW;EACzB,MAAM,cAAc,mBAAmB,WAAW,OAAO;EACzD,MAAM,EAAC,WAAW,kBAAiB;EACnC,MAAM,EAAC,MAAM,YAAY,sBAAqB;EAC9C,MAAM,YAAY,IAAI,IAAI,cAAc,IAAI,aAAa,EAAE,SAAS;EACpE,GAAG,OACD,0BAA0B,UAAU,IAAI,KAAK,GAAG,kBAAkB,IAClE,EAAC,SAAQ,CACX;EAWA,OAAO;GAAC;GAAS;GAAW,iBAAA,MAVE,eAC5B,IACA;IAAC,GAAG;IAAY;GAAS,GACzB,aACA,aACA,WAAW,aACP;IAAC,MAAM;IAAQ,SAAS;GAAa,IACrC,KAAA,GACJ,OACF;EAC2C;CAC7C;CACA,OAAO;EAAC;EAAS;CAAS;AAC5B;AAMA,eAAe,oBACb,IACA,KACA,OACA,OACA,cACA,aACA,EAAC,YAAY,kBAAkB,WAC/B,eAC0B;CAC1B,MAAM,SAAS,MAAM,gCACnB,IACA,KACA,OACA,OACA,cACA,aACF;CACA,IAAI,CAAC,QAEH,OAAO;EAAC,SAAS,KAAA;EAAW,WAAW;CAAI;CAE7C,MAAM,EAAC,aAAa,aAAa,SAAS,gBAAe;CAQzD,MAAM,gBADJ,YAAY,OAAO,YAAY,MAAM,YAAY,cAE/C,KAAA,IACA,qBAAqB,OAAO;CAKhC,SAAS,UAAU,oBAAoB,YAAY,cACjD,YAAY,QAAQ,CACtB;CAoBA,MAAM,OAAO,mBAAmB,MAAM,iBAAiB,aAAa,IAAI;CACxE,MAAM,wBAAwB,SAAS;CACvC,MAAM,YAAY,SAAS,gBAAgB,OAAO;CAIlD,MAAM,cAAc,mBAAmB,WAAW,WAAW;CAE7D,IAAI,YAAY,WAAW;EACzB,MAAM,EAAC,WAAW,kBAAiB;EACnC,MAAM,EAAC,MAAM,YAAY,sBAAqB;EAC9C,MAAM,YAAY,IAAI,IAAI,cAAc,IAAI,aAAa,EAAE,SAAS;EACpE,GAAG,OACD,0BAA0B,UAAU,IAAI,KAAK,GAAG,kBAAkB,IAClE;GAAC;GAAa;EAAW,CAC3B;EAWA,OAAO;GACL,SAAS;GACT;GACA;GACA,iBAAA,MAd4B,eAC5B,IACA;IAAC,GAAG;IAAY;GAAS,GACzB,aACA,aACA,WAAW,aACP;IAAC,MAAM;IAAQ,SAAS;GAAa,IACrC,KAAA,GACJ,OACF;GAME;EACF;CACF;CACA,OAAO;EACL,SAAS;EACT;EACA;EACA;CACF;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8BA,SAAgB,kBACd,kBACA,eACA,eACA,WACoB;CACpB,IAAI,eACF,OAAO;CAET,IAAI,kBAAkB,KAAA,GACpB,OAAO,mBAAmB,gBAAgB,mBAAmB;CAE/D,OAAO,WAAW;AACpB;AAEA,SAAS,mBACP,WACA,EAAC,cACmB;CACpB,OAAO,aAAa;EAAC,gBAAgB;EAAY,cAAc;CAAE;AACnE;AAEA,eAAe,cAAc,KAA+B;CAC1D,MAAM,CAAC,EAAC,SAAQ,MAAM,GAAiB;CACvC,OAAO;AACT;AAEA,IAAM,2BAA2B;AAGjC,IAAM,2BAA2B;AACjC,IAAM,gCAAgC;AACtC,IAAM,2CAA2C;AAGjD,eAAsB,gCACpB,IACA,KACA,OACA,OACA,cACA,eACA,iBAAiB,0BAcjB;CACA,MAAM,gCAAgB,IAAI,IAAoB;CAC9C,IAAI;CAEJ,IAAI;CACJ,MAAM,6BAAa,IAAI,IAAoB;CAC3C,MAAM,aAAa,KAAa,eAAuB;EACrD,MAAM,MAAM,KAAK,IAAI;EACrB,MAAM,OAAO,WAAW,IAAI,GAAG;EAC/B,IAAI,SAAS,KAAA,KAAa,MAAM,OAAO,YACrC,OAAO;EAET,WAAW,IAAI,KAAK,GAAG;EACvB,OAAO;CACT;CAEA,IAAI;EACF,KAAK,IAAI,IAAI,IAAK,KAAK;GACrB,IAAI,IAAI,GACN,MAAM,MAAM,cAAc;GAE5B,MAAM,WAAW,MAAM,qBAAqB,KAAK,OAAO,KAAK;GAC7D,IAAI,UAAU,YAAY,wBAAwB,GAChD,GAAG,OAAO,6BAA6B,SAAS,EAAC,SAAQ,CAAC;GAE5D,IAAI,SAAS,WAAW,GAAG;IACzB,GAAG,OAAO,wCAAwC,EAAC,SAAQ,CAAC;IAC5D,MAAM,WAAW,aAAa,cAAc;IAC5C;GACF;GACA,KAAK,MAAM,WAAW,UAAU;IAE9B,IAAI,QAAQ,QACV,cAAc,OAAO,QAAQ,EAAE;SAC1B,IAAI,CAAC,cAAc,IAAI,QAAQ,EAAE,GACtC,cAAc,IAAI,QAAQ,IAAI,KAAK,IAAI,CAAC;IAK1C,IAAI,QAAQ,UAAA,GAAuB;KAEjC,IACE,UACE,gBAAgB,QAAQ,MACxB,6BACF,GAEA,GAAG,OAAO,+BAA+B,QAAQ,MAAM,EAAC,QAAO,CAAC;KAElE;IACF;IAEA,IAAI,QAAQ,UAAA,GAAqB;KAC/B,MAAM,EAAC,QAAQ,sBAAqB;KACpC,IAAI,QAAQ;MAEV,gBAAgB,MAAM,qBACpB,IACA,KACA,wBACA,OACA,OACA,KAAK,IAAI,EAAE,SAAS,GACpB,cACA;OACE,YAAY;OACZ,UAAU;OAGV,aAAa;MACf,SACM,aACN,CACF;MAQA,YAAY,MAAM,cAAc,GAAG;MACnC,IAAI,SAAS,iBAAiB,IAAI,SAAS,OAAO,GAAG;OACnD,IAAI,UAAU,QAAQ,QAAQ,MAAM,wBAAwB,GAC1D,GAAG,OACD,eAAe,QAAQ,GAAG,GAAG,kBAAkB,YAAY,QAAQ,UAAU,YAAY,KAAK,UAAU,GAAG,YAAY,KAAK,iBAAiB,IAC7I,EAAC,QAAO,CACV;OAEF;MACF;KACF;KAOA,IACE,eACA,WACA,SAAS,iBAAiB,KAAK,SAAS,OAAO,GAC/C;MACA,GAAG,OAAO,mBAAmB,QAAQ,GAAG,GAAG,QAAQ,QAAQ,EACzD,QACF,CAAC;MAQD,OAAO;OACL,aAAa;OACb,aATkB,KAClB,MAAM,mBAAmB,KAAK,OAAO;QACnC,UAAU,QAAQ;QAClB,QAAQ,YAAY,QAAQ;OAC9B,CAAC,GACD,WAAW,YAAY,QAAQ,GAAG,aAIlC;OACA,SAAS,YAAY,KAAK;OAC1B,aAAa,YAAY;MAC3B;KACF;IACF;IAEA,IACE,QAAQ,UAAU,KAClB,QAAQ,UACR,QAAQ,OAAO,aAAa,QAAQ;SAIlC,UAAU,aAAa,QAAQ,MAAM,6BAA6B,GAElE,GAAG,OACD,uBAAuB,QAAQ,GAAG,GAAG,QAAQ,KAAK,kCAClD,EAAC,QAAO,CACV;IAAA;IAIJ,IAAI,QAAQ,cAAc,CAAC,QAAQ,QAAQ;KAQzC,MAAM,MAAM,KAAK,IAAI;KACrB,MAAM,aAAa,OAAO,cAAc,IAAI,QAAQ,EAAE,KAAK;KAC3D,IACE,cAAc,iBACd,UAAU,YAAY,QAAQ,MAAM,wBAAwB,GAE5D,GAAG,OACD,WAAW,QAAQ,GAAG,GAAG,QAAQ,KAAK,yBAAyB,WAAW,KAC1E,EAAC,QAAO,CACV;KAEF,IAAI,cAAc,eAAe;MAC/B,MAAM,WAAW,MAAM,uBACrB,IACA,KACA,OACA,kBACA,OACF;MACA,IAAI,CAAC,UACH,GAAG,OACD,4BAA4B,QAAQ,GAAG,GAAG,QAAQ,MACpD;WACK;OACL,GAAG,OAAO,oBAAoB,QAAQ,GAAG,GAAG,QAAQ,MAAM;OAG1D,MAAM,WAAW,aAAa,cAAc;OAC5C,OAAO;QACL,aAAa;QACb,aAAa,SAAS;QAEtB,SAAS,QAAQ;QACjB,aAAa,SAAS;OACxB;MACF;KACF;IACF;GACF;EACF;CACF,SAAS,GAAG;EACV,MAAM,WAAW,aAAa,cAAc;EAC5C,MAAM;CACR;AACF;AAEA,eAAe,uBACb,IACA,KACA,OACA,EACE,gBACA,cAAc,YACd,sBAEF,WACA;CACA,MAAM,kBAAkB,MAAM,oBAC5B,IACA,KACA,OACA,gBACA,WACA,kBACF;CACA,IAAI,CAAC,iBACH,MAAM,IAAI,gBACR,8CAA8C,eAAe,UAAU,UAAU,EACnF;CAIF,MAAM,YAAY,MAAM,aAAa,SAAS;CAC9C,MAAM,aAAa,gBAAgB,aAChC,QAAO,MAAK,CAAC,EAAE,WAAW,0BAA0B,KAAK,CAAC,CAAC,EAC3D,KAAK;CACR,IAAI,CAAC,UAAU,WAAW,UAAU,GAAG;EACrC,GAAG,OAAO,8CAA8C,UAAU,EAAE;EACpE,MAAM,IAAI,OAAO,UAAU,MAAM,OAAO,MAAM,QAAQ,CAAC;EACvD,MAAM,IAAI,gBACR,2BAA2B,UAAU,2CACjB,WAAW,EACjC;CACF;CAKA,IAAI,CAAC,UAAU,gBAAgB,cAAc,UAAU,GACrD,MAAM,IAAI,gBACR,0BAA0B,gBAAgB,aAAa,0CACnB,WAAW,EACjD;CAIF,MAAM,SAAS,MAAM,GAAG;0DACgC,IAAI,UAAU,EAAE;IACtE,OAAO;CACT,IAAI,OAAO,WAAW,WAAW,QAC/B,MAAM,IAAI,gBACR,0BAA0B,OAAO,KAAK,EAAE,gDACN,WAAW,EAC/C;CAGF,MAAM,EAAC,SAAQ;CACf,MAAM,SAAS,MAAM,GACb;;0BAEgB;CACxB,IAAI,OAAO,WAAW,GACpB,MAAM,IAAI,gBAAgB,oBAAoB,KAAK,YAAY;CAEjE,MAAM,CAAC,EAAC,YAAY,eAAc;CAClC,IAAI,eAAe,QAAQ,cAAc,QACvC,MAAM,IAAI,gBACR,oBAAoB,KAAK,+DAC3B;CAEF,MAAM,CAAC,EAAC,eAAc,MAAM,GAAmC;;CAE/D,OAAO;EAAC;EAAiB;CAAS;AACpC"}
+{"version":3,"file":"change-source-init.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source-init.ts"],"sourcesContent":["import type {Readable} from 'node:stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../../shared/src/resolved-promises.ts';\nimport {sleep} from '../../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport {connectPgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {type ShardConfig, type ShardID} from '../../../types/shards.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport type {ReplicaConstraints} from '../../litestream/commands.ts';\nimport {\n  getSubscriptionStateAndContext,\n  type SubscriptionStateAndContext,\n} from '../../replicator/schema/replication-state.ts';\nimport {\n  restoreReplica,\n  type ConstrainingPurgeLock,\n  type InitializeResult,\n  type RestoreOptions,\n  type ResumableBackup,\n} from '../common/replica-restore.ts';\nimport {initReplica} from '../common/replica-schema.ts';\nimport {PostgresChangeSource} from './change-source.ts';\nimport {\n  initialSync,\n  type InitialSyncOptions,\n  type ReplicaOptions,\n  type ServerContext,\n} from './initial-sync.ts';\nimport {endSession} from './logical-replication/stream.ts';\nimport {toBigInt, toStateVersionString, type LSN} from './lsn.ts';\nimport {\n  claimSlotForResumption,\n  createReplicaAndSlot,\n  type ReplicationSlotResult,\n} from './replication-slots.ts';\nimport {ensureShardSchema} from './schema/init.ts';\nimport * as ReplicaStage from './schema/replica-stage-enum.ts';\nimport {InitialSync, Replicate} from './schema/replica-stage-enum.ts';\nimport {\n  dropShard,\n  getActiveReplicas,\n  getReplicaAtVersion,\n  getRestoreCandidates,\n  initRestoreReplica,\n  internalPublicationPrefix,\n  type ReplicaState,\n} from './schema/shard.ts';\n\nexport type InitializeOptions = ReplicaOptions & {\n  // Create a new slot for the replica rather than taking over\n  // an existing one (i.e. high-availability mode).\n  slotPerReplica: boolean | undefined;\n\n  inactiveReplicaGracePeriodMs: number;\n\n  /**\n   * Whether this replication-manager maintains the PG change log, i.e. acks\n   * the replication slot only after transactions are stored in it. Recorded\n   * with the replica (see `pgChangeLog` in `ReplicaState`) when its slot is\n   * taken over.\n   */\n  pgChangeLog: boolean;\n};\n\n/**\n * Initializes a Postgres change source, including the initial sync of the\n * replica, before streaming changes from the corresponding logical replication\n * stream.\n */\nexport async function initializePostgresChangeSource(\n  lc: LogContext,\n  upstreamURI: string,\n  shard: ShardConfig,\n  replicaDbFile: string,\n  syncOptions: InitialSyncOptions,\n  context: ServerContext,\n  lagReportIntervalMs = 0,\n  restoreOptions: RestoreOptions = {},\n  {\n    epoch,\n    slotPerReplica,\n    backupV5,\n    inactiveReplicaGracePeriodMs,\n    pgChangeLog,\n  }: InitializeOptions = {\n    epoch: 0,\n    slotPerReplica: false,\n    backupV5: true,\n    inactiveReplicaGracePeriodMs: DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS,\n    pgChangeLog: false,\n  },\n  streamInboundTimeoutMs?: number | undefined,\n): Promise<InitializeResult> {\n  const db = await connectPgClient(lc, upstreamURI, 'change-source-init');\n  try {\n    await ensureShardSchema(\n      lc,\n      db,\n      shard,\n      syncOptions.installPartialIndexTriggers,\n    );\n\n    const {\n      replica: restoredReplica,\n      purgeLock,\n      pgChangeLogBehindSlot,\n      resumableBackup,\n      slotWatermark,\n    } = slotPerReplica\n      ? await forkOrResumeReplica(\n          lc,\n          db,\n          shard,\n          epoch,\n          syncOptions.replicationSlotFailover ?? false,\n          replicaDbFile,\n          restoreOptions,\n          inactiveReplicaGracePeriodMs,\n        )\n      : await selectAndRestoreReplica(\n          lc,\n          db,\n          shard,\n          replicaDbFile,\n          restoreOptions,\n        );\n\n    let initialSyncedReplica: ReplicaState | undefined;\n    await initReplica(\n      lc,\n      `replica-${shard.appID}-${shard.shardNum}`,\n      replicaDbFile,\n      async (log, tx) => {\n        // Falling back to initial sync abandons the restore, so release the\n        // resources acquired for it, e.g. a replication slot claimed (or\n        // created) for a resumed (or forked) replica, which would otherwise\n        // remain active indefinitely.\n        await restoreOptions.cleanup?.release();\n        // The purge lock on the change-db must be released before performing\n        // initial sync; if the change-db and upstream are the same db, a lock-holding\n        // transaction will prevent a replication slot from being created. This awkward\n        // dependency can go away with RMv2. (The purge lock is also released by\n        // the cleanup when it is registered with it; releasing it is idempotent.)\n        await purgeLock?.release();\n        initialSyncedReplica = await initialSync(\n          log,\n          shard,\n          tx,\n          upstreamURI,\n          syncOptions,\n          context,\n          {epoch, backupV5, pgChangeLog},\n          restoreOptions.cleanup,\n        );\n      },\n    );\n\n    const replica = new Database(lc, replicaDbFile);\n    const subscriptionState = getSubscriptionStateAndContext(\n      new StatementRunner(replica),\n    );\n    replica.close();\n\n    // Check that upstream is properly setup, and throw an AutoReset to re-run\n    // initial sync if not.\n    const {upstreamReplica, pgVersion} = await checkAndUpdateUpstream(\n      lc,\n      db,\n      shard,\n      subscriptionState,\n      (initialSyncedReplica ?? restoredReplica)?.id,\n    );\n\n    // With litestream v5, every replication-manager backs up to its own\n    // backup lineage.\n    const newBackupLineage = backupV5 && !resumableBackup;\n    const backupPath = initialSyncedReplica\n      ? // If initial sync was performed, use that initial backupPath.\n        initialSyncedReplica.backupPath\n      : // Otherwise, use the lineage prepared by the restore, or a new, unique\n        // path when backing up with litestream v5. This will be recorded in the\n        // replicas table by the PostgresChangeSource.\n        backupV5\n        ? (resumableBackup?.backupPath ?? String(Date.now()))\n        : (restoredReplica?.backupPath ?? null);\n\n    const changeSource = new PostgresChangeSource(\n      lc,\n      upstreamURI,\n      shard,\n      upstreamReplica,\n      pgVersion,\n      {backupPath, backupV5, pgChangeLog},\n      context,\n      lagReportIntervalMs,\n      syncOptions.textCopy,\n      streamInboundTimeoutMs,\n      minStartWatermark(\n        subscriptionState.watermark,\n        initialSyncedReplica !== undefined,\n        slotWatermark,\n        purgeLock,\n      ),\n    );\n\n    const destinationBackupURL =\n      backupPath && restoreOptions.litestream?.backupURL\n        ? new URL(backupPath, restoreOptions.litestream.backupURL).toString()\n        : // For legacy RMv1 replicas (on litestream-v3), backup to the same location\n          restoreOptions.litestream?.backupURL;\n\n    return {\n      subscriptionState,\n      changeSource,\n      destinationBackupURL,\n      // The replica this change stream belongs to. It is part of the identity\n      // the SQLite change log records, because a generation (i.e.\n      // `replicaVersion`) is shared by every sibling of a forked replica and so\n      // cannot distinguish two siblings' logs.\n      replicaID: upstreamReplica.id,\n      waitForBackupBeforeServing:\n        // Wait for the first backup if there was an initial sync,\n        initialSyncedReplica !== undefined ||\n        // or if the destination differs from where it was restored\n        // (i.e. backupV5).\n        backupPath !== (restoredReplica?.backupPath ?? null),\n      pgChangeLogBehindSlot,\n      newBackupLineage,\n    };\n  } finally {\n    await db.end();\n  }\n}\n\ntype RestoredReplica = {\n  replica: ReplicaState | undefined;\n  purgeLock: ConstrainingPurgeLock | null;\n  pgChangeLogBehindSlot?: boolean | undefined;\n  resumableBackup?: ResumableBackup | undefined;\n  /**\n   * The position (as a watermark) of the replication slot from which\n   * replication resumes, when it was created (for a fork) or claimed (for a\n   * resumption), i.e. while it is held by this process. Absent for a shared\n   * slot (RMv1), whose position is advanced by its current owner until it is\n   * taken over, and for a resumed slot that was acked only once transactions\n   * were stored in the PG change-log (RMv1.5), whose position may be past the\n   * change-log's head for acked keepalives.\n   */\n  slotWatermark?: string | undefined;\n};\n\n// RMv1: Selects a replica to restore from and returns it, with the\n//       intention of taking over the slot (in the ChangeSource).\nasync function selectAndRestoreReplica(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  replicaFile: string,\n  {litestream, acquirePurgeLock}: RestoreOptions,\n): Promise<RestoredReplica> {\n  // The purge lock constrains the generation of the replica to restore. With\n  // a shared replication slot, the change-log is resumed from its head\n  // without being checked against the slot's position: the slot is still\n  // being acked by the previous replication-manager, on an idle upstream past\n  // the change-log's head (see PgChangeLogPurgeLocker).\n  const lock = await acquirePurgeLock?.();\n  assert(lock !== 'behind-slot', 'unexpected purge-lock result');\n  const purgeLock = lock ?? null;\n  const replicas = (await getActiveReplicas(lc, sql, shard)).filter(\n    // filter to the generation specified by the purge lock, if present\n    ({generation}) => generation === (purgeLock?.replicaVersion ?? generation),\n  );\n  if (replicas.length === 0) {\n    lc.info?.(`no suitable replicas to restore from`, {replicas});\n    return {replica: undefined, purgeLock};\n  }\n  const [replica] = replicas;\n\n  if (litestream?.backupURL) {\n    const constraints = restoreConstraints(purgeLock, replica);\n    const {backupURL: backupBaseURL} = litestream;\n    const {slot, backupPath, confirmedFlushLsn} = replica;\n    const backupURL = new URL(backupPath ?? '', backupBaseURL).toString();\n    lc.info?.(\n      `restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`,\n      {replicas},\n    );\n    const resumableBackup = await restoreReplica(\n      lc,\n      {...litestream, backupURL}, // includes the replica's backup sub-path\n      replicaFile,\n      constraints,\n      litestream.forkBackup\n        ? {type: 'fork', baseURL: backupBaseURL}\n        : undefined,\n    );\n    return {replica, purgeLock, resumableBackup};\n  }\n  return {replica, purgeLock};\n}\n\n// RMv2: Restores from an active replica and creates a new replica / slot\n// to continue replication (i.e. \"fork\"). If only orphaned replicas remain,\n// claims a slot, restores the backup, and \"resumes\" replication for that\n// slot.\nasync function forkOrResumeReplica(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  epoch: number,\n  slotFailover: boolean,\n  replicaFile: string,\n  {litestream, acquirePurgeLock, cleanup}: RestoreOptions,\n  gracePeriodMs: number,\n): Promise<RestoredReplica> {\n  const result = await getSourceAndDestinationReplicas(\n    lc,\n    sql,\n    shard,\n    epoch,\n    slotFailover,\n    gracePeriodMs,\n  );\n  if (!result) {\n    // can't restore, must initial-sync\n    return {replica: undefined, purgeLock: null};\n  }\n  const {restoreFrom, replicateTo, slotLSN, slotSession} = result;\n  // A resumed slot (i.e. of the same replica) that was acked only once\n  // transactions were stored in the PG change-log (RMv1.5) cannot have been\n  // acked past a transaction missing from the change-log. Its position past\n  // the change-log's head only reflects acked keepalives, so it is not\n  // compared with the head (see below).\n  const resumedAfterPgChangeLog =\n    restoreFrom.id === replicateTo.id && restoreFrom.pgChangeLog;\n  const slotWatermark = resumedAfterPgChangeLog\n    ? undefined\n    : toStateVersionString(slotLSN);\n  // The slot is held active by a session in this process until it is taken\n  // over by the PostgresChangeSource. If initialization fails before then,\n  // the session must be released, or the slot remains active indefinitely,\n  // preventing the replica from being resumed (or forked) by a retry.\n  cleanup?.onFailure(`replication slot ${replicateTo.slot}`, () =>\n    slotSession.destroy(),\n  );\n\n  // Acquired only after the replication slot has been created (or claimed),\n  // and only if the change-log can be resumed from the slot's position.\n  //\n  // For a fork, this is the case whenever the change-log is maintained (i.e.\n  // by a replication-manager that acks the slot only once a transaction has\n  // been stored in the change-log): the fork waits for the source replica's\n  // slot to pass the position after the creation of the new replica, which is\n  // a transaction (in the publication) after the new slot's position.\n  //\n  // For a resumed slot, the slot can also have been acked past the head of a\n  // maintained change-log, for keepalives (e.g. for changes outside of the\n  // publication, or on an idle upstream). If the slot was acked by a\n  // replication-manager that maintains the change-log, it is therefore not\n  // checked against the slot's position, which would otherwise re-initialize\n  // the change-log unnecessarily, resetting subscribers behind the replica.\n  // Otherwise (e.g. RMv2, or unknown), it is: the change-log is then\n  // re-initialized from the replica if it does not contain the replica's\n  // changes.\n  const lock = acquirePurgeLock ? await acquirePurgeLock(slotWatermark) : null;\n  const pgChangeLogBehindSlot = lock === 'behind-slot';\n  const purgeLock = lock === 'behind-slot' ? null : lock;\n  // Absent a purge lock, still constrain the restore to the source's\n  // generation, so that a local replica left behind by a different\n  // generation (e.g. a failed initial sync) is not mistaken for it.\n  const constraints = restoreConstraints(purgeLock, restoreFrom);\n\n  if (litestream?.backupURL) {\n    const {backupURL: backupBaseURL} = litestream;\n    const {slot, backupPath, confirmedFlushLsn} = restoreFrom;\n    const backupURL = new URL(backupPath ?? '', backupBaseURL).toString();\n    lc.info?.(\n      `restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`,\n      {restoreFrom, replicateTo},\n    );\n    const resumableBackup = await restoreReplica(\n      lc,\n      {...litestream, backupURL}, // includes the replica's backup sub-path\n      replicaFile,\n      constraints,\n      litestream.forkBackup\n        ? {type: 'fork', baseURL: backupBaseURL}\n        : undefined,\n    );\n    return {\n      replica: replicateTo,\n      purgeLock,\n      pgChangeLogBehindSlot,\n      resumableBackup,\n      slotWatermark,\n    };\n  }\n  return {\n    replica: replicateTo,\n    purgeLock,\n    pgChangeLogBehindSlot,\n    slotWatermark,\n  };\n}\n\n/**\n * The watermark below which the replication stream cannot (correctly) be\n * started, i.e. from which it would skip transactions that the change log\n * does not contain. It reflects the order in which each mode stores a\n * transaction and then acks it to the slot, such that a correctly initialized\n * stream always starts at or past it:\n *\n * * After an initial sync, the replica's watermark, which is the position\n *   at which the new slot was created, and the change log is (re)initialized\n *   at it.\n * * With a slot per replica (RMv1.5 and RMv2), the minimum of the restored\n *   replica's watermark and the slot's position. Transactions are only acked\n *   once backed up (so a skipped interval ends at or before the replica), and\n *   if the change log is behind the slot it is re-initialized from the\n *   replica. Note that the slot can be past the replica (acked for keepalives,\n *   with nothing skipped), and the replica past the change log (forwarded\n *   before being stored, and streamed again from the slot), but the stream\n *   never starts below both. (With RMv1.5's purge lock, the change log's head\n *   is at or past the slot, so it does not lower the minimum.)\n * * With a shared slot (RMv1), or a resumed slot that was acked only once\n *   transactions were stored in the change log (RMv1.5), the change log's\n *   head when it was purge locked, as the stream resumes from the head, which\n *   only advances. The slot is not a reference: a shared slot is acked by its\n *   owner until it is taken over, and a resumed RMv1.5 slot can be past the\n *   head for acked keepalives. Without a purge lock (i.e. an empty change\n *   log), there is no floor.\n */\n// Exported for testing.\nexport function minStartWatermark(\n  replicaWatermark: string,\n  initialSynced: boolean,\n  slotWatermark: string | undefined,\n  purgeLock: ConstrainingPurgeLock | null,\n): string | undefined {\n  if (initialSynced) {\n    return replicaWatermark;\n  }\n  if (slotWatermark !== undefined) {\n    return replicaWatermark < slotWatermark ? replicaWatermark : slotWatermark;\n  }\n  return purgeLock?.headWatermark;\n}\n\nfunction restoreConstraints(\n  purgeLock: ConstrainingPurgeLock | null,\n  {generation}: ReplicaState,\n): ReplicaConstraints {\n  return purgeLock ?? {replicaVersion: generation, minWatermark: ''};\n}\n\nasync function currentWalLSN(sql: PostgresDB): Promise<LSN> {\n  const [{lsn}] = await sql<{lsn: LSN}[]>`SELECT pg_current_wal_lsn() AS lsn`;\n  return lsn;\n}\n\nconst REPLICA_POLL_INTERVAL_MS = 500;\n// Polling is frequent so that a fork proceeds as soon as the source replica\n// passes the fork LSN; logging is throttled by time rather than per poll.\nconst PROGRESS_LOG_INTERVAL_MS = 5_000;\nconst SLOW_PROGRESS_LOG_INTERVAL_MS = 60_000;\nconst DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS = 20_000;\n\n// Exported for testing.\nexport async function getSourceAndDestinationReplicas(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  epoch: number,\n  slotFailover: boolean,\n  gracePeriodMs: number,\n  pollIntervalMs = REPLICA_POLL_INTERVAL_MS,\n): Promise<\n  | {\n      restoreFrom: ReplicaState;\n      replicateTo: ReplicaState;\n      /** The position of the slot from which replication resumes. */\n      slotLSN: LSN;\n      /**\n       * The session holding the destination slot active until it is taken\n       * over by the PostgresChangeSource. Destroying it releases the slot.\n       */\n      slotSession: Readable;\n    }\n  | undefined\n> {\n  const inactiveSince = new Map<string, number>(); // tracks replica inactivity\n  let destination: ReplicationSlotResult<void> | undefined;\n  // The position after the creation of the destination replica.\n  let forkLSN: LSN | undefined;\n  const lastLogged = new Map<string, number>();\n  const shouldLog = (key: string, intervalMs: number) => {\n    const now = Date.now();\n    const last = lastLogged.get(key);\n    if (last !== undefined && now - last < intervalMs) {\n      return false;\n    }\n    lastLogged.set(key, now);\n    return true;\n  };\n\n  try {\n    for (let i = 0; ; i++) {\n      if (i > 0) {\n        await sleep(pollIntervalMs);\n      }\n      const replicas = await getRestoreCandidates(sql, shard, epoch);\n      if (shouldLog('replicas', PROGRESS_LOG_INTERVAL_MS)) {\n        lc.info?.(`current replicas at epoch ${epoch}`, {replicas});\n      }\n      if (replicas.length === 0) {\n        lc.info?.(`no suitable replicas to restore from`, {replicas});\n        await endSession(destination?.initialSession);\n        return undefined;\n      }\n      for (const replica of replicas) {\n        // Track inactivity to resume replicas after a grace period.\n        if (replica.active) {\n          inactiveSince.delete(replica.id);\n        } else if (!inactiveSince.has(replica.id)) {\n          inactiveSince.set(replica.id, Date.now());\n        }\n\n        // Note: Only `active` InitialSync replicas are returned from\n        // getRestoreCandidates().\n        if (replica.stage === InitialSync) {\n          // Log periodically; initial-sync can be long\n          if (\n            shouldLog(\n              `initial-sync:${replica.id}`,\n              SLOW_PROGRESS_LOG_INTERVAL_MS,\n            )\n          ) {\n            lc.info?.(`waiting for initial sync of ${replica.id}`, {replica});\n          }\n          break;\n        }\n\n        if (replica.stage === Replicate) {\n          const {active, confirmedFlushLsn} = replica;\n          if (active) {\n            // Create a replication slot to fork the active replica.\n            destination ??= await createReplicaAndSlot(\n              lc,\n              sql,\n              'fork-replica-session',\n              shard,\n              epoch,\n              Date.now().toString(), // replicaID\n              slotFailover,\n              {\n                backupPath: null, // set only after the backup has been confirmed\n                backupV5: true, // RMv2 requires backupV5\n                // The slot is not acked until it is taken over, which sets\n                // this. Until then, it is unknown.\n                pgChangeLog: false,\n              },\n              () => promiseVoid,\n              ReplicaStage.Restore,\n            );\n            // The replica row of the destination is inserted in a transaction\n            // (in the publication) after the destination slot's position.\n            // Waiting for the source replica's slot to pass it (rather than\n            // just the destination slot's position) ensures that a source\n            // replica that maintains the PG change-log has stored a\n            // transaction past the destination slot's position, i.e. that the\n            // change-log can be resumed from the destination slot.\n            forkLSN ??= await currentWalLSN(sql);\n            if (toBigInt(confirmedFlushLsn) < toBigInt(forkLSN)) {\n              if (shouldLog(`fork:${replica.id}`, PROGRESS_LOG_INTERVAL_MS)) {\n                lc.info?.(\n                  `waiting for ${replica.id}@${confirmedFlushLsn} to reach ${forkLSN} (after ${destination.slot.slot_name}@${destination.slot.consistent_point})`,\n                  {replica},\n                );\n              }\n              break;\n            }\n          }\n\n          // If the replica is past the destination LSN, fork it, regardless of\n          // whether its slot is still active. Forking is preferable to resuming,\n          // as resuming removes the replica from being a candidate for\n          // subsequent forks, and carries the risk of stealing the slot from a\n          // task attempting to reconnect.\n          if (\n            destination &&\n            forkLSN &&\n            toBigInt(confirmedFlushLsn) >= toBigInt(forkLSN)\n          ) {\n            lc.info?.(`forking replica ${replica.id}@${replica.slot}`, {\n              replica,\n            });\n            const replicateTo = must(\n              await initRestoreReplica(sql, shard, {\n                sourceID: replica.id,\n                destID: destination.replica.id,\n              }),\n              `replica ${destination.replica.id} was deleted`,\n            );\n            return {\n              restoreFrom: replica,\n              replicateTo,\n              slotLSN: destination.slot.consistent_point,\n              slotSession: destination.initialSession,\n            };\n          }\n        }\n\n        if (\n          replica.stage === ReplicaStage.Restore &&\n          replica.active &&\n          replica.id !== destination?.replica.id\n        ) {\n          // Another task is restoring (e.g. resuming) this replica.\n          if (\n            shouldLog(`restoring:${replica.id}`, SLOW_PROGRESS_LOG_INTERVAL_MS)\n          ) {\n            lc.info?.(\n              `waiting for replica ${replica.id}@${replica.slot} being restored by another task`,\n              {replica},\n            );\n          }\n        }\n\n        if (replica.backupPath && !replica.active) {\n          // An inactive replica that has a backupPath may be orphaned, or it\n          // may be an active task that was temporarily disconnected and\n          // attempting to reestablish a session.\n          //\n          // Resume the replica after a grace period to avoid stealing the slot\n          // from an active task that was temporarily disconnected. Resumption\n          // is a last resort only taken in the absence of alternatives.\n          const now = Date.now();\n          const inactiveMs = now - (inactiveSince.get(replica.id) ?? now);\n          if (\n            inactiveMs >= gracePeriodMs ||\n            shouldLog(`inactive:${replica.id}`, PROGRESS_LOG_INTERVAL_MS)\n          ) {\n            lc.info?.(\n              `replica ${replica.id}@${replica.slot} has been inactive for ${inactiveMs}ms`,\n              {replica},\n            );\n          }\n          if (inactiveMs >= gracePeriodMs) {\n            const reserved = await claimSlotForResumption(\n              lc,\n              sql,\n              shard,\n              'resume-replica',\n              replica,\n            );\n            if (!reserved) {\n              lc.warn?.(\n                `unable to resume replica ${replica.id}@${replica.slot}`,\n              );\n            } else {\n              lc.info?.(`resuming replica ${replica.id}@${replica.slot}`);\n              // If a new replication slot was created in anticipation of forking,\n              // cancel it\n              await endSession(destination?.initialSession);\n              return {\n                restoreFrom: replica,\n                replicateTo: reserved.replica,\n                // The slot is inactive, so its position is fixed.\n                slotLSN: replica.confirmedFlushLsn,\n                slotSession: reserved.reservation,\n              };\n            }\n          }\n        }\n      }\n    }\n  } catch (e) {\n    await endSession(destination?.initialSession);\n    throw e;\n  }\n}\n\nasync function checkAndUpdateUpstream(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardConfig,\n  {\n    replicaVersion,\n    publications: subscribed,\n    initialSyncContext,\n  }: SubscriptionStateAndContext,\n  replicaID: string | undefined,\n) {\n  const upstreamReplica = await getReplicaAtVersion(\n    lc,\n    sql,\n    shard,\n    replicaVersion,\n    replicaID,\n    initialSyncContext,\n  );\n  if (!upstreamReplica) {\n    throw new AutoResetSignal(\n      `No replication slot for replica at version ${replicaVersion} and id ${replicaID}}`,\n    );\n  }\n\n  // Verify that the publications match what is being replicated.\n  const requested = shard.publications.toSorted();\n  const replicated = upstreamReplica.publications\n    .filter(p => !p.startsWith(internalPublicationPrefix(shard)))\n    .sort();\n  if (!deepEqual(requested, replicated)) {\n    lc.warn?.(`Dropping shard to change publications to: [${requested}]`);\n    await sql.unsafe(dropShard(shard.appID, shard.shardNum));\n    throw new AutoResetSignal(\n      `Requested publications [${requested}] do not match configured ` +\n        `publications: [${replicated}]`,\n    );\n  }\n\n  // Sanity check: The subscription state on the replica should have the\n  // same publications. This should be guaranteed by the equivalence of the\n  // replicaVersion, but it doesn't hurt to verify.\n  if (!deepEqual(upstreamReplica.publications, subscribed)) {\n    throw new AutoResetSignal(\n      `Upstream publications [${upstreamReplica.publications}] do not ` +\n        `match subscribed publications [${subscribed}]`,\n    );\n  }\n\n  // Verify that the publications exist.\n  const exists = await sql`\n    SELECT pubname FROM pg_publication WHERE pubname IN ${sql(subscribed)};\n  `.values();\n  if (exists.length !== subscribed.length) {\n    throw new AutoResetSignal(\n      `Upstream publications [${exists.flat()}] do not contain ` +\n        `all subscribed publications [${subscribed}]`,\n    );\n  }\n\n  const {slot} = upstreamReplica;\n  const result = await sql<{restartLSN: LSN | null; walStatus: string | null}[]>\n  /*sql*/ `\n    SELECT restart_lsn as \"restartLSN\", wal_status as \"walStatus\" FROM pg_replication_slots\n      WHERE slot_name = ${slot}`;\n  if (result.length === 0) {\n    throw new AutoResetSignal(`replication slot ${slot} is missing`);\n  }\n  const [{restartLSN, walStatus}] = result;\n  if (restartLSN === null || walStatus === 'lost') {\n    throw new AutoResetSignal(\n      `replication slot ${slot} has been invalidated for exceeding the max_slot_wal_keep_size`,\n    );\n  }\n  const [{pgVersion}] = await sql<{pgVersion: number}[]> /*sql*/ `\n    SELECT current_setting('server_version_num')::int as \"pgVersion\"`;\n  return {upstreamReplica, pgVersion};\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAyEA,eAAsB,+BACpB,IACA,aACA,OACA,eACA,aACA,SACA,sBAAsB,GACtB,iBAAiC,CAAC,GAClC,EACE,OACA,gBACA,UACA,8BACA,gBACqB;CACrB,OAAO;CACP,gBAAgB;CAChB,UAAU;CACV,8BAA8B;CAC9B,aAAa;AACf,GACA,wBAC2B;CAC3B,MAAM,KAAK,MAAM,gBAAgB,IAAI,aAAa,oBAAoB;CACtE,IAAI;EACF,MAAM,kBACJ,IACA,IACA,OACA,YAAY,2BACd;EAEA,MAAM,EACJ,SAAS,iBACT,WACA,uBACA,iBACA,kBACE,iBACA,MAAM,oBACJ,IACA,IACA,OACA,OACA,YAAY,2BAA2B,OACvC,eACA,gBACA,4BACF,IACA,MAAM,wBACJ,IACA,IACA,OACA,eACA,cACF;EAEJ,IAAI;EACJ,MAAM,YACJ,IACA,WAAW,MAAM,MAAM,GAAG,MAAM,YAChC,eACA,OAAO,KAAK,OAAO;GAKjB,MAAM,eAAe,SAAS,QAAQ;GAMtC,MAAM,WAAW,QAAQ;GACzB,uBAAuB,MAAM,YAC3B,KACA,OACA,IACA,aACA,aACA,SACA;IAAC;IAAO;IAAU;GAAW,GAC7B,eAAe,OACjB;EACF,CACF;EAEA,MAAM,UAAU,IAAI,SAAS,IAAI,aAAa;EAC9C,MAAM,oBAAoB,+BACxB,IAAI,gBAAgB,OAAO,CAC7B;EACA,QAAQ,MAAM;EAId,MAAM,EAAC,iBAAiB,cAAa,MAAM,uBACzC,IACA,IACA,OACA,oBACC,wBAAwB,kBAAkB,EAC7C;EAIA,MAAM,mBAAmB,YAAY,CAAC;EACtC,MAAM,aAAa,uBAEf,qBAAqB,aAIrB,WACG,iBAAiB,cAAc,OAAO,KAAK,IAAI,CAAC,IAChD,iBAAiB,cAAc;EA2BtC,OAAO;GACL;GACA,cAAA,IA3BuB,qBACvB,IACA,aACA,OACA,iBACA,WACA;IAAC;IAAY;IAAU;GAAW,GAClC,SACA,qBACA,YAAY,UACZ,wBACA,kBACE,kBAAkB,WAClB,yBAAyB,KAAA,GACzB,eACA,SACF,CAWA;GACA,sBARA,cAAc,eAAe,YAAY,YACrC,IAAI,IAAI,YAAY,eAAe,WAAW,SAAS,EAAE,SAAS,IAElE,eAAe,YAAY;GAU/B,WAAW,gBAAgB;GAC3B,4BAEE,yBAAyB,KAAA,KAGzB,gBAAgB,iBAAiB,cAAc;GACjD;GACA;EACF;CACF,UAAU;EACR,MAAM,GAAG,IAAI;CACf;AACF;AAqBA,eAAe,wBACb,IACA,KACA,OACA,aACA,EAAC,YAAY,oBACa;CAM1B,MAAM,OAAO,MAAM,mBAAmB;CACtC,OAAO,SAAS,eAAe,8BAA8B;CAC7D,MAAM,YAAY,QAAQ;CAC1B,MAAM,YAAY,MAAM,kBAAkB,IAAI,KAAK,KAAK,GAAG,QAExD,EAAC,iBAAgB,gBAAgB,WAAW,kBAAkB,WACjE;CACA,IAAI,SAAS,WAAW,GAAG;EACzB,GAAG,OAAO,wCAAwC,EAAC,SAAQ,CAAC;EAC5D,OAAO;GAAC,SAAS,KAAA;GAAW;EAAS;CACvC;CACA,MAAM,CAAC,WAAW;CAElB,IAAI,YAAY,WAAW;EACzB,MAAM,cAAc,mBAAmB,WAAW,OAAO;EACzD,MAAM,EAAC,WAAW,kBAAiB;EACnC,MAAM,EAAC,MAAM,YAAY,sBAAqB;EAC9C,MAAM,YAAY,IAAI,IAAI,cAAc,IAAI,aAAa,EAAE,SAAS;EACpE,GAAG,OACD,0BAA0B,UAAU,IAAI,KAAK,GAAG,kBAAkB,IAClE,EAAC,SAAQ,CACX;EAUA,OAAO;GAAC;GAAS;GAAW,iBAAA,MATE,eAC5B,IACA;IAAC,GAAG;IAAY;GAAS,GACzB,aACA,aACA,WAAW,aACP;IAAC,MAAM;IAAQ,SAAS;GAAa,IACrC,KAAA,CACN;EAC2C;CAC7C;CACA,OAAO;EAAC;EAAS;CAAS;AAC5B;AAMA,eAAe,oBACb,IACA,KACA,OACA,OACA,cACA,aACA,EAAC,YAAY,kBAAkB,WAC/B,eAC0B;CAC1B,MAAM,SAAS,MAAM,gCACnB,IACA,KACA,OACA,OACA,cACA,aACF;CACA,IAAI,CAAC,QAEH,OAAO;EAAC,SAAS,KAAA;EAAW,WAAW;CAAI;CAE7C,MAAM,EAAC,aAAa,aAAa,SAAS,gBAAe;CAQzD,MAAM,gBADJ,YAAY,OAAO,YAAY,MAAM,YAAY,cAE/C,KAAA,IACA,qBAAqB,OAAO;CAKhC,SAAS,UAAU,oBAAoB,YAAY,cACjD,YAAY,QAAQ,CACtB;CAoBA,MAAM,OAAO,mBAAmB,MAAM,iBAAiB,aAAa,IAAI;CACxE,MAAM,wBAAwB,SAAS;CACvC,MAAM,YAAY,SAAS,gBAAgB,OAAO;CAIlD,MAAM,cAAc,mBAAmB,WAAW,WAAW;CAE7D,IAAI,YAAY,WAAW;EACzB,MAAM,EAAC,WAAW,kBAAiB;EACnC,MAAM,EAAC,MAAM,YAAY,sBAAqB;EAC9C,MAAM,YAAY,IAAI,IAAI,cAAc,IAAI,aAAa,EAAE,SAAS;EACpE,GAAG,OACD,0BAA0B,UAAU,IAAI,KAAK,GAAG,kBAAkB,IAClE;GAAC;GAAa;EAAW,CAC3B;EAUA,OAAO;GACL,SAAS;GACT;GACA;GACA,iBAAA,MAb4B,eAC5B,IACA;IAAC,GAAG;IAAY;GAAS,GACzB,aACA,aACA,WAAW,aACP;IAAC,MAAM;IAAQ,SAAS;GAAa,IACrC,KAAA,CACN;GAME;EACF;CACF;CACA,OAAO;EACL,SAAS;EACT;EACA;EACA;CACF;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8BA,SAAgB,kBACd,kBACA,eACA,eACA,WACoB;CACpB,IAAI,eACF,OAAO;CAET,IAAI,kBAAkB,KAAA,GACpB,OAAO,mBAAmB,gBAAgB,mBAAmB;CAE/D,OAAO,WAAW;AACpB;AAEA,SAAS,mBACP,WACA,EAAC,cACmB;CACpB,OAAO,aAAa;EAAC,gBAAgB;EAAY,cAAc;CAAE;AACnE;AAEA,eAAe,cAAc,KAA+B;CAC1D,MAAM,CAAC,EAAC,SAAQ,MAAM,GAAiB;CACvC,OAAO;AACT;AAEA,IAAM,2BAA2B;AAGjC,IAAM,2BAA2B;AACjC,IAAM,gCAAgC;AACtC,IAAM,2CAA2C;AAGjD,eAAsB,gCACpB,IACA,KACA,OACA,OACA,cACA,eACA,iBAAiB,0BAcjB;CACA,MAAM,gCAAgB,IAAI,IAAoB;CAC9C,IAAI;CAEJ,IAAI;CACJ,MAAM,6BAAa,IAAI,IAAoB;CAC3C,MAAM,aAAa,KAAa,eAAuB;EACrD,MAAM,MAAM,KAAK,IAAI;EACrB,MAAM,OAAO,WAAW,IAAI,GAAG;EAC/B,IAAI,SAAS,KAAA,KAAa,MAAM,OAAO,YACrC,OAAO;EAET,WAAW,IAAI,KAAK,GAAG;EACvB,OAAO;CACT;CAEA,IAAI;EACF,KAAK,IAAI,IAAI,IAAK,KAAK;GACrB,IAAI,IAAI,GACN,MAAM,MAAM,cAAc;GAE5B,MAAM,WAAW,MAAM,qBAAqB,KAAK,OAAO,KAAK;GAC7D,IAAI,UAAU,YAAY,wBAAwB,GAChD,GAAG,OAAO,6BAA6B,SAAS,EAAC,SAAQ,CAAC;GAE5D,IAAI,SAAS,WAAW,GAAG;IACzB,GAAG,OAAO,wCAAwC,EAAC,SAAQ,CAAC;IAC5D,MAAM,WAAW,aAAa,cAAc;IAC5C;GACF;GACA,KAAK,MAAM,WAAW,UAAU;IAE9B,IAAI,QAAQ,QACV,cAAc,OAAO,QAAQ,EAAE;SAC1B,IAAI,CAAC,cAAc,IAAI,QAAQ,EAAE,GACtC,cAAc,IAAI,QAAQ,IAAI,KAAK,IAAI,CAAC;IAK1C,IAAI,QAAQ,UAAA,GAAuB;KAEjC,IACE,UACE,gBAAgB,QAAQ,MACxB,6BACF,GAEA,GAAG,OAAO,+BAA+B,QAAQ,MAAM,EAAC,QAAO,CAAC;KAElE;IACF;IAEA,IAAI,QAAQ,UAAA,GAAqB;KAC/B,MAAM,EAAC,QAAQ,sBAAqB;KACpC,IAAI,QAAQ;MAEV,gBAAgB,MAAM,qBACpB,IACA,KACA,wBACA,OACA,OACA,KAAK,IAAI,EAAE,SAAS,GACpB,cACA;OACE,YAAY;OACZ,UAAU;OAGV,aAAa;MACf,SACM,aACN,CACF;MAQA,YAAY,MAAM,cAAc,GAAG;MACnC,IAAI,SAAS,iBAAiB,IAAI,SAAS,OAAO,GAAG;OACnD,IAAI,UAAU,QAAQ,QAAQ,MAAM,wBAAwB,GAC1D,GAAG,OACD,eAAe,QAAQ,GAAG,GAAG,kBAAkB,YAAY,QAAQ,UAAU,YAAY,KAAK,UAAU,GAAG,YAAY,KAAK,iBAAiB,IAC7I,EAAC,QAAO,CACV;OAEF;MACF;KACF;KAOA,IACE,eACA,WACA,SAAS,iBAAiB,KAAK,SAAS,OAAO,GAC/C;MACA,GAAG,OAAO,mBAAmB,QAAQ,GAAG,GAAG,QAAQ,QAAQ,EACzD,QACF,CAAC;MAQD,OAAO;OACL,aAAa;OACb,aATkB,KAClB,MAAM,mBAAmB,KAAK,OAAO;QACnC,UAAU,QAAQ;QAClB,QAAQ,YAAY,QAAQ;OAC9B,CAAC,GACD,WAAW,YAAY,QAAQ,GAAG,aAIlC;OACA,SAAS,YAAY,KAAK;OAC1B,aAAa,YAAY;MAC3B;KACF;IACF;IAEA,IACE,QAAQ,UAAU,KAClB,QAAQ,UACR,QAAQ,OAAO,aAAa,QAAQ;SAIlC,UAAU,aAAa,QAAQ,MAAM,6BAA6B,GAElE,GAAG,OACD,uBAAuB,QAAQ,GAAG,GAAG,QAAQ,KAAK,kCAClD,EAAC,QAAO,CACV;IAAA;IAIJ,IAAI,QAAQ,cAAc,CAAC,QAAQ,QAAQ;KAQzC,MAAM,MAAM,KAAK,IAAI;KACrB,MAAM,aAAa,OAAO,cAAc,IAAI,QAAQ,EAAE,KAAK;KAC3D,IACE,cAAc,iBACd,UAAU,YAAY,QAAQ,MAAM,wBAAwB,GAE5D,GAAG,OACD,WAAW,QAAQ,GAAG,GAAG,QAAQ,KAAK,yBAAyB,WAAW,KAC1E,EAAC,QAAO,CACV;KAEF,IAAI,cAAc,eAAe;MAC/B,MAAM,WAAW,MAAM,uBACrB,IACA,KACA,OACA,kBACA,OACF;MACA,IAAI,CAAC,UACH,GAAG,OACD,4BAA4B,QAAQ,GAAG,GAAG,QAAQ,MACpD;WACK;OACL,GAAG,OAAO,oBAAoB,QAAQ,GAAG,GAAG,QAAQ,MAAM;OAG1D,MAAM,WAAW,aAAa,cAAc;OAC5C,OAAO;QACL,aAAa;QACb,aAAa,SAAS;QAEtB,SAAS,QAAQ;QACjB,aAAa,SAAS;OACxB;MACF;KACF;IACF;GACF;EACF;CACF,SAAS,GAAG;EACV,MAAM,WAAW,aAAa,cAAc;EAC5C,MAAM;CACR;AACF;AAEA,eAAe,uBACb,IACA,KACA,OACA,EACE,gBACA,cAAc,YACd,sBAEF,WACA;CACA,MAAM,kBAAkB,MAAM,oBAC5B,IACA,KACA,OACA,gBACA,WACA,kBACF;CACA,IAAI,CAAC,iBACH,MAAM,IAAI,gBACR,8CAA8C,eAAe,UAAU,UAAU,EACnF;CAIF,MAAM,YAAY,MAAM,aAAa,SAAS;CAC9C,MAAM,aAAa,gBAAgB,aAChC,QAAO,MAAK,CAAC,EAAE,WAAW,0BAA0B,KAAK,CAAC,CAAC,EAC3D,KAAK;CACR,IAAI,CAAC,UAAU,WAAW,UAAU,GAAG;EACrC,GAAG,OAAO,8CAA8C,UAAU,EAAE;EACpE,MAAM,IAAI,OAAO,UAAU,MAAM,OAAO,MAAM,QAAQ,CAAC;EACvD,MAAM,IAAI,gBACR,2BAA2B,UAAU,2CACjB,WAAW,EACjC;CACF;CAKA,IAAI,CAAC,UAAU,gBAAgB,cAAc,UAAU,GACrD,MAAM,IAAI,gBACR,0BAA0B,gBAAgB,aAAa,0CACnB,WAAW,EACjD;CAIF,MAAM,SAAS,MAAM,GAAG;0DACgC,IAAI,UAAU,EAAE;IACtE,OAAO;CACT,IAAI,OAAO,WAAW,WAAW,QAC/B,MAAM,IAAI,gBACR,0BAA0B,OAAO,KAAK,EAAE,gDACN,WAAW,EAC/C;CAGF,MAAM,EAAC,SAAQ;CACf,MAAM,SAAS,MAAM,GACb;;0BAEgB;CACxB,IAAI,OAAO,WAAW,GACpB,MAAM,IAAI,gBAAgB,oBAAoB,KAAK,YAAY;CAEjE,MAAM,CAAC,EAAC,YAAY,eAAc;CAClC,IAAI,eAAe,QAAQ,cAAc,QACvC,MAAM,IAAI,gBACR,oBAAoB,KAAK,+DAC3B;CAEF,MAAM,CAAC,EAAC,eAAc,MAAM,GAAmC;;CAE/D,OAAO;EAAC;EAAiB;CAAS;AACpC"}

package/out/zero-cache/src/services/change-streamer/change-streamer-http.js

@@ -13,7 +13,7 @@ import { discoverChangeStreamerAddress } from "./schema/tables.js";
 import "./change-streamer.js";
 import "./snapshot-message.js";
 import { subscribeDownstreamSchema, subscribeUpstreamSchema } from "./subscribe.js";
-import WebSocket$1 from "ws";
+import WebSocket from "ws";
 import websocket from "@fastify/websocket";
 //#region ../zero-cache/src/services/change-streamer/change-streamer-http.ts
 var MIN_SUPPORTED_PROTOCOL_VERSION = 4;
@@ -210,7 +210,7 @@ var ChangeStreamerHttpClient = class {
 	* replication-manager is the one that serves the subscription.
 	*/
 	async connect(taskID) {
-		const ws = new WebSocket$1(await this.#resolveChangeStreamer(SUBSCRIBE_PATH) + `?${new URLSearchParams({ taskID }).toString()}`, { handshakeTimeout: CONNECT_TIMEOUT_MS });
+		const ws = new WebSocket(await this.#resolveChangeStreamer(SUBSCRIBE_PATH) + `?${new URLSearchParams({ taskID }).toString()}`, { handshakeTimeout: CONNECT_TIMEOUT_MS });
 		const outbound = Subscription.create();
 		return {
 			instream: await streamInternalWithSize(this.#lc, ws, subscribeDownstreamSchema, outbound),

package/out/zero-cache/src/services/change-streamer/change-streamer-http.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-http.js","names":["#lc","#opts","#changeStreamer","#changes","#reserveSnapshot","#subscribe","#receiveWebsocket","#ensureChangeStreamerStarted","#forwardReservation","#changeStreamerStarted","#shardID","#changeDB","#changeStreamerURI","#resolveChangeStreamer"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport websocket from '@fastify/websocket';\nimport type {LogContext} from '@rocicorp/logger';\nimport WebSocket from 'ws';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport type {IncomingMessageSubset} from '../../types/http.ts';\nimport {pgClient, type PostgresDB} from '../../types/pg.ts';\nimport {type Worker} from '../../types/processes.ts';\nimport {type ShardID} from '../../types/shards.ts';\nimport {\n  readHead,\n  streamInternal,\n  streamInternalStringified,\n  streamInternalWithSize,\n  type Sized,\n  type Source,\n} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {URLParams} from '../../types/url-params.ts';\nimport {installWebSocketReceiver} from '../../types/websocket-handoff.ts';\nimport {closeWithError, PROTOCOL_ERROR} from '../../types/ws.ts';\nimport {HttpService, type Options as HttpOptions} from '../http-service.ts';\nimport {handleProfzRequest} from '../profz.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport {\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type ChangeStreamerService,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport {discoverChangeStreamerAddress} from './schema/tables.ts';\nimport {type SnapshotMessage} from './snapshot-message.ts';\nimport {\n  subscribeDownstreamSchema,\n  subscribeUpstreamSchema,\n  type ReservedMessage,\n  type SnapshotStatus,\n  type SubscribeContext,\n  type SubscribeDownstream,\n  type SubscribeUpstream,\n} from './subscribe.ts';\n\nconst MIN_SUPPORTED_PROTOCOL_VERSION = 4;\n\n// The merged `/subscribe` protocol was introduced at v7.\nconst MIN_SUBSCRIBE_PROTOCOL_VERSION = 7;\n\nconst SNAPSHOT_PATH_PATTERN = '/replication/:version/snapshot';\nconst CHANGES_PATH_PATTERN = '/replication/:version/changes';\nconst SUBSCRIBE_PATH_PATTERN = '/replication/:version/subscribe';\nconst PATH_REGEX =\n  /\\/replication\\/v(?<version>\\d+)\\/(changes|snapshot|subscribe)$/;\n\nconst SUBSCRIBE_PATH = `/replication/v${PROTOCOL_VERSION}/subscribe`;\n\n// Max time for a client to establish the /subscribe websocket (TCP connect\n// plus upgrade) before abandoning the attempt.\nconst CONNECT_TIMEOUT_MS = 10_000;\n\ntype Options = HttpOptions & {\n  startupDelayMs: number;\n  config?:\n    | Pick<NormalizedZeroConfig, 'adminPassword' | 'operatorPassword'>\n    | undefined;\n  getProfileWorker?: (() => Promise<Worker>) | undefined;\n};\n\nexport class ChangeStreamerHttpServer extends HttpService {\n  readonly id = 'change-streamer-http-server';\n  readonly #lc: LogContext;\n  readonly #opts: Options;\n  readonly #changeStreamer: ChangeStreamerService;\n\n  constructor(\n    lc: LogContext,\n    opts: Options,\n    parent: Worker,\n    changeStreamer: ChangeStreamerService,\n  ) {\n    super('change-streamer-http-server', lc, opts, async fastify => {\n      await fastify.register(websocket);\n\n      fastify.get(CHANGES_PATH_PATTERN, {websocket: true}, this.#changes);\n      fastify.get(\n        SNAPSHOT_PATH_PATTERN,\n        {websocket: true},\n        this.#reserveSnapshot,\n      );\n      fastify.get(SUBSCRIBE_PATH_PATTERN, {websocket: true}, this.#subscribe);\n\n      fastify.get('/profz', (req, res) =>\n        handleProfzRequest(\n          lc,\n          opts.config ?? {\n            adminPassword: undefined,\n            operatorPassword: undefined,\n          },\n          req,\n          res,\n          opts.getProfileWorker,\n          undefined,\n          'change-streamer',\n        ),\n      );\n\n      installWebSocketReceiver<'snapshot' | 'changes' | 'subscribe'>(\n        lc,\n        fastify.websocketServer,\n        this.#receiveWebsocket,\n        parent,\n      );\n    });\n\n    this.#lc = lc;\n    this.#opts = opts;\n    this.#changeStreamer = changeStreamer;\n  }\n\n  // Called when receiving a web socket via the main dispatcher handoff.\n  readonly #receiveWebsocket = (\n    ws: WebSocket,\n    action: 'changes' | 'snapshot' | 'subscribe',\n    msg: IncomingMessageSubset,\n  ) => {\n    switch (action) {\n      case 'snapshot':\n        return this.#reserveSnapshot(ws, msg);\n      case 'changes':\n        return this.#changes(ws, msg);\n      case 'subscribe':\n        return this.#subscribe(ws, msg);\n      default:\n        closeWithError(\n          this._lc,\n          ws,\n          `invalid action \"${action}\" received in handoff`,\n        );\n        return;\n    }\n  };\n\n  readonly #reserveSnapshot = async (ws: WebSocket, req: RequestHeaders) => {\n    this.#ensureChangeStreamerStarted('incoming snapshot reservation');\n    try {\n      const url = new URL(\n        req.url ?? '',\n        req.headers.origin ?? 'http://localhost',\n      );\n      checkProtocolVersion(url.pathname);\n      const taskID = url.searchParams.get('taskID');\n      if (!taskID) {\n        throw new Error('Missing taskID in snapshot request');\n      }\n      const downstream =\n        await this.#changeStreamer.startSnapshotReservation(taskID);\n      // Send-only for now: the reservation carries no inbound application\n      // messages (only transport acks, demuxed internally).\n      void streamInternal(this._lc, ws, undefined, downstream);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  readonly #changes = async (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const ctx = getSubscriberContext(req);\n      if (ctx.mode === 'serving') {\n        this.#ensureChangeStreamerStarted('incoming subscription');\n      }\n\n      const downstream = await this.#changeStreamer.subscribe(ctx);\n      // Send-only for now: the subscription carries no inbound application\n      // messages (only transport acks, demuxed internally).\n      void streamInternalStringified(this._lc, ws, undefined, downstream, {\n        batched: ctx.wsBatched,\n      });\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  /**\n   * The merged v7 `/subscribe` handler. A single bidirectional connection\n   * carries, in sequence: an optional reservation phase (`reserve-snapshot` →\n   * `reserved`) and then the subscription phase (`start-subscription` → change\n   * stream). This ensures that the subscriber subscribes to the same task from\n   * which it restored the backup.\n   */\n  readonly #subscribe = async (ws: WebSocket, req: RequestHeaders) => {\n    let reservation: Source<SnapshotMessage> | undefined;\n    const outbound = Subscription.create<string | PreSerializedBatch>();\n\n    try {\n      const url = new URL(\n        req.url ?? '',\n        req.headers.origin ?? 'http://localhost',\n      );\n      const protocolVersion = checkProtocolVersion(url.pathname);\n      if (protocolVersion < MIN_SUBSCRIBE_PROTOCOL_VERSION) {\n        throw new Error(\n          `the /subscribe endpoint requires protocol v${MIN_SUBSCRIBE_PROTOCOL_VERSION} ` +\n            `(client is at v${protocolVersion})`,\n        );\n      }\n\n      const instream = await streamInternalStringified(\n        this._lc,\n        ws,\n        subscribeUpstreamSchema,\n        outbound,\n        {batched: true},\n      );\n\n      // Keep reading upstream control messages for the life of the connection.\n      // Do not manually exit loop, as that would close the socket. The loop\n      // exits naturally when the connection closes.\n      for await (const msg of instream) {\n        if (msg[0] === 'reserve-snapshot') {\n          const {taskID} = msg[1];\n          this.#ensureChangeStreamerStarted('incoming snapshot reservation');\n          reservation =\n            await this.#changeStreamer.startSnapshotReservation(taskID);\n          void this.#forwardReservation(reservation, outbound);\n        } else if (msg[0] === 'start-subscription') {\n          const ctx: SubscriberContext = {\n            ...msg[1],\n            protocolVersion,\n            wsBatched: true,\n          };\n          if (ctx.mode === 'serving') {\n            this.#ensureChangeStreamerStarted('incoming subscription');\n          }\n          // subscribe() registers the subscriber and then releases any\n          // reservation for that task.\n          reservation = undefined;\n          await this.#changeStreamer.subscribe(ctx, outbound);\n        }\n      }\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    } finally {\n      // Covers a reservation-phase disconnect (instream ended before a\n      // subscription started); idempotent with the close handler above.\n      reservation?.cancel();\n    }\n  };\n\n  /**\n   * Forwards a snapshot reservation's single `['status', ...]` message onto\n   * the merged connection's `outbound` sink as a `['reserved', ...]` message.\n   * The loop then blocks on the held-open reservation until it is cancelled,\n   * by `subscribe()` at the subscription handoff, or by the reservation-phase\n   * teardown on disconnect.\n   */\n  async #forwardReservation(\n    reservation: Source<SnapshotMessage>,\n    outbound: Subscription<string | PreSerializedBatch>,\n  ) {\n    try {\n      for await (const [, status] of reservation) {\n        // Translate the /snapshot protocol's ['status', {tag: 'status', ...}]\n        // message to the /subscribe protocol's ['reserved', {tag: 'snapshot', ...}].\n        outbound.push(\n          BigIntJSON.stringify([\n            'reserved',\n            {...status, tag: 'snapshot'},\n          ] satisfies ReservedMessage),\n        );\n      }\n    } catch (e) {\n      this._lc.warn?.(`error forwarding snapshot reservation`, e);\n      outbound.fail(e instanceof Error ? e : new Error(String(e)));\n    }\n  }\n\n  #changeStreamerStarted = false;\n\n  #ensureChangeStreamerStarted(reason?: string) {\n    if (!this.#changeStreamerStarted && this._state.shouldRun()) {\n      this.#lc.info?.(\n        `starting ChangeStreamerService ${reason ? `(${reason})` : ''}`,\n      );\n      void this.#changeStreamer\n        .run()\n        .catch(e =>\n          this.#lc.warn?.(`ChangeStreamerService ended with error`, e),\n        )\n        .finally(() => this.stop());\n\n      this.#changeStreamerStarted = true;\n    }\n  }\n\n  protected override _onStart(): void {\n    const {startupDelayMs, readinessGate = promiseVoid} = this.#opts;\n    if (startupDelayMs === 0) {\n      // In RMv2, there is no need to delay starting the change streamer\n      // because RM startup is non-disruptive.\n      this.#ensureChangeStreamerStarted();\n    } else {\n      // In RMv1, starting the change-streamer forcibly shuts down the\n      // previous change-streamer, causing view-syncers to reconnect.\n      // If this replication-manager has just started, the routing layer may\n      // not have registered it with DNS, as that only happens after it\n      // confirms health checks. To minimize downtime, the takeover is\n      // delayed for the configured startupDelayMs _after_ beginning to\n      // advertise readiness.\n      void readinessGate.then(() => {\n        this.#lc.info?.(\n          `waiting ${startupDelayMs}ms before taking over the change log`,\n        );\n        this._state.setTimeout(\n          () =>\n            this.#ensureChangeStreamerStarted(\n              `startup delay elapsed (${startupDelayMs} ms)`,\n            ),\n          startupDelayMs,\n        );\n      });\n    }\n  }\n\n  protected override async _onStop(): Promise<void> {\n    if (this.#changeStreamerStarted) {\n      await this.#changeStreamer.stop();\n    }\n  }\n}\n\nexport interface SnapshotReserver {\n  reserveSnapshot(taskID: string): Promise<{\n    reserved: SnapshotStatus;\n    followup: ReservationFollowup;\n  }>;\n}\n\nexport interface SubscriptionStarter {\n  subscribe(\n    ctx: SubscriberContext,\n  ): Promise<Source<Sized<SubscribeDownstream>>>;\n}\n\nexport interface ReservationFollowup extends SubscriptionStarter {\n  /**\n   * A signal to indicate that the followup is no longer valid (i.e. the\n   * connection to the reserving task has been severed).\n   */\n  readonly signal: AbortSignal;\n\n  /**\n   * Abandons the reservation without subscribing, closing the connection that\n   * was held open to pin the change-log floor. Used when the caller cannot\n   * proceed to `subscribe()` (e.g. process shutdown while a restore is in\n   * progress) and must not leave the reservation's connection open forever.\n   */\n  cancel(reason?: Error): void;\n}\n\nexport class ChangeStreamerHttpClient\n  implements ChangeStreamer, SnapshotReserver, SubscriptionStarter\n{\n  readonly #lc: LogContext;\n  readonly #shardID: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #changeStreamerURI: string | undefined;\n\n  constructor(\n    lc: LogContext,\n    shardID: ShardID,\n    changeDB: string,\n    changeStreamerURI: string | undefined,\n  ) {\n    this.#lc = lc;\n    this.#shardID = shardID;\n    // Create a pg client with a single short-lived connection for the purpose\n    // of change-streamer discovery (i.e. ChangeDB as DNS).\n    this.#changeDB = pgClient(lc, changeDB, 'change-streamer-discovery', {\n      max: 1,\n      ['idle_timeout']: 15,\n    });\n    this.#changeStreamerURI = changeStreamerURI;\n  }\n\n  async #resolveChangeStreamer(path: string) {\n    let baseURL = this.#changeStreamerURI;\n    if (!baseURL) {\n      const address = await discoverChangeStreamerAddress(\n        this.#shardID,\n        this.#changeDB,\n      );\n      if (!address) {\n        throw new Error(`no change-streamer is running`);\n      }\n      baseURL = address.includes('://') ? `${address}/` : `ws://${address}/`;\n    }\n    const uri = new URL(path, baseURL);\n    this.#lc.info?.(`connecting to change-streamer@${uri}`);\n    return uri;\n  }\n\n  async reserveSnapshot(taskID: string): Promise<{\n    reserved: SnapshotStatus;\n    followup: ReservationFollowup;\n  }> {\n    const {instream, outbound} = await this.connect(taskID);\n    outbound.push(['reserve-snapshot', {taskID}]);\n\n    const {head, rest} = readHead(instream);\n    const item = await head.next();\n    if (!item.value) {\n      throw new Error('stream closed without snapshot reservation');\n    }\n    const {data: msg} = item.value;\n    if (msg[0] !== 'reserved') {\n      throw new Error(`expected \"reserved\" message but got ${msg[0]}`);\n    }\n    const reserved: SnapshotStatus = msg[1];\n    const followup: ReservationFollowup = {\n      subscribe: ctx => {\n        outbound.push(['start-subscription', toStartSubscriptionContext(ctx)]);\n        return Promise.resolve(rest);\n      },\n      signal: outbound.signal,\n      cancel: reason => outbound.cancel(reason),\n    };\n\n    return {reserved, followup};\n  }\n\n  async subscribe(\n    ctx: SubscriberContext,\n  ): Promise<Source<Sized<SubscribeDownstream>>> {\n    const {instream, outbound} = await this.connect(ctx.taskID);\n    outbound.push(['start-subscription', toStartSubscriptionContext(ctx)]);\n    return instream;\n  }\n\n  /**\n   * Opens a merged v7 `/subscribe` connection (see subscribe.ts). Returns the\n   * inbound change/reservation stream and an `outbound` sink into which the\n   * caller pushes the `reserve-snapshot` / `start-subscription` control\n   * messages that drive the connection's phases. The single connection spans\n   * the reservation, backup restore, and subscription, so the reserving\n   * replication-manager is the one that serves the subscription.\n   */\n  // exported for testing\n  async connect(taskID: string): Promise<{\n    instream: Source<Sized<SubscribeDownstream>>;\n    outbound: Subscription<SubscribeUpstream>;\n  }> {\n    const uri = await this.#resolveChangeStreamer(SUBSCRIBE_PATH);\n\n    // taskID is carried in the query for observability/routing; the server\n    // reads the authoritative taskID from the control messages themselves.\n    const params = new URLSearchParams({taskID});\n    // Bound the TCP connect + HTTP upgrade. Without this, a SYN black-holed\n    // by a stale Service endpoint (e.g. a just-terminated replication-manager\n    // pod) is retransmitted by the kernel for ~2 minutes before ETIMEDOUT.\n    // Failing fast lets the caller's retry open a fresh connection that is\n    // routed to a live endpoint.\n    const ws = new WebSocket(uri + `?${params.toString()}`, {\n      handshakeTimeout: CONNECT_TIMEOUT_MS,\n    });\n\n    const outbound = Subscription.create<SubscribeUpstream>();\n    const instream = await streamInternalWithSize(\n      this.#lc,\n      ws,\n      subscribeDownstreamSchema,\n      outbound,\n    );\n    return {instream, outbound};\n  }\n}\n\ntype RequestHeaders = Pick<IncomingMessage, 'url' | 'headers'>;\n\nexport function getSubscriberContext(req: RequestHeaders): SubscriberContext {\n  const url = new URL(req.url ?? '', req.headers.origin ?? 'http://localhost');\n  const protocolVersion = checkProtocolVersion(url.pathname);\n  const params = new URLParams(url);\n\n  return {\n    protocolVersion,\n    id: params.get('id', true),\n    taskID: params.get('taskID', true),\n    mode: params.get('mode', false) === 'backup' ? 'backup' : 'serving',\n    replicaVersion: params.get('replicaVersion', true),\n    watermark: params.get('watermark', true),\n    // Absent for subscribers that predate the parameter, which is the safe\n    // default: the barrier falls back to polling rather than waiting on an\n    // ACK that would never be attributed to a writer.\n    wsBatched: params.getBoolean('wsBatched'),\n  };\n}\n\nfunction checkProtocolVersion(pathname: string): number {\n  const match = PATH_REGEX.exec(pathname);\n  if (!match) {\n    throw new Error(`invalid path: ${pathname}`);\n  }\n  const v = Number(match.groups?.version);\n  if (\n    Number.isNaN(v) ||\n    v > PROTOCOL_VERSION ||\n    v < MIN_SUPPORTED_PROTOCOL_VERSION\n  ) {\n    throw new Error(\n      `Cannot service client at protocol v${v}. ` +\n        `Supported protocols: [v${MIN_SUPPORTED_PROTOCOL_VERSION} ... v${PROTOCOL_VERSION}]`,\n    );\n  }\n  return v;\n}\n\n// Projects a client-side SubscriberContext onto the start-subscription\n// message payload, dropping the fields that are implicit on the merged\n// connection: protocolVersion (from the request path), wsBatched (always on).\nfunction toStartSubscriptionContext(ctx: SubscriberContext): SubscribeContext {\n  const {taskID, id, mode, replicaVersion, watermark, backfills} = ctx;\n  return {taskID, id, mode, replicaVersion, watermark, backfills};\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AA4CA,IAAM,iCAAiC;AAGvC,IAAM,iCAAiC;AAEvC,IAAM,wBAAwB;AAC9B,IAAM,uBAAuB;AAC7B,IAAM,yBAAyB;AAC/B,IAAM,aACJ;AAEF,IAAM,iBAAiB;AAIvB,IAAM,qBAAqB;AAU3B,IAAa,2BAAb,cAA8C,YAAY;CACxD,KAAc;CACd;CACA;CACA;CAEA,YACE,IACA,MACA,QACA,gBACA;EACA,MAAM,+BAA+B,IAAI,MAAM,OAAM,YAAW;GAC9D,MAAM,QAAQ,SAAS,SAAS;GAEhC,QAAQ,IAAI,sBAAsB,EAAC,WAAW,KAAI,GAAG,KAAKG,QAAQ;GAClE,QAAQ,IACN,uBACA,EAAC,WAAW,KAAI,GAChB,KAAKC,gBACP;GACA,QAAQ,IAAI,wBAAwB,EAAC,WAAW,KAAI,GAAG,KAAKC,UAAU;GAEtE,QAAQ,IAAI,WAAW,KAAK,QAC1B,mBACE,IACA,KAAK,UAAU;IACb,eAAe,KAAA;IACf,kBAAkB,KAAA;GACpB,GACA,KACA,KACA,KAAK,kBACL,KAAA,GACA,iBACF,CACF;GAEA,yBACE,IACA,QAAQ,iBACR,KAAKC,mBACL,MACF;EACF,CAAC;EAED,KAAKN,MAAM;EACX,KAAKC,QAAQ;EACb,KAAKC,kBAAkB;CACzB;CAGA,qBACE,IACA,QACA,QACG;EACH,QAAQ,QAAR;GACE,KAAK,YACH,OAAO,KAAKE,iBAAiB,IAAI,GAAG;GACtC,KAAK,WACH,OAAO,KAAKD,SAAS,IAAI,GAAG;GAC9B,KAAK,aACH,OAAO,KAAKE,WAAW,IAAI,GAAG;GAChC;IACE,eACE,KAAK,KACL,IACA,mBAAmB,OAAO,sBAC5B;IACA;EACJ;CACF;CAEA,mBAA4B,OAAO,IAAe,QAAwB;EACxE,KAAKE,6BAA6B,+BAA+B;EACjE,IAAI;GACF,MAAM,MAAM,IAAI,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,kBACxB;GACA,qBAAqB,IAAI,QAAQ;GACjC,MAAM,SAAS,IAAI,aAAa,IAAI,QAAQ;GAC5C,IAAI,CAAC,QACH,MAAM,IAAI,MAAM,oCAAoC;GAEtD,MAAM,aACJ,MAAM,KAAKL,gBAAgB,yBAAyB,MAAM;GAG5D,eAAoB,KAAK,KAAK,IAAI,KAAA,GAAW,UAAU;EACzD,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;CAEA,WAAoB,OAAO,IAAe,QAAwB;EAChE,IAAI;GACF,MAAM,MAAM,qBAAqB,GAAG;GACpC,IAAI,IAAI,SAAS,WACf,KAAKK,6BAA6B,uBAAuB;GAG3D,MAAM,aAAa,MAAM,KAAKL,gBAAgB,UAAU,GAAG;GAG3D,0BAA+B,KAAK,KAAK,IAAI,KAAA,GAAW,YAAY,EAClE,SAAS,IAAI,UACf,CAAC;EACH,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;;;;;;;;CASA,aAAsB,OAAO,IAAe,QAAwB;EAClE,IAAI;EACJ,MAAM,WAAW,aAAa,OAAoC;EAElE,IAAI;GAKF,MAAM,kBAAkB,qBAAqB,IAJ7B,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,kBAEqB,EAAI,QAAQ;GACzD,IAAI,kBAAkB,gCACpB,MAAM,IAAI,MACR,8CAA8C,+BAA+B,kBACzD,gBAAgB,EACtC;GAGF,MAAM,WAAW,MAAM,0BACrB,KAAK,KACL,IACA,yBACA,UACA,EAAC,SAAS,KAAI,CAChB;GAKA,WAAW,MAAM,OAAO,UACtB,IAAI,IAAI,OAAO,oBAAoB;IACjC,MAAM,EAAC,WAAU,IAAI;IACrB,KAAKK,6BAA6B,+BAA+B;IACjE,cACE,MAAM,KAAKL,gBAAgB,yBAAyB,MAAM;IAC5D,KAAUM,oBAAoB,aAAa,QAAQ;GACrD,OAAO,IAAI,IAAI,OAAO,sBAAsB;IAC1C,MAAM,MAAyB;KAC7B,GAAG,IAAI;KACP;KACA,WAAW;IACb;IACA,IAAI,IAAI,SAAS,WACf,KAAKD,6BAA6B,uBAAuB;IAI3D,cAAc,KAAA;IACd,MAAM,KAAKL,gBAAgB,UAAU,KAAK,QAAQ;GACpD;EAEJ,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD,UAAU;GAGR,aAAa,OAAO;EACtB;CACF;;;;;;;;CASA,MAAMM,oBACJ,aACA,UACA;EACA,IAAI;GACF,WAAW,MAAM,GAAG,WAAW,aAG7B,SAAS,KACP,WAAW,UAAU,CACnB,YACA;IAAC,GAAG;IAAQ,KAAK;GAAU,CAC7B,CAA2B,CAC7B;EAEJ,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,yCAAyC,CAAC;GAC1D,SAAS,KAAK,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,CAAC,CAAC,CAAC;EAC7D;CACF;CAEA,yBAAyB;CAEzB,6BAA6B,QAAiB;EAC5C,IAAI,CAAC,KAAKC,0BAA0B,KAAK,OAAO,UAAU,GAAG;GAC3D,KAAKT,IAAI,OACP,kCAAkC,SAAS,IAAI,OAAO,KAAK,IAC7D;GACA,KAAUE,gBACP,IAAI,EACJ,OAAM,MACL,KAAKF,IAAI,OAAO,0CAA0C,CAAC,CAC7D,EACC,cAAc,KAAK,KAAK,CAAC;GAE5B,KAAKS,yBAAyB;EAChC;CACF;CAEA,WAAoC;EAClC,MAAM,EAAC,gBAAgB,gBAAgB,gBAAe,KAAKR;EAC3D,IAAI,mBAAmB,GAGrB,KAAKM,6BAA6B;OASlC,cAAmB,WAAW;GAC5B,KAAKP,IAAI,OACP,WAAW,eAAe,qCAC5B;GACA,KAAK,OAAO,iBAER,KAAKO,6BACH,0BAA0B,eAAe,KAC3C,GACF,cACF;EACF,CAAC;CAEL;CAEA,MAAyB,UAAyB;EAChD,IAAI,KAAKE,wBACP,MAAM,KAAKP,gBAAgB,KAAK;CAEpC;AACF;AA+BA,IAAa,2BAAb,MAEA;CACE;CACA;CACA;CACA;CAEA,YACE,IACA,SACA,UACA,mBACA;EACA,KAAKF,MAAM;EACX,KAAKU,WAAW;EAGhB,KAAKC,YAAY,SAAS,IAAI,UAAU,6BAA6B;GACnE,KAAK;IACJ,iBAAiB;EACpB,CAAC;EACD,KAAKC,qBAAqB;CAC5B;CAEA,MAAMC,uBAAuB,MAAc;EACzC,IAAI,UAAU,KAAKD;EACnB,IAAI,CAAC,SAAS;GACZ,MAAM,UAAU,MAAM,8BACpB,KAAKF,UACL,KAAKC,SACP;GACA,IAAI,CAAC,SACH,MAAM,IAAI,MAAM,+BAA+B;GAEjD,UAAU,QAAQ,SAAS,KAAK,IAAI,GAAG,QAAQ,KAAK,QAAQ,QAAQ;EACtE;EACA,MAAM,MAAM,IAAI,IAAI,MAAM,OAAO;EACjC,KAAKX,IAAI,OAAO,iCAAiC,KAAK;EACtD,OAAO;CACT;CAEA,MAAM,gBAAgB,QAGnB;EACD,MAAM,EAAC,UAAU,aAAY,MAAM,KAAK,QAAQ,MAAM;EACtD,SAAS,KAAK,CAAC,oBAAoB,EAAC,OAAM,CAAC,CAAC;EAE5C,MAAM,EAAC,MAAM,SAAQ,SAAS,QAAQ;EACtC,MAAM,OAAO,MAAM,KAAK,KAAK;EAC7B,IAAI,CAAC,KAAK,OACR,MAAM,IAAI,MAAM,4CAA4C;EAE9D,MAAM,EAAC,MAAM,QAAO,KAAK;EACzB,IAAI,IAAI,OAAO,YACb,MAAM,IAAI,MAAM,uCAAuC,IAAI,IAAI;EAYjE,OAAO;GAAC,UAVyB,IAAI;GAUnB,UAAA;IARhB,YAAW,QAAO;KAChB,SAAS,KAAK,CAAC,sBAAsB,2BAA2B,GAAG,CAAC,CAAC;KACrE,OAAO,QAAQ,QAAQ,IAAI;IAC7B;IACA,QAAQ,SAAS;IACjB,SAAQ,WAAU,SAAS,OAAO,MAAM;GAGxB;EAAQ;CAC5B;CAEA,MAAM,UACJ,KAC6C;EAC7C,MAAM,EAAC,UAAU,aAAY,MAAM,KAAK,QAAQ,IAAI,MAAM;EAC1D,SAAS,KAAK,CAAC,sBAAsB,2BAA2B,GAAG,CAAC,CAAC;EACrE,OAAO;CACT;;;;;;;;;CAWA,MAAM,QAAQ,QAGX;EAWD,MAAM,KAAK,IAAI,YAAU,MAVP,KAAKa,uBAAuB,cAAc,IAU7B,IAAI,IANhB,gBAAgB,EAAC,OAAM,CAMP,EAAO,SAAS,KAAK,EACtD,kBAAkB,mBACpB,CAAC;EAED,MAAM,WAAW,aAAa,OAA0B;EAOxD,OAAO;GAAC,UAAA,MANe,uBACrB,KAAKb,KACL,IACA,2BACA,QACF;GACkB;EAAQ;CAC5B;AACF;AAIA,SAAgB,qBAAqB,KAAwC;CAC3E,MAAM,MAAM,IAAI,IAAI,IAAI,OAAO,IAAI,IAAI,QAAQ,UAAU,kBAAkB;CAC3E,MAAM,kBAAkB,qBAAqB,IAAI,QAAQ;CACzD,MAAM,SAAS,IAAI,UAAU,GAAG;CAEhC,OAAO;EACL;EACA,IAAI,OAAO,IAAI,MAAM,IAAI;EACzB,QAAQ,OAAO,IAAI,UAAU,IAAI;EACjC,MAAM,OAAO,IAAI,QAAQ,KAAK,MAAM,WAAW,WAAW;EAC1D,gBAAgB,OAAO,IAAI,kBAAkB,IAAI;EACjD,WAAW,OAAO,IAAI,aAAa,IAAI;EAIvC,WAAW,OAAO,WAAW,WAAW;CAC1C;AACF;AAEA,SAAS,qBAAqB,UAA0B;CACtD,MAAM,QAAQ,WAAW,KAAK,QAAQ;CACtC,IAAI,CAAC,OACH,MAAM,IAAI,MAAM,iBAAiB,UAAU;CAE7C,MAAM,IAAI,OAAO,MAAM,QAAQ,OAAO;CACtC,IACE,OAAO,MAAM,CAAC,KACd,IAAA,KACA,IAAI,gCAEJ,MAAM,IAAI,MACR,sCAAsC,EAAE,2BACZ,+BAA+B,SAC7D;CAEF,OAAO;AACT;AAKA,SAAS,2BAA2B,KAA0C;CAC5E,MAAM,EAAC,QAAQ,IAAI,MAAM,gBAAgB,WAAW,cAAa;CACjE,OAAO;EAAC;EAAQ;EAAI;EAAM;EAAgB;EAAW;CAAS;AAChE"}
+{"version":3,"file":"change-streamer-http.js","names":["#lc","#opts","#changeStreamer","#changes","#reserveSnapshot","#subscribe","#receiveWebsocket","#ensureChangeStreamerStarted","#forwardReservation","#changeStreamerStarted","#shardID","#changeDB","#changeStreamerURI","#resolveChangeStreamer"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport websocket from '@fastify/websocket';\nimport type {LogContext} from '@rocicorp/logger';\nimport WebSocket from 'ws';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport type {IncomingMessageSubset} from '../../types/http.ts';\nimport {pgClient, type PostgresDB} from '../../types/pg.ts';\nimport {type Worker} from '../../types/processes.ts';\nimport {type ShardID} from '../../types/shards.ts';\nimport {\n  readHead,\n  streamInternal,\n  streamInternalStringified,\n  streamInternalWithSize,\n  type Sized,\n  type Source,\n} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {URLParams} from '../../types/url-params.ts';\nimport {installWebSocketReceiver} from '../../types/websocket-handoff.ts';\nimport {closeWithError, PROTOCOL_ERROR} from '../../types/ws.ts';\nimport {HttpService, type Options as HttpOptions} from '../http-service.ts';\nimport {handleProfzRequest} from '../profz.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport {\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type ChangeStreamerService,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport {discoverChangeStreamerAddress} from './schema/tables.ts';\nimport {type SnapshotMessage} from './snapshot-message.ts';\nimport {\n  subscribeDownstreamSchema,\n  subscribeUpstreamSchema,\n  type ReservedMessage,\n  type SnapshotStatus,\n  type SubscribeContext,\n  type SubscribeDownstream,\n  type SubscribeUpstream,\n} from './subscribe.ts';\n\nconst MIN_SUPPORTED_PROTOCOL_VERSION = 4;\n\n// The merged `/subscribe` protocol was introduced at v7.\nconst MIN_SUBSCRIBE_PROTOCOL_VERSION = 7;\n\nconst SNAPSHOT_PATH_PATTERN = '/replication/:version/snapshot';\nconst CHANGES_PATH_PATTERN = '/replication/:version/changes';\nconst SUBSCRIBE_PATH_PATTERN = '/replication/:version/subscribe';\nconst PATH_REGEX =\n  /\\/replication\\/v(?<version>\\d+)\\/(changes|snapshot|subscribe)$/;\n\nconst SUBSCRIBE_PATH = `/replication/v${PROTOCOL_VERSION}/subscribe`;\n\n// Max time for a client to establish the /subscribe websocket (TCP connect\n// plus upgrade) before abandoning the attempt.\nconst CONNECT_TIMEOUT_MS = 10_000;\n\ntype Options = HttpOptions & {\n  startupDelayMs: number;\n  config?:\n    | Pick<NormalizedZeroConfig, 'adminPassword' | 'operatorPassword'>\n    | undefined;\n  getProfileWorker?: (() => Promise<Worker>) | undefined;\n};\n\nexport class ChangeStreamerHttpServer extends HttpService {\n  readonly id = 'change-streamer-http-server';\n  readonly #lc: LogContext;\n  readonly #opts: Options;\n  readonly #changeStreamer: ChangeStreamerService;\n\n  constructor(\n    lc: LogContext,\n    opts: Options,\n    parent: Worker,\n    changeStreamer: ChangeStreamerService,\n  ) {\n    super('change-streamer-http-server', lc, opts, async fastify => {\n      await fastify.register(websocket);\n\n      fastify.get(CHANGES_PATH_PATTERN, {websocket: true}, this.#changes);\n      fastify.get(\n        SNAPSHOT_PATH_PATTERN,\n        {websocket: true},\n        this.#reserveSnapshot,\n      );\n      fastify.get(SUBSCRIBE_PATH_PATTERN, {websocket: true}, this.#subscribe);\n\n      fastify.get('/profz', (req, res) =>\n        handleProfzRequest(\n          lc,\n          opts.config ?? {\n            adminPassword: undefined,\n            operatorPassword: undefined,\n          },\n          req,\n          res,\n          opts.getProfileWorker,\n          undefined,\n          'change-streamer',\n        ),\n      );\n\n      installWebSocketReceiver<'snapshot' | 'changes' | 'subscribe'>(\n        lc,\n        fastify.websocketServer,\n        this.#receiveWebsocket,\n        parent,\n      );\n    });\n\n    this.#lc = lc;\n    this.#opts = opts;\n    this.#changeStreamer = changeStreamer;\n  }\n\n  // Called when receiving a web socket via the main dispatcher handoff.\n  readonly #receiveWebsocket = (\n    ws: WebSocket,\n    action: 'changes' | 'snapshot' | 'subscribe',\n    msg: IncomingMessageSubset,\n  ) => {\n    switch (action) {\n      case 'snapshot':\n        return this.#reserveSnapshot(ws, msg);\n      case 'changes':\n        return this.#changes(ws, msg);\n      case 'subscribe':\n        return this.#subscribe(ws, msg);\n      default:\n        closeWithError(\n          this._lc,\n          ws,\n          `invalid action \"${action}\" received in handoff`,\n        );\n        return;\n    }\n  };\n\n  readonly #reserveSnapshot = async (ws: WebSocket, req: RequestHeaders) => {\n    this.#ensureChangeStreamerStarted('incoming snapshot reservation');\n    try {\n      const url = new URL(\n        req.url ?? '',\n        req.headers.origin ?? 'http://localhost',\n      );\n      checkProtocolVersion(url.pathname);\n      const taskID = url.searchParams.get('taskID');\n      if (!taskID) {\n        throw new Error('Missing taskID in snapshot request');\n      }\n      const downstream =\n        await this.#changeStreamer.startSnapshotReservation(taskID);\n      // Send-only for now: the reservation carries no inbound application\n      // messages (only transport acks, demuxed internally).\n      void streamInternal(this._lc, ws, undefined, downstream);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  readonly #changes = async (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const ctx = getSubscriberContext(req);\n      if (ctx.mode === 'serving') {\n        this.#ensureChangeStreamerStarted('incoming subscription');\n      }\n\n      const downstream = await this.#changeStreamer.subscribe(ctx);\n      // Send-only for now: the subscription carries no inbound application\n      // messages (only transport acks, demuxed internally).\n      void streamInternalStringified(this._lc, ws, undefined, downstream, {\n        batched: ctx.wsBatched,\n      });\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  /**\n   * The merged v7 `/subscribe` handler. A single bidirectional connection\n   * carries, in sequence: an optional reservation phase (`reserve-snapshot` →\n   * `reserved`) and then the subscription phase (`start-subscription` → change\n   * stream). This ensures that the subscriber subscribes to the same task from\n   * which it restored the backup.\n   */\n  readonly #subscribe = async (ws: WebSocket, req: RequestHeaders) => {\n    let reservation: Source<SnapshotMessage> | undefined;\n    const outbound = Subscription.create<string | PreSerializedBatch>();\n\n    try {\n      const url = new URL(\n        req.url ?? '',\n        req.headers.origin ?? 'http://localhost',\n      );\n      const protocolVersion = checkProtocolVersion(url.pathname);\n      if (protocolVersion < MIN_SUBSCRIBE_PROTOCOL_VERSION) {\n        throw new Error(\n          `the /subscribe endpoint requires protocol v${MIN_SUBSCRIBE_PROTOCOL_VERSION} ` +\n            `(client is at v${protocolVersion})`,\n        );\n      }\n\n      const instream = await streamInternalStringified(\n        this._lc,\n        ws,\n        subscribeUpstreamSchema,\n        outbound,\n        {batched: true},\n      );\n\n      // Keep reading upstream control messages for the life of the connection.\n      // Do not manually exit loop, as that would close the socket. The loop\n      // exits naturally when the connection closes.\n      for await (const msg of instream) {\n        if (msg[0] === 'reserve-snapshot') {\n          const {taskID} = msg[1];\n          this.#ensureChangeStreamerStarted('incoming snapshot reservation');\n          reservation =\n            await this.#changeStreamer.startSnapshotReservation(taskID);\n          void this.#forwardReservation(reservation, outbound);\n        } else if (msg[0] === 'start-subscription') {\n          const ctx: SubscriberContext = {\n            ...msg[1],\n            protocolVersion,\n            wsBatched: true,\n          };\n          if (ctx.mode === 'serving') {\n            this.#ensureChangeStreamerStarted('incoming subscription');\n          }\n          // subscribe() registers the subscriber and then releases any\n          // reservation for that task.\n          reservation = undefined;\n          await this.#changeStreamer.subscribe(ctx, outbound);\n        }\n      }\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    } finally {\n      // Covers a reservation-phase disconnect (instream ended before a\n      // subscription started); idempotent with the close handler above.\n      reservation?.cancel();\n    }\n  };\n\n  /**\n   * Forwards a snapshot reservation's single `['status', ...]` message onto\n   * the merged connection's `outbound` sink as a `['reserved', ...]` message.\n   * The loop then blocks on the held-open reservation until it is cancelled,\n   * by `subscribe()` at the subscription handoff, or by the reservation-phase\n   * teardown on disconnect.\n   */\n  async #forwardReservation(\n    reservation: Source<SnapshotMessage>,\n    outbound: Subscription<string | PreSerializedBatch>,\n  ) {\n    try {\n      for await (const [, status] of reservation) {\n        // Translate the /snapshot protocol's ['status', {tag: 'status', ...}]\n        // message to the /subscribe protocol's ['reserved', {tag: 'snapshot', ...}].\n        outbound.push(\n          BigIntJSON.stringify([\n            'reserved',\n            {...status, tag: 'snapshot'},\n          ] satisfies ReservedMessage),\n        );\n      }\n    } catch (e) {\n      this._lc.warn?.(`error forwarding snapshot reservation`, e);\n      outbound.fail(e instanceof Error ? e : new Error(String(e)));\n    }\n  }\n\n  #changeStreamerStarted = false;\n\n  #ensureChangeStreamerStarted(reason?: string) {\n    if (!this.#changeStreamerStarted && this._state.shouldRun()) {\n      this.#lc.info?.(\n        `starting ChangeStreamerService ${reason ? `(${reason})` : ''}`,\n      );\n      void this.#changeStreamer\n        .run()\n        .catch(e =>\n          this.#lc.warn?.(`ChangeStreamerService ended with error`, e),\n        )\n        .finally(() => this.stop());\n\n      this.#changeStreamerStarted = true;\n    }\n  }\n\n  protected override _onStart(): void {\n    const {startupDelayMs, readinessGate = promiseVoid} = this.#opts;\n    if (startupDelayMs === 0) {\n      // In RMv2, there is no need to delay starting the change streamer\n      // because RM startup is non-disruptive.\n      this.#ensureChangeStreamerStarted();\n    } else {\n      // In RMv1, starting the change-streamer forcibly shuts down the\n      // previous change-streamer, causing view-syncers to reconnect.\n      // If this replication-manager has just started, the routing layer may\n      // not have registered it with DNS, as that only happens after it\n      // confirms health checks. To minimize downtime, the takeover is\n      // delayed for the configured startupDelayMs _after_ beginning to\n      // advertise readiness.\n      void readinessGate.then(() => {\n        this.#lc.info?.(\n          `waiting ${startupDelayMs}ms before taking over the change log`,\n        );\n        this._state.setTimeout(\n          () =>\n            this.#ensureChangeStreamerStarted(\n              `startup delay elapsed (${startupDelayMs} ms)`,\n            ),\n          startupDelayMs,\n        );\n      });\n    }\n  }\n\n  protected override async _onStop(): Promise<void> {\n    if (this.#changeStreamerStarted) {\n      await this.#changeStreamer.stop();\n    }\n  }\n}\n\nexport interface SnapshotReserver {\n  reserveSnapshot(taskID: string): Promise<{\n    reserved: SnapshotStatus;\n    followup: ReservationFollowup;\n  }>;\n}\n\nexport interface SubscriptionStarter {\n  subscribe(\n    ctx: SubscriberContext,\n  ): Promise<Source<Sized<SubscribeDownstream>>>;\n}\n\nexport interface ReservationFollowup extends SubscriptionStarter {\n  /**\n   * A signal to indicate that the followup is no longer valid (i.e. the\n   * connection to the reserving task has been severed).\n   */\n  readonly signal: AbortSignal;\n\n  /**\n   * Abandons the reservation without subscribing, closing the connection that\n   * was held open to pin the change-log floor. Used when the caller cannot\n   * proceed to `subscribe()` (e.g. process shutdown while a restore is in\n   * progress) and must not leave the reservation's connection open forever.\n   */\n  cancel(reason?: Error): void;\n}\n\nexport class ChangeStreamerHttpClient\n  implements ChangeStreamer, SnapshotReserver, SubscriptionStarter\n{\n  readonly #lc: LogContext;\n  readonly #shardID: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #changeStreamerURI: string | undefined;\n\n  constructor(\n    lc: LogContext,\n    shardID: ShardID,\n    changeDB: string,\n    changeStreamerURI: string | undefined,\n  ) {\n    this.#lc = lc;\n    this.#shardID = shardID;\n    // Create a pg client with a single short-lived connection for the purpose\n    // of change-streamer discovery (i.e. ChangeDB as DNS).\n    this.#changeDB = pgClient(lc, changeDB, 'change-streamer-discovery', {\n      max: 1,\n      ['idle_timeout']: 15,\n    });\n    this.#changeStreamerURI = changeStreamerURI;\n  }\n\n  async #resolveChangeStreamer(path: string) {\n    let baseURL = this.#changeStreamerURI;\n    if (!baseURL) {\n      const address = await discoverChangeStreamerAddress(\n        this.#shardID,\n        this.#changeDB,\n      );\n      if (!address) {\n        throw new Error(`no change-streamer is running`);\n      }\n      baseURL = address.includes('://') ? `${address}/` : `ws://${address}/`;\n    }\n    const uri = new URL(path, baseURL);\n    this.#lc.info?.(`connecting to change-streamer@${uri}`);\n    return uri;\n  }\n\n  async reserveSnapshot(taskID: string): Promise<{\n    reserved: SnapshotStatus;\n    followup: ReservationFollowup;\n  }> {\n    const {instream, outbound} = await this.connect(taskID);\n    outbound.push(['reserve-snapshot', {taskID}]);\n\n    const {head, rest} = readHead(instream);\n    const item = await head.next();\n    if (!item.value) {\n      throw new Error('stream closed without snapshot reservation');\n    }\n    const {data: msg} = item.value;\n    if (msg[0] !== 'reserved') {\n      throw new Error(`expected \"reserved\" message but got ${msg[0]}`);\n    }\n    const reserved: SnapshotStatus = msg[1];\n    const followup: ReservationFollowup = {\n      subscribe: ctx => {\n        outbound.push(['start-subscription', toStartSubscriptionContext(ctx)]);\n        return Promise.resolve(rest);\n      },\n      signal: outbound.signal,\n      cancel: reason => outbound.cancel(reason),\n    };\n\n    return {reserved, followup};\n  }\n\n  async subscribe(\n    ctx: SubscriberContext,\n  ): Promise<Source<Sized<SubscribeDownstream>>> {\n    const {instream, outbound} = await this.connect(ctx.taskID);\n    outbound.push(['start-subscription', toStartSubscriptionContext(ctx)]);\n    return instream;\n  }\n\n  /**\n   * Opens a merged v7 `/subscribe` connection (see subscribe.ts). Returns the\n   * inbound change/reservation stream and an `outbound` sink into which the\n   * caller pushes the `reserve-snapshot` / `start-subscription` control\n   * messages that drive the connection's phases. The single connection spans\n   * the reservation, backup restore, and subscription, so the reserving\n   * replication-manager is the one that serves the subscription.\n   */\n  // exported for testing\n  async connect(taskID: string): Promise<{\n    instream: Source<Sized<SubscribeDownstream>>;\n    outbound: Subscription<SubscribeUpstream>;\n  }> {\n    const uri = await this.#resolveChangeStreamer(SUBSCRIBE_PATH);\n\n    // taskID is carried in the query for observability/routing; the server\n    // reads the authoritative taskID from the control messages themselves.\n    const params = new URLSearchParams({taskID});\n    // Bound the TCP connect + HTTP upgrade. Without this, a SYN black-holed\n    // by a stale Service endpoint (e.g. a just-terminated replication-manager\n    // pod) is retransmitted by the kernel for ~2 minutes before ETIMEDOUT.\n    // Failing fast lets the caller's retry open a fresh connection that is\n    // routed to a live endpoint.\n    const ws = new WebSocket(uri + `?${params.toString()}`, {\n      handshakeTimeout: CONNECT_TIMEOUT_MS,\n    });\n\n    const outbound = Subscription.create<SubscribeUpstream>();\n    const instream = await streamInternalWithSize(\n      this.#lc,\n      ws,\n      subscribeDownstreamSchema,\n      outbound,\n    );\n    return {instream, outbound};\n  }\n}\n\ntype RequestHeaders = Pick<IncomingMessage, 'url' | 'headers'>;\n\nexport function getSubscriberContext(req: RequestHeaders): SubscriberContext {\n  const url = new URL(req.url ?? '', req.headers.origin ?? 'http://localhost');\n  const protocolVersion = checkProtocolVersion(url.pathname);\n  const params = new URLParams(url);\n\n  return {\n    protocolVersion,\n    id: params.get('id', true),\n    taskID: params.get('taskID', true),\n    mode: params.get('mode', false) === 'backup' ? 'backup' : 'serving',\n    replicaVersion: params.get('replicaVersion', true),\n    watermark: params.get('watermark', true),\n    // Absent for subscribers that predate the parameter, which is the safe\n    // default: the barrier falls back to polling rather than waiting on an\n    // ACK that would never be attributed to a writer.\n    wsBatched: params.getBoolean('wsBatched'),\n  };\n}\n\nfunction checkProtocolVersion(pathname: string): number {\n  const match = PATH_REGEX.exec(pathname);\n  if (!match) {\n    throw new Error(`invalid path: ${pathname}`);\n  }\n  const v = Number(match.groups?.version);\n  if (\n    Number.isNaN(v) ||\n    v > PROTOCOL_VERSION ||\n    v < MIN_SUPPORTED_PROTOCOL_VERSION\n  ) {\n    throw new Error(\n      `Cannot service client at protocol v${v}. ` +\n        `Supported protocols: [v${MIN_SUPPORTED_PROTOCOL_VERSION} ... v${PROTOCOL_VERSION}]`,\n    );\n  }\n  return v;\n}\n\n// Projects a client-side SubscriberContext onto the start-subscription\n// message payload, dropping the fields that are implicit on the merged\n// connection: protocolVersion (from the request path), wsBatched (always on).\nfunction toStartSubscriptionContext(ctx: SubscriberContext): SubscribeContext {\n  const {taskID, id, mode, replicaVersion, watermark, backfills} = ctx;\n  return {taskID, id, mode, replicaVersion, watermark, backfills};\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AA4CA,IAAM,iCAAiC;AAGvC,IAAM,iCAAiC;AAEvC,IAAM,wBAAwB;AAC9B,IAAM,uBAAuB;AAC7B,IAAM,yBAAyB;AAC/B,IAAM,aACJ;AAEF,IAAM,iBAAiB;AAIvB,IAAM,qBAAqB;AAU3B,IAAa,2BAAb,cAA8C,YAAY;CACxD,KAAc;CACd;CACA;CACA;CAEA,YACE,IACA,MACA,QACA,gBACA;EACA,MAAM,+BAA+B,IAAI,MAAM,OAAM,YAAW;GAC9D,MAAM,QAAQ,SAAS,SAAS;GAEhC,QAAQ,IAAI,sBAAsB,EAAC,WAAW,KAAI,GAAG,KAAKG,QAAQ;GAClE,QAAQ,IACN,uBACA,EAAC,WAAW,KAAI,GAChB,KAAKC,gBACP;GACA,QAAQ,IAAI,wBAAwB,EAAC,WAAW,KAAI,GAAG,KAAKC,UAAU;GAEtE,QAAQ,IAAI,WAAW,KAAK,QAC1B,mBACE,IACA,KAAK,UAAU;IACb,eAAe,KAAA;IACf,kBAAkB,KAAA;GACpB,GACA,KACA,KACA,KAAK,kBACL,KAAA,GACA,iBACF,CACF;GAEA,yBACE,IACA,QAAQ,iBACR,KAAKC,mBACL,MACF;EACF,CAAC;EAED,KAAKN,MAAM;EACX,KAAKC,QAAQ;EACb,KAAKC,kBAAkB;CACzB;CAGA,qBACE,IACA,QACA,QACG;EACH,QAAQ,QAAR;GACE,KAAK,YACH,OAAO,KAAKE,iBAAiB,IAAI,GAAG;GACtC,KAAK,WACH,OAAO,KAAKD,SAAS,IAAI,GAAG;GAC9B,KAAK,aACH,OAAO,KAAKE,WAAW,IAAI,GAAG;GAChC;IACE,eACE,KAAK,KACL,IACA,mBAAmB,OAAO,sBAC5B;IACA;EACJ;CACF;CAEA,mBAA4B,OAAO,IAAe,QAAwB;EACxE,KAAKE,6BAA6B,+BAA+B;EACjE,IAAI;GACF,MAAM,MAAM,IAAI,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,kBACxB;GACA,qBAAqB,IAAI,QAAQ;GACjC,MAAM,SAAS,IAAI,aAAa,IAAI,QAAQ;GAC5C,IAAI,CAAC,QACH,MAAM,IAAI,MAAM,oCAAoC;GAEtD,MAAM,aACJ,MAAM,KAAKL,gBAAgB,yBAAyB,MAAM;GAG5D,eAAoB,KAAK,KAAK,IAAI,KAAA,GAAW,UAAU;EACzD,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;CAEA,WAAoB,OAAO,IAAe,QAAwB;EAChE,IAAI;GACF,MAAM,MAAM,qBAAqB,GAAG;GACpC,IAAI,IAAI,SAAS,WACf,KAAKK,6BAA6B,uBAAuB;GAG3D,MAAM,aAAa,MAAM,KAAKL,gBAAgB,UAAU,GAAG;GAG3D,0BAA+B,KAAK,KAAK,IAAI,KAAA,GAAW,YAAY,EAClE,SAAS,IAAI,UACf,CAAC;EACH,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;;;;;;;;CASA,aAAsB,OAAO,IAAe,QAAwB;EAClE,IAAI;EACJ,MAAM,WAAW,aAAa,OAAoC;EAElE,IAAI;GAKF,MAAM,kBAAkB,qBAAqB,IAJ7B,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,kBAEqB,EAAI,QAAQ;GACzD,IAAI,kBAAkB,gCACpB,MAAM,IAAI,MACR,8CAA8C,+BAA+B,kBACzD,gBAAgB,EACtC;GAGF,MAAM,WAAW,MAAM,0BACrB,KAAK,KACL,IACA,yBACA,UACA,EAAC,SAAS,KAAI,CAChB;GAKA,WAAW,MAAM,OAAO,UACtB,IAAI,IAAI,OAAO,oBAAoB;IACjC,MAAM,EAAC,WAAU,IAAI;IACrB,KAAKK,6BAA6B,+BAA+B;IACjE,cACE,MAAM,KAAKL,gBAAgB,yBAAyB,MAAM;IAC5D,KAAUM,oBAAoB,aAAa,QAAQ;GACrD,OAAO,IAAI,IAAI,OAAO,sBAAsB;IAC1C,MAAM,MAAyB;KAC7B,GAAG,IAAI;KACP;KACA,WAAW;IACb;IACA,IAAI,IAAI,SAAS,WACf,KAAKD,6BAA6B,uBAAuB;IAI3D,cAAc,KAAA;IACd,MAAM,KAAKL,gBAAgB,UAAU,KAAK,QAAQ;GACpD;EAEJ,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD,UAAU;GAGR,aAAa,OAAO;EACtB;CACF;;;;;;;;CASA,MAAMM,oBACJ,aACA,UACA;EACA,IAAI;GACF,WAAW,MAAM,GAAG,WAAW,aAG7B,SAAS,KACP,WAAW,UAAU,CACnB,YACA;IAAC,GAAG;IAAQ,KAAK;GAAU,CAC7B,CAA2B,CAC7B;EAEJ,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,yCAAyC,CAAC;GAC1D,SAAS,KAAK,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,CAAC,CAAC,CAAC;EAC7D;CACF;CAEA,yBAAyB;CAEzB,6BAA6B,QAAiB;EAC5C,IAAI,CAAC,KAAKC,0BAA0B,KAAK,OAAO,UAAU,GAAG;GAC3D,KAAKT,IAAI,OACP,kCAAkC,SAAS,IAAI,OAAO,KAAK,IAC7D;GACA,KAAUE,gBACP,IAAI,EACJ,OAAM,MACL,KAAKF,IAAI,OAAO,0CAA0C,CAAC,CAC7D,EACC,cAAc,KAAK,KAAK,CAAC;GAE5B,KAAKS,yBAAyB;EAChC;CACF;CAEA,WAAoC;EAClC,MAAM,EAAC,gBAAgB,gBAAgB,gBAAe,KAAKR;EAC3D,IAAI,mBAAmB,GAGrB,KAAKM,6BAA6B;OASlC,cAAmB,WAAW;GAC5B,KAAKP,IAAI,OACP,WAAW,eAAe,qCAC5B;GACA,KAAK,OAAO,iBAER,KAAKO,6BACH,0BAA0B,eAAe,KAC3C,GACF,cACF;EACF,CAAC;CAEL;CAEA,MAAyB,UAAyB;EAChD,IAAI,KAAKE,wBACP,MAAM,KAAKP,gBAAgB,KAAK;CAEpC;AACF;AA+BA,IAAa,2BAAb,MAEA;CACE;CACA;CACA;CACA;CAEA,YACE,IACA,SACA,UACA,mBACA;EACA,KAAKF,MAAM;EACX,KAAKU,WAAW;EAGhB,KAAKC,YAAY,SAAS,IAAI,UAAU,6BAA6B;GACnE,KAAK;IACJ,iBAAiB;EACpB,CAAC;EACD,KAAKC,qBAAqB;CAC5B;CAEA,MAAMC,uBAAuB,MAAc;EACzC,IAAI,UAAU,KAAKD;EACnB,IAAI,CAAC,SAAS;GACZ,MAAM,UAAU,MAAM,8BACpB,KAAKF,UACL,KAAKC,SACP;GACA,IAAI,CAAC,SACH,MAAM,IAAI,MAAM,+BAA+B;GAEjD,UAAU,QAAQ,SAAS,KAAK,IAAI,GAAG,QAAQ,KAAK,QAAQ,QAAQ;EACtE;EACA,MAAM,MAAM,IAAI,IAAI,MAAM,OAAO;EACjC,KAAKX,IAAI,OAAO,iCAAiC,KAAK;EACtD,OAAO;CACT;CAEA,MAAM,gBAAgB,QAGnB;EACD,MAAM,EAAC,UAAU,aAAY,MAAM,KAAK,QAAQ,MAAM;EACtD,SAAS,KAAK,CAAC,oBAAoB,EAAC,OAAM,CAAC,CAAC;EAE5C,MAAM,EAAC,MAAM,SAAQ,SAAS,QAAQ;EACtC,MAAM,OAAO,MAAM,KAAK,KAAK;EAC7B,IAAI,CAAC,KAAK,OACR,MAAM,IAAI,MAAM,4CAA4C;EAE9D,MAAM,EAAC,MAAM,QAAO,KAAK;EACzB,IAAI,IAAI,OAAO,YACb,MAAM,IAAI,MAAM,uCAAuC,IAAI,IAAI;EAYjE,OAAO;GAAC,UAVyB,IAAI;GAUnB,UAAA;IARhB,YAAW,QAAO;KAChB,SAAS,KAAK,CAAC,sBAAsB,2BAA2B,GAAG,CAAC,CAAC;KACrE,OAAO,QAAQ,QAAQ,IAAI;IAC7B;IACA,QAAQ,SAAS;IACjB,SAAQ,WAAU,SAAS,OAAO,MAAM;GAGxB;EAAQ;CAC5B;CAEA,MAAM,UACJ,KAC6C;EAC7C,MAAM,EAAC,UAAU,aAAY,MAAM,KAAK,QAAQ,IAAI,MAAM;EAC1D,SAAS,KAAK,CAAC,sBAAsB,2BAA2B,GAAG,CAAC,CAAC;EACrE,OAAO;CACT;;;;;;;;;CAWA,MAAM,QAAQ,QAGX;EAWD,MAAM,KAAK,IAAI,UAAU,MAVP,KAAKa,uBAAuB,cAAc,IAU7B,IAAI,IANhB,gBAAgB,EAAC,OAAM,CAMP,EAAO,SAAS,KAAK,EACtD,kBAAkB,mBACpB,CAAC;EAED,MAAM,WAAW,aAAa,OAA0B;EAOxD,OAAO;GAAC,UAAA,MANe,uBACrB,KAAKb,KACL,IACA,2BACA,QACF;GACkB;EAAQ;CAC5B;AACF;AAIA,SAAgB,qBAAqB,KAAwC;CAC3E,MAAM,MAAM,IAAI,IAAI,IAAI,OAAO,IAAI,IAAI,QAAQ,UAAU,kBAAkB;CAC3E,MAAM,kBAAkB,qBAAqB,IAAI,QAAQ;CACzD,MAAM,SAAS,IAAI,UAAU,GAAG;CAEhC,OAAO;EACL;EACA,IAAI,OAAO,IAAI,MAAM,IAAI;EACzB,QAAQ,OAAO,IAAI,UAAU,IAAI;EACjC,MAAM,OAAO,IAAI,QAAQ,KAAK,MAAM,WAAW,WAAW;EAC1D,gBAAgB,OAAO,IAAI,kBAAkB,IAAI;EACjD,WAAW,OAAO,IAAI,aAAa,IAAI;EAIvC,WAAW,OAAO,WAAW,WAAW;CAC1C;AACF;AAEA,SAAS,qBAAqB,UAA0B;CACtD,MAAM,QAAQ,WAAW,KAAK,QAAQ;CACtC,IAAI,CAAC,OACH,MAAM,IAAI,MAAM,iBAAiB,UAAU;CAE7C,MAAM,IAAI,OAAO,MAAM,QAAQ,OAAO;CACtC,IACE,OAAO,MAAM,CAAC,KACd,IAAA,KACA,IAAI,gCAEJ,MAAM,IAAI,MACR,sCAAsC,EAAE,2BACZ,+BAA+B,SAC7D;CAEF,OAAO;AACT;AAKA,SAAS,2BAA2B,KAA0C;CAC5E,MAAM,EAAC,QAAQ,IAAI,MAAM,gBAAgB,WAAW,cAAa;CACjE,OAAO;EAAC;EAAQ;EAAI;EAAM;EAAgB;EAAW;CAAS;AAChE"}

package/out/zero-cache/src/services/litestream/commands.d.ts

@@ -1,9 +1,7 @@
 import type { ChildProcess } from 'node:child_process';
 import type { LogContext } from '@rocicorp/logger';
 import { type LitestreamConfig } from '../../config/normalize.ts';
-import type { InitCleanup } from '../change-source/common/init-cleanup.ts';
 import { type LitestreamRole } from './metrics.ts';
-import { WalKeeper } from './wal-keeper.ts';
 export type ReplicaConstraints = {
     replicaVersion: string;
     minWatermark: string;
@@ -14,11 +12,11 @@ export type RestoreAttempt = {
     backupURL: string | undefined;
     result: RestoreResult;
     /**
-     * Set when the restore prepared the backup to continue from the restored
-     * replica (see {@link ForkBackup}). It is registered with the
-     * `cleanup`; otherwise the caller must release it (see {@link WalKeeper}).
+     * Set when the restore forked the backup (see {@link ForkBackup}), which
+     * then continues from the restored replica's WAL. The WAL is kept when the
+     * last connection closes (see `Database.persistWal()`).
      */
-    walKeeper?: WalKeeper | undefined;
+    forked?: boolean | undefined;
 };
 /**
  * Prepares the restored replica to continue from a forked backup
@@ -27,11 +25,6 @@ export type RestoreAttempt = {
 export type ForkBackup = {
     /** The restore also forks its backup into the (new, empty) one at this URL. */
     url: string;
-    /**
-     * Registers the resulting {@link WalKeeper} to be released if the
-     * initialization fails.
-     */
-    cleanup?: InitCleanup | undefined;
 };
 export declare class BackupNotFoundException extends Error {
     static readonly name = "BackupNotFoundException";

package/out/zero-cache/src/services/litestream/commands.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"commands.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAIrD,OAAO,KAAK,EAAC,UAAU,EAAW,MAAM,kBAAkB,CAAC;AAK3D,OAAO,EAAC,KAAK,gBAAgB,EAAC,MAAM,2BAA2B,CAAC;AAOhE,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,yCAAyC,CAAC;AAIzE,OAAO,EAWL,KAAK,cAAc,EACpB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAC,SAAS,EAAC,MAAM,iBAAiB,CAAC;AAE1C,MAAM,MAAM,kBAAkB,GAAG;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,aAAa,GACrB,SAAS,GACT,WAAW,GACX,iBAAiB,GACjB,OAAO,CAAC;AAEZ,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,EAAE,OAAO,CAAC;IAClB,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,MAAM,EAAE,aAAa,CAAC;IACtB;;;;OAIG;IACH,SAAS,CAAC,EAAE,SAAS,GAAG,SAAS,CAAC;CACnC,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,UAAU,GAAG;IACvB,+EAA+E;IAC/E,GAAG,EAAE,MAAM,CAAC;IACZ;;;OAGG;IACH,OAAO,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC;CACnC,CAAC;AAKF,qBAAa,uBAAwB,SAAQ,KAAK;IAChD,MAAM,CAAC,QAAQ,CAAC,IAAI,6BAA6B;IAEjD,YAAY,SAAS,EAAE,MAAM,GAAG,SAAS,EAExC;CACF;AAED;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,IAAI,CAAC,gBAAgB,EAAE,WAAW,GAAG,mBAAmB,CAAC,GAChE,MAAM,GAAG,SAAS,CAEpB;AA6FD,wBAAsB,UAAU,CAC9B,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,MAAM,EACnB,kBAAkB,EAAE,kBAAkB,GAAG,SAAS,EAClD,IAAI,EAAE,cAAc,EACpB,MAAM,CAAC,EAAE,WAAW,EACpB,UAAU,CAAC,EAAE,UAAU,GAAG,SAAS,GAClC,OAAO,CAAC,cAAc,CAAC,CA2BzB;AAiOD;;;GAGG;AACH,wBAAgB,uBAAuB,CAAC,WAAW,EAAE,MAAM,QAK1D;AA2GD,wBAAgB,yBAAyB,CACvC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,MAAM,GAClB,YAAY,CAuCd;AAQD;;;;;;;;;;;;;GAaG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,IAAI,CAAC,CAgBf;AA8DD;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CACrC,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,WAAW,GAAG,KAAK,EAC5B,MAAM,EAAE,MAAM,GACb,IAAI,EAAE,CAoBR"}
+{"version":3,"file":"commands.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAIrD,OAAO,KAAK,EAAC,UAAU,EAAW,MAAM,kBAAkB,CAAC;AAK3D,OAAO,EAAC,KAAK,gBAAgB,EAAC,MAAM,2BAA2B,CAAC;AAUhE,OAAO,EAWL,KAAK,cAAc,EACpB,MAAM,cAAc,CAAC;AAEtB,MAAM,MAAM,kBAAkB,GAAG;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,aAAa,GACrB,SAAS,GACT,WAAW,GACX,iBAAiB,GACjB,OAAO,CAAC;AAEZ,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,EAAE,OAAO,CAAC;IAClB,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,MAAM,EAAE,aAAa,CAAC;IACtB;;;;OAIG;IACH,MAAM,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CAC9B,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,UAAU,GAAG;IACvB,+EAA+E;IAC/E,GAAG,EAAE,MAAM,CAAC;CACb,CAAC;AAKF,qBAAa,uBAAwB,SAAQ,KAAK;IAChD,MAAM,CAAC,QAAQ,CAAC,IAAI,6BAA6B;IAEjD,YAAY,SAAS,EAAE,MAAM,GAAG,SAAS,EAExC;CACF;AAED;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,IAAI,CAAC,gBAAgB,EAAE,WAAW,GAAG,mBAAmB,CAAC,GAChE,MAAM,GAAG,SAAS,CAEpB;AA6FD,wBAAsB,UAAU,CAC9B,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,MAAM,EACnB,kBAAkB,EAAE,kBAAkB,GAAG,SAAS,EAClD,IAAI,EAAE,cAAc,EACpB,MAAM,CAAC,EAAE,WAAW,EACpB,UAAU,CAAC,EAAE,UAAU,GAAG,SAAS,GAClC,OAAO,CAAC,cAAc,CAAC,CA2BzB;AAsND;;;GAGG;AACH,wBAAgB,uBAAuB,CAAC,WAAW,EAAE,MAAM,QAK1D;AA2GD,wBAAgB,yBAAyB,CACvC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,MAAM,GAClB,YAAY,CAuCd;AAQD;;;;;;;;;;;;;GAaG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,IAAI,CAAC,CAgBf;AA8DD;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CACrC,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,WAAW,GAAG,KAAK,EAC5B,MAAM,EAAE,MAAM,GACb,IAAI,EAAE,CAoBR"}

package/out/zero-cache/src/services/litestream/commands.js

@@ -9,7 +9,6 @@ import { deleteLiteDB } from "../../db/delete-lite-db.js";
 import { deleteChangeLogDB } from "../replicator/change-log-db.js";
 import { litestreamSocketPath } from "./litestream-controller.js";
 import { litestreamBackupListDuration, litestreamBackupMetricAttrs, litestreamBackupProcessDuration, litestreamBackupProcessMetricAttrs, litestreamBackupProcessRuns, litestreamRestoreAttempts, litestreamRestoreMetricAttrs, litestreamRestoreProcessDuration, litestreamRestoreValidationDuration, litestreamRestoredDbBytes } from "./metrics.js";
-import { WalKeeper } from "./wal-keeper.js";
 import { resolver } from "@rocicorp/resolver";
 import { spawn } from "node:child_process";
 import { existsSync, readdirSync, rmSync, statSync } from "node:fs";
@@ -150,7 +149,7 @@ async function restoreOnce(lc, config, replicaFile, replicaConstraints, role, si
 			};
 		}
 		const validationStart = performance.now();
-		let db = openValidReplica(lc, replicaFile, replicaConstraints);
+		const db = openValidReplica(lc, replicaFile, replicaConstraints);
 		litestreamRestoreValidationDuration().recordMs(performance.now() - validationStart, {
 			...attrs,
 			result: db ? "success" : "invalid_replica"
@@ -177,20 +176,16 @@ async function restoreOnce(lc, config, replicaFile, replicaConstraints, role, si
 					result: "success"
 				});
 			}
-			const walKeeper = forkBackup && !reusedExisting && restoreForked(lc, stdout) ? new WalKeeper(lc, db) : void 0;
-			if (walKeeper) {
-				db = void 0;
-				forkBackup?.cleanup?.onFailure("WAL keeper", () => walKeeper.release("initialization failed"));
-			}
+			const forked = forkBackup !== void 0 && !reusedExisting && restoreForked(lc, stdout);
 			return {
 				restored: true,
 				backupURL,
 				result,
 				reusedExisting,
-				walKeeper
+				forked
 			};
 		} finally {
-			db?.close();
+			db.close();
 		}
 	} finally {
 		try {

package/out/zero-cache/src/services/litestream/commands.js.map

@@ -1 +1 @@
-{"version":3,"file":"commands.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync, readdirSync, rmSync, statSync} from 'node:fs';\nimport {basename, dirname, join} from 'node:path';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../../shared/src/must.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {type LitestreamConfig} from '../../config/normalize.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {\n  isSQLiteCorruption,\n  logSQLiteCorruptionDiagnostics,\n} from '../../db/sqlite-corruption.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport type {InitCleanup} from '../change-source/common/init-cleanup.ts';\nimport {deleteChangeLogDB} from '../replicator/change-log-db.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {litestreamSocketPath} from './litestream-controller.ts';\nimport {\n  litestreamBackupListDuration,\n  litestreamBackupMetricAttrs,\n  litestreamBackupProcessDuration,\n  litestreamBackupProcessMetricAttrs,\n  litestreamBackupProcessRuns,\n  litestreamRestoreAttempts,\n  litestreamRestoredDbBytes,\n  litestreamRestoreMetricAttrs,\n  litestreamRestoreProcessDuration,\n  litestreamRestoreValidationDuration,\n  type LitestreamRole,\n} from './metrics.ts';\nimport {WalKeeper} from './wal-keeper.ts';\n\nexport type ReplicaConstraints = {\n  replicaVersion: string;\n  minWatermark: string;\n};\n\nexport type RestoreResult =\n  | 'success'\n  | 'no_backup'\n  | 'invalid_replica'\n  | 'error';\n\nexport type RestoreAttempt = {\n  restored: boolean;\n  backupURL: string | undefined;\n  result: RestoreResult;\n  /**\n   * Set when the restore prepared the backup to continue from the restored\n   * replica (see {@link ForkBackup}). It is registered with the\n   * `cleanup`; otherwise the caller must release it (see {@link WalKeeper}).\n   */\n  walKeeper?: WalKeeper | undefined;\n};\n\n/**\n * Prepares the restored replica to continue from a forked backup\n * instead of starting a new lineage with a full snapshot.\n */\nexport type ForkBackup = {\n  /** The restore also forks its backup into the (new, empty) one at this URL. */\n  url: string;\n  /**\n   * Registers the resulting {@link WalKeeper} to be released if the\n   * initialization fails.\n   */\n  cleanup?: InitCleanup | undefined;\n};\n\nconst MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES = 100;\nconst LEGACY_RESTORE_WAL_INDEX = /^[0-9a-f]{8}$/;\n\nexport class BackupNotFoundException extends Error {\n  static readonly name = 'BackupNotFoundException';\n\n  constructor(backupURL: string | undefined) {\n    super(`backup not found at ${backupURL}`);\n  }\n}\n\n/**\n * The effective backup URL: the `backupURLOverride` if present (i.e. a\n * lineage-specific location computed by the replication-manager), or\n * otherwise the configured `backupURL`.\n */\nexport function backupDestinationURL(\n  config: Pick<LitestreamConfig, 'backupURL' | 'backupURLOverride'>,\n): string | undefined {\n  return config.backupURLOverride ?? config.backupURL;\n}\n\nfunction getLitestream(\n  mode: 'restore' | 'replicate',\n  config: LitestreamConfig,\n  replicaFile: string,\n  logLevelOverride?: LogLevel,\n): {\n  litestream: string;\n  env: NodeJS.ProcessEnv;\n} {\n  const {\n    executable,\n    executableV5,\n    restoreUsingV5,\n    backupUsingV5,\n    logLevel,\n    configPath,\n    configPathV5,\n    endpoint,\n    region,\n    port,\n    checkpointThresholdMB,\n    // Note: This assumes the default page size. If page size configuration\n    // is added, this computation must be adjusted accordingly.\n    minCheckpointPageCount = checkpointThresholdMB * 256, // 1 MiB / 4 KiB page size\n    maxCheckpointPageCount = minCheckpointPageCount * 10,\n    forceCheckpointThresholdMB,\n    incrementalBackupIntervalMinutes,\n    incrementalBackupIntervalSeconds,\n    snapshotBackupIntervalHours,\n    multipartConcurrency,\n    multipartSize,\n  } = config;\n\n  const v5 =\n    (mode === 'restore' && restoreUsingV5) ||\n    (mode === 'replicate' && backupUsingV5);\n  const litestream =\n    (v5 ? executableV5 : executable) ??\n    must(executable, `Missing --litestream-executable`);\n  const litestreamConfig = v5 ? configPathV5 : configPath;\n  // Disable truncate-page-n if forced checkpoints are enabled,\n  // and otherwise use litestream's default.\n  const truncatePageN = forceCheckpointThresholdMB ? -1 : 121359;\n\n  return {\n    litestream,\n    env: {\n      ...process.env,\n      ['ZERO_REPLICA_FILE']: replicaFile,\n      ['ZERO_LITESTREAM_BACKUP_URL']: must(backupDestinationURL(config)),\n      ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n        minCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n        maxCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n        incrementalBackupIntervalMinutes, // v3 only\n      ),\n      ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_SECONDS']: String(\n        incrementalBackupIntervalSeconds, // v5 only\n      ),\n      // v5 only\n      ['ZERO_LITESTREAM_MONITOR_INTERVAL_SECONDS']: String(\n        // zero-cache explicitly requests syncs at the incremental backup\n        // interval (see LitestreamSyncRequester), and litestream's\n        // own monitor becomes a backstop that keeps backups and checkpoints\n        // progressing if zero-cache stops requesting them.\n        2 * incrementalBackupIntervalSeconds,\n      ),\n      ['ZERO_LITESTREAM_TRUNCATE_PAGE_N']: String(truncatePageN),\n      ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n      ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_HOURS']: String(\n        snapshotBackupIntervalHours,\n      ),\n      ['ZERO_LITESTREAM_SNAPSHOT_RETENTION_INTERVAL_HOURS']: String(\n        snapshotBackupIntervalHours + 6, // delete old snapshots after 6 hours\n      ),\n      ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n      ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n      ['ZERO_LOG_FORMAT']: 'json',\n      ['ZERO_LITESTREAM_SOCKET_PATH']: litestreamSocketPath(replicaFile),\n      ['LITESTREAM_CONFIG']: litestreamConfig,\n      ['LITESTREAM_PORT']: String(port),\n      ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n      ...(region ? {['ZERO_LITESTREAM_REGION']: region} : {}),\n      ['LITESTREAM_COMPACTION_SERIALIZE']: 'false',\n    },\n  };\n}\n\nexport async function tryRestore(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  replicaConstraints: ReplicaConstraints | undefined,\n  role: LitestreamRole,\n  signal?: AbortSignal,\n  forkBackup?: ForkBackup | undefined,\n): Promise<RestoreAttempt> {\n  const {reusedExisting, ...attempt} = await restoreOnce(\n    lc,\n    config,\n    replicaFile,\n    replicaConstraints,\n    role,\n    signal,\n    forkBackup,\n  );\n  if (attempt.result === 'invalid_replica' && reusedExisting) {\n    // `-if-db-not-exists` made the restore a no-op, and the existing local\n    // replica (e.g. one left behind by a failed initial sync of a different\n    // generation) was invalid. It has since been deleted, so restore from the\n    // backup.\n    const {reusedExisting: _, ...retry} = await restoreOnce(\n      lc,\n      config,\n      replicaFile,\n      replicaConstraints,\n      role,\n      signal,\n      forkBackup,\n    );\n    return retry;\n  }\n  return attempt;\n}\n\nasync function restoreOnce(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  replicaConstraints: ReplicaConstraints | undefined,\n  role: LitestreamRole,\n  signal: AbortSignal | undefined,\n  forkBackup: ForkBackup | undefined,\n): Promise<RestoreAttempt & {reusedExisting: boolean}> {\n  const {backupUsingV5} = config;\n  const backupURL = backupDestinationURL(config);\n  if (forkBackup && !backupUsingV5) {\n    forkBackup = undefined; // Ignore forkBackup if v5 is disabled.\n  }\n  const attrs = litestreamRestoreMetricAttrs(config, role, backupURL);\n  let result: RestoreResult = 'error';\n  try {\n    logRestoreDirectoryContents(lc, replicaFile, 'before-temp-cleanup');\n    deleteRestoreTempFiles(replicaFile);\n    const reusedExisting = existsSync(replicaFile);\n    if (!reusedExisting) {\n      // The local litestream state (and anything recorded in it) describes a\n      // replica that no longer exists. Discard it before the restore, which\n      // (if it seeds a backup) writes the state that belongs to the new one.\n      deleteLitestreamMetaDir(replicaFile);\n    }\n    const {litestream, env} = getLitestream(\n      'restore',\n      config,\n      replicaFile,\n      'debug', // Include all output from `litestream restore`, as it's minimal.\n    );\n    const {\n      restoreParallelism: parallelism,\n      multipartConcurrency,\n      multipartSize,\n    } = config;\n    lc.info?.(`starting litestream restore`, {\n      restoreParallelism: parallelism,\n      multipartConcurrency,\n      multipartSize,\n    });\n    // Pipe (rather than inherit) litestream's stdout/stderr so that its own\n    // `\"level\":\"ERROR\"` output on a failed restore does not go straight to the\n    // pod's stdout — where a log-scraper alert would page on it — before our\n    // code has decided how to handle the failure. The captured output is\n    // included in the thrown error so the caller can log the final outcome.\n    // See INC-961.\n    const proc = spawn(\n      litestream,\n      [\n        'restore',\n        '-if-db-not-exists',\n        '-if-replica-exists',\n        '-parallelism',\n        String(parallelism),\n        ...(forkBackup ? ['-json', '-fork-to-url', forkBackup.url] : []),\n        replicaFile,\n      ],\n      {env, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true, signal},\n    );\n    let stdout = '';\n    let stderr = '';\n    proc.stdout.setEncoding('utf-8');\n    proc.stderr.setEncoding('utf-8');\n    // Output AND capture the stdout and stderr, the former for observability\n    // and the latter for error handling.\n    proc.stdout.on('data', chunk => {\n      process.stdout.write(chunk);\n      stdout += chunk;\n    });\n    proc.stderr.on('data', chunk => {\n      process.stderr.write(chunk);\n      stderr += chunk;\n    });\n    const {promise, resolve, reject} = resolver();\n    proc.on('error', reject);\n    proc.on('close', (code, signal) => {\n      if (signal) {\n        reject(`litestream killed with ${signal}`);\n      } else if (code !== 0) {\n        reject(`litestream exited with code ${code}`);\n      } else {\n        resolve();\n      }\n    });\n    const processStart = performance.now();\n    try {\n      await promise;\n      litestreamRestoreProcessDuration().recordMs(\n        performance.now() - processStart,\n        {...attrs, result: 'success'},\n      );\n    } catch (e) {\n      litestreamRestoreProcessDuration().recordMs(\n        performance.now() - processStart,\n        {...attrs, result: 'error'},\n      );\n      const output = [stdout, stderr]\n        .map(value => value.trim())\n        .filter(Boolean)\n        .join('\\n');\n      throw new Error(output ? `${String(e)}\\n${output}` : String(e), {\n        cause: e,\n      });\n    }\n    if (!existsSync(replicaFile)) {\n      result = 'no_backup';\n      return {restored: false, backupURL, result, reusedExisting};\n    }\n    const validationStart = performance.now();\n    // On success, the validating connection is kept open until it is either\n    // closed or kept as the WalKeeper. Until then, its close would be the last\n    // one, which checkpoints and deletes the WAL that a seeded backup continues\n    // from.\n    let db = openValidReplica(lc, replicaFile, replicaConstraints);\n    litestreamRestoreValidationDuration().recordMs(\n      performance.now() - validationStart,\n      {...attrs, result: db ? 'success' : 'invalid_replica'},\n    );\n    if (!db) {\n      result = 'invalid_replica';\n      lc.info?.(`Deleting local replica and retrying restore`);\n      deleteLiteDB(replicaFile);\n      deleteChangeLogDB(replicaFile);\n      // The litestream state (e.g. the anchor of a seeded backup) describes\n      // the deleted replica.\n      deleteLitestreamMetaDir(replicaFile);\n      return {restored: false, backupURL, result, reusedExisting};\n    }\n    try {\n      result = 'success';\n      if (!reusedExisting) {\n        // This restore materialized the replica file, so any change log beside it\n        // was written against a replica that is no longer there. Reconciliation\n        // is the safety net — a restored replica keeps its replicaVersion, so the\n        // rule falls through to truncate-or-reseed on head mismatch — but the log\n        // is a cache and deleting it here is the explicit statement that nothing\n        // in it survives the restore.\n        //\n        // A restore that reuses an existing replica (`-if-db-not-exists` made it\n        // a no-op) deliberately keeps the log: that is the process-restart path,\n        // where the log is still the one written beside this exact replica and\n        // discarding it would cost every reconnecting subscriber a `too-old`.\n        deleteChangeLogDB(replicaFile);\n        litestreamRestoredDbBytes().add(statSync(replicaFile).size, {\n          ...attrs,\n          result: 'success',\n        });\n      }\n      // Last, so that nothing can fail after the WalKeeper is created.\n      const walKeeper =\n        forkBackup && !reusedExisting && restoreForked(lc, stdout)\n          ? new WalKeeper(lc, db)\n          : undefined;\n      if (walKeeper) {\n        db = undefined; // now owned by the WalKeeper\n        forkBackup?.cleanup?.onFailure('WAL keeper', () =>\n          walKeeper.release('initialization failed'),\n        );\n      }\n      return {restored: true, backupURL, result, reusedExisting, walKeeper};\n    } finally {\n      db?.close();\n    }\n  } finally {\n    try {\n      deleteRestoreTempFiles(replicaFile);\n    } catch (e) {\n      lc.warn?.('Unable to clean up Litestream restore temporary files', {\n        replicaFile,\n        error: String(e),\n      });\n    }\n    logRestoreDirectoryContents(lc, replicaFile, 'after-temp-cleanup');\n    litestreamRestoreAttempts().add(1, {...attrs, result});\n  }\n}\n\n// From litestream/cmd/litestream/restore.go\nconst restoreForkOutputSchema = v.object({\n  fork: v\n    .object({\n      forked: v.boolean(),\n      txid: v.string().optional(),\n    })\n    .optional(),\n});\n\ntype RestoreForkOutput = v.Infer<typeof restoreForkOutputSchema>;\n\n/**\n * Whether the `-json` output of `litestream restore -fork-to-url` reports that\n * the backup was forked. A restore plan that cannot fork the backup is\n * restored without forking.\n */\nfunction restoreForked(lc: LogContext, stdout: string): boolean {\n  const start = stdout.indexOf('{');\n  const end = stdout.lastIndexOf('}');\n  if (start < 0 || end < start) {\n    lc.warn?.('litestream restore did not report a fork result');\n    return false;\n  }\n  let output: RestoreForkOutput;\n  try {\n    output = v.parse(\n      JSON.parse(stdout.slice(start, end + 1)),\n      restoreForkOutputSchema,\n      'passthrough',\n    );\n  } catch (e) {\n    lc.warn?.('unable to parse the litestream restore result', e);\n    return false;\n  }\n  const forked = output.fork?.forked === true;\n  lc.info?.(\n    forked\n      ? `forked the backup at txid ${output.fork?.txid}`\n      : 'restored without forking the backup',\n  );\n  return forked;\n}\n\n/**\n * Deletes litestream's local state for `replicaFile` (its default meta\n * directory, `.<file>-litestream`).\n */\nexport function deleteLitestreamMetaDir(replicaFile: string) {\n  rmSync(join(dirname(replicaFile), `.${basename(replicaFile)}-litestream`), {\n    recursive: true,\n    force: true,\n  });\n}\n\nfunction deleteRestoreTempFiles(replicaFile: string) {\n  const temporaryReplicaFile = `${replicaFile}.tmp`;\n  deleteLiteDB(temporaryReplicaFile);\n\n  // rocicorp/litestream zero@v0.0.9 downloads WAL indexes in parallel to\n  // `<output>.tmp-<8 lowercase hex digits>-wal` before applying them.\n  const directory = dirname(temporaryReplicaFile);\n  if (!existsSync(directory)) {\n    return;\n  }\n  const prefix = `${basename(temporaryReplicaFile)}-`;\n  for (const name of readdirSync(directory)) {\n    if (!name.startsWith(prefix) || !name.endsWith('-wal')) {\n      continue;\n    }\n    const index = name.slice(prefix.length, -'-wal'.length);\n    if (LEGACY_RESTORE_WAL_INDEX.test(index)) {\n      rmSync(join(directory, name), {force: true});\n    }\n  }\n}\n\nfunction logRestoreDirectoryContents(\n  lc: LogContext,\n  replicaFile: string,\n  phase: 'before-temp-cleanup' | 'after-temp-cleanup',\n) {\n  const directory = dirname(replicaFile);\n  try {\n    const names = readdirSync(directory).sort();\n    const entries = names\n      .slice(0, MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES)\n      .map(name => ({name, sizeBytes: statSync(join(directory, name)).size}));\n    lc.info?.('Litestream restore directory contents', {\n      phase,\n      directory,\n      replicaFile,\n      entryCount: names.length,\n      omittedEntryCount: Math.max(\n        0,\n        names.length - MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES,\n      ),\n      entries,\n    });\n  } catch (e) {\n    lc.warn?.('Unable to inspect Litestream restore directory', {\n      phase,\n      directory,\n      replicaFile,\n      error: String(e),\n    });\n  }\n}\n\n/**\n * Opens and validates the restored replica. Returns the open connection if\n * the replica is valid, which the caller must close; otherwise closes it and\n * returns `undefined`.\n */\nfunction openValidReplica(\n  lc: LogContext,\n  replica: string,\n  constraints: ReplicaConstraints | undefined,\n): Database | undefined {\n  let db: Database | undefined;\n  try {\n    // Note: Open the database and read the subscription state as a\n    // sanity / corruption check, even if there are no constraints.\n    db = new Database(lc, replica);\n    const {replicaVersion, watermark} = getSubscriptionState(\n      new StatementRunner(db),\n    );\n    if (constraints) {\n      if (replicaVersion !== constraints.replicaVersion) {\n        lc.warn?.(\n          `Local replica version ${replicaVersion} does not match expected replicaVersion ${constraints.replicaVersion}`,\n          constraints,\n        );\n        return undefined;\n      }\n      if (watermark < constraints.minWatermark) {\n        lc.warn?.(\n          `Local replica watermark ${watermark} is earlier than minWatermark ${constraints.minWatermark}`,\n        );\n        return undefined;\n      }\n      lc.info?.(\n        `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible`,\n        constraints,\n      );\n    }\n    const valid = db;\n    db = undefined; // owned by the caller\n    return valid;\n  } catch (e) {\n    if (isSQLiteCorruption(e)) {\n      logSQLiteCorruptionDiagnostics(lc, 'restored replica', replica, e);\n    }\n    lc.error?.('Error while validating restored replica', e);\n    return undefined;\n  } finally {\n    db?.close();\n  }\n}\n\nexport function startReplicaBackupProcess(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n): ChildProcess {\n  const {litestream, env} = getLitestream('replicate', config, replicaFile);\n  const attrs = litestreamBackupProcessMetricAttrs(config);\n  lc.info?.(`starting litestream backup to ${backupDestinationURL(config)}`);\n  const start = performance.now();\n  const proc = spawn(litestream, ['replicate'], {\n    env,\n    stdio: 'inherit',\n    windowsHide: true,\n  });\n  let recorded = false;\n  const record = (result: 'success' | 'error' | 'stopped') => {\n    if (recorded) {\n      return;\n    }\n    recorded = true;\n    const labels = {...attrs, result};\n    litestreamBackupProcessRuns().add(1, labels);\n    litestreamBackupProcessDuration().recordMs(\n      performance.now() - start,\n      labels,\n    );\n  };\n  proc.on('error', e => {\n    lc.warn?.(`litestream backup process error`, e);\n    record('error');\n  });\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      lc.info?.(`litestream backup process stopped`, {signal});\n      record('stopped');\n    } else if (code === 0) {\n      record('success');\n    } else {\n      lc.warn?.(`litestream backup process exited with code ${code}`);\n      record('error');\n    }\n  });\n  return proc;\n}\n\n// Listing the backup state requires a few S3 LIST requests, which should\n// normally complete well within this timeout.\nconst LIST_BACKUP_TIMEOUT_MS = 30_000;\n\nconst wsRe = /\\s+/;\n\n/**\n * Returns the time of the most recent object (snapshot or WAL segment)\n * actually uploaded to the backup replica destination, as listed by the\n * bundled litestream CLI (`litestream snapshots` / `litestream wal`).\n *\n * This queries the replica destination (e.g. S3) directly, and thus serves\n * as a source of truth for backup durability. This is in contrast to the\n * `litestream_replica_progress` metric, which is exported when litestream\n * *believes* an upload has succeeded, and has been observed to advance even\n * when nothing is actually written to the destination.\n *\n * Rejects if the backup state cannot be determined (spawn error, non-zero\n * exit, timeout, or empty/unparseable listing).\n */\nexport async function getLastBackupTime(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n): Promise<Date> {\n  const [snapshots, wal] = await Promise.all([\n    listBackupCreatedTimes(lc, config, replicaFile, 'snapshots'),\n    listBackupCreatedTimes(lc, config, replicaFile, 'wal'),\n  ]);\n  const times = [...snapshots, ...wal];\n  if (times.length === 0) {\n    // Note: the litestream CLI exits with code 0 and logs listing errors\n    // (e.g. S3 failures) to stderr, so an empty listing cannot be\n    // distinguished from a failed one. Since a valid backup always contains\n    // at least one snapshot, an empty listing is treated as a failure.\n    throw new Error(\n      `no snapshots or WAL segments listed at ${backupDestinationURL(config)}`,\n    );\n  }\n  return new Date(Math.max(...times.map(time => time.getTime())));\n}\n\n/**\n * Runs `litestream <snapshots|wal> <replica-file>` with the same config /\n * environment used by the `litestream replicate` process (so that the\n * backupURL, endpoint, region, and credentials are identical), and parses\n * the `created` column (RFC3339) of the tab-formatted output, e.g.:\n *\n * ```\n * replica  generation        index  size     created\n * s3       1862f44967b3863f  0      4546445  2026-06-10T01:11:32Z\n * ```\n */\nasync function listBackupCreatedTimes(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  command: 'snapshots' | 'wal',\n): Promise<Date[]> {\n  const start = performance.now();\n  let result: 'success' | 'empty' | 'timeout' | 'error' = 'error';\n  const {litestream, env} = getLitestream('replicate', config, replicaFile);\n  const proc = spawn(litestream, [command, replicaFile], {\n    env,\n    stdio: ['ignore', 'pipe', 'inherit'],\n    windowsHide: true,\n  });\n  const {promise, resolve, reject} = resolver<string>();\n  let stdout = '';\n  proc.stdout.setEncoding('utf-8');\n  proc.stdout.on('data', chunk => (stdout += chunk));\n  proc.on('error', reject);\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      reject(new Error(`litestream ${command} killed with ${signal}`));\n    } else if (code !== 0) {\n      reject(new Error(`litestream ${command} exited with code ${code}`));\n    } else {\n      resolve(stdout);\n    }\n  });\n  const timeout = setTimeout(() => {\n    result = 'timeout';\n    reject(new Error(`timed out listing backup state (litestream ${command})`));\n    proc.kill('SIGKILL');\n  }, LIST_BACKUP_TIMEOUT_MS);\n\n  try {\n    const output = await promise;\n    const times = parseBackupCreatedTimes(lc, command, output);\n    result = times.length ? 'success' : 'empty';\n    return times;\n  } finally {\n    clearTimeout(timeout);\n    litestreamBackupListDuration().recordMs(performance.now() - start, {\n      ...litestreamBackupMetricAttrs(config),\n      command,\n      result,\n    });\n  }\n}\n\n/**\n * Parses the `created` column (the last, RFC3339-formatted column) from the\n * tab-formatted output of `litestream snapshots` / `litestream wal`. The\n * header row and any unparseable lines are skipped.\n *\n * Exported for testing.\n */\nexport function parseBackupCreatedTimes(\n  lc: LogContext,\n  command: 'snapshots' | 'wal',\n  output: string,\n): Date[] {\n  const times: Date[] = [];\n  for (const line of output.split('\\n')) {\n    const cols = line.trim().split(wsRe);\n    const created = cols.at(-1);\n    if (\n      cols.length < 2 ||\n      created === undefined ||\n      created === 'created' /* header row */\n    ) {\n      continue;\n    }\n    const time = new Date(created);\n    if (Number.isNaN(time.getTime())) {\n      lc.warn?.(`unexpected line in litestream ${command} output: ${line}`);\n      continue;\n    }\n    times.push(time);\n  }\n  return times;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAwEA,IAAM,uCAAuC;AAC7C,IAAM,2BAA2B;;;;;;AAejC,SAAgB,qBACd,QACoB;CACpB,OAAO,OAAO,qBAAqB,OAAO;AAC5C;AAEA,SAAS,cACP,MACA,QACA,aACA,kBAIA;CACA,MAAM,EACJ,YACA,cACA,gBACA,eACA,UACA,YACA,cACA,UACA,QACA,MACA,uBAGA,yBAAyB,wBAAwB,KACjD,yBAAyB,yBAAyB,IAClD,4BACA,kCACA,kCACA,6BACA,sBACA,kBACE;CAEJ,MAAM,KACH,SAAS,aAAa,kBACtB,SAAS,eAAe;CAC3B,MAAM,cACH,KAAK,eAAe,eACrB,KAAK,YAAY,iCAAiC;CACpD,MAAM,mBAAmB,KAAK,eAAe;CAG7C,MAAM,gBAAgB,6BAA6B,KAAK;CAExD,OAAO;EACL;EACA,KAAK;GACH,GAAG,QAAQ;IACV,sBAAsB;IACtB,+BAA+B,KAAK,qBAAqB,MAAM,CAAC;IAChE,8CAA8C,OAC7C,sBACF;IACC,8CAA8C,OAC7C,sBACF;IACC,wDAAwD,OACvD,gCACF;IACC,wDAAwD,OACvD,gCACF;IAEC,6CAA6C,OAK5C,IAAI,gCACN;IACC,oCAAoC,OAAO,aAAa;IACxD,8BAA8B,oBAAoB;IAClD,mDAAmD,OAClD,2BACF;IACC,sDAAsD,OACrD,8BAA8B,CAChC;IACC,0CAA0C,OAAO,oBAAoB;IACrE,mCAAmC,OAAO,aAAa;IACvD,oBAAoB;IACpB,gCAAgC,qBAAqB,WAAW;IAChE,sBAAsB;IACtB,oBAAoB,OAAO,IAAI;GAChC,GAAI,WAAW,GAAE,6BAA6B,SAAQ,IAAI,CAAC;GAC3D,GAAI,SAAS,GAAE,2BAA2B,OAAM,IAAI,CAAC;IACpD,oCAAoC;EACvC;CACF;AACF;AAEA,eAAsB,WACpB,IACA,QACA,aACA,oBACA,MACA,QACA,YACyB;CACzB,MAAM,EAAC,gBAAgB,GAAG,YAAW,MAAM,YACzC,IACA,QACA,aACA,oBACA,MACA,QACA,UACF;CACA,IAAI,QAAQ,WAAW,qBAAqB,gBAAgB;EAK1D,MAAM,EAAC,gBAAgB,GAAG,GAAG,UAAS,MAAM,YAC1C,IACA,QACA,aACA,oBACA,MACA,QACA,UACF;EACA,OAAO;CACT;CACA,OAAO;AACT;AAEA,eAAe,YACb,IACA,QACA,aACA,oBACA,MACA,QACA,YACqD;CACrD,MAAM,EAAC,kBAAiB;CACxB,MAAM,YAAY,qBAAqB,MAAM;CAC7C,IAAI,cAAc,CAAC,eACjB,aAAa,KAAA;CAEf,MAAM,QAAQ,6BAA6B,QAAQ,MAAM,SAAS;CAClE,IAAI,SAAwB;CAC5B,IAAI;EACF,4BAA4B,IAAI,aAAa,qBAAqB;EAClE,uBAAuB,WAAW;EAClC,MAAM,iBAAiB,WAAW,WAAW;EAC7C,IAAI,CAAC,gBAIH,wBAAwB,WAAW;EAErC,MAAM,EAAC,YAAY,QAAO,cACxB,WACA,QACA,aACA,OACF;EACA,MAAM,EACJ,oBAAoB,aACpB,sBACA,kBACE;EACJ,GAAG,OAAO,+BAA+B;GACvC,oBAAoB;GACpB;GACA;EACF,CAAC;EAOD,MAAM,OAAO,MACX,YACA;GACE;GACA;GACA;GACA;GACA,OAAO,WAAW;GAClB,GAAI,aAAa;IAAC;IAAS;IAAgB,WAAW;GAAG,IAAI,CAAC;GAC9D;EACF,GACA;GAAC;GAAK,OAAO;IAAC;IAAU;IAAQ;GAAM;GAAG,aAAa;GAAM;EAAM,CACpE;EACA,IAAI,SAAS;EACb,IAAI,SAAS;EACb,KAAK,OAAO,YAAY,OAAO;EAC/B,KAAK,OAAO,YAAY,OAAO;EAG/B,KAAK,OAAO,GAAG,SAAQ,UAAS;GAC9B,QAAQ,OAAO,MAAM,KAAK;GAC1B,UAAU;EACZ,CAAC;EACD,KAAK,OAAO,GAAG,SAAQ,UAAS;GAC9B,QAAQ,OAAO,MAAM,KAAK;GAC1B,UAAU;EACZ,CAAC;EACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAS;EAC5C,KAAK,GAAG,SAAS,MAAM;EACvB,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,IAAI,QACF,OAAO,0BAA0B,QAAQ;QACpC,IAAI,SAAS,GAClB,OAAO,+BAA+B,MAAM;QAE5C,QAAQ;EAEZ,CAAC;EACD,MAAM,eAAe,YAAY,IAAI;EACrC,IAAI;GACF,MAAM;GACN,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAS,CAC9B;EACF,SAAS,GAAG;GACV,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAO,CAC5B;GACA,MAAM,SAAS,CAAC,QAAQ,MAAM,EAC3B,KAAI,UAAS,MAAM,KAAK,CAAC,EACzB,OAAO,OAAO,EACd,KAAK,IAAI;GACZ,MAAM,IAAI,MAAM,SAAS,GAAG,OAAO,CAAC,EAAE,IAAI,WAAW,OAAO,CAAC,GAAG,EAC9D,OAAO,EACT,CAAC;EACH;EACA,IAAI,CAAC,WAAW,WAAW,GAAG;GAC5B,SAAS;GACT,OAAO;IAAC,UAAU;IAAO;IAAW;IAAQ;GAAc;EAC5D;EACA,MAAM,kBAAkB,YAAY,IAAI;EAKxC,IAAI,KAAK,iBAAiB,IAAI,aAAa,kBAAkB;EAC7D,oCAAoC,EAAE,SACpC,YAAY,IAAI,IAAI,iBACpB;GAAC,GAAG;GAAO,QAAQ,KAAK,YAAY;EAAiB,CACvD;EACA,IAAI,CAAC,IAAI;GACP,SAAS;GACT,GAAG,OAAO,6CAA6C;GACvD,aAAa,WAAW;GACxB,kBAAkB,WAAW;GAG7B,wBAAwB,WAAW;GACnC,OAAO;IAAC,UAAU;IAAO;IAAW;IAAQ;GAAc;EAC5D;EACA,IAAI;GACF,SAAS;GACT,IAAI,CAAC,gBAAgB;IAYnB,kBAAkB,WAAW;IAC7B,0BAA0B,EAAE,IAAI,SAAS,WAAW,EAAE,MAAM;KAC1D,GAAG;KACH,QAAQ;IACV,CAAC;GACH;GAEA,MAAM,YACJ,cAAc,CAAC,kBAAkB,cAAc,IAAI,MAAM,IACrD,IAAI,UAAU,IAAI,EAAE,IACpB,KAAA;GACN,IAAI,WAAW;IACb,KAAK,KAAA;IACL,YAAY,SAAS,UAAU,oBAC7B,UAAU,QAAQ,uBAAuB,CAC3C;GACF;GACA,OAAO;IAAC,UAAU;IAAM;IAAW;IAAQ;IAAgB;GAAS;EACtE,UAAU;GACR,IAAI,MAAM;EACZ;CACF,UAAU;EACR,IAAI;GACF,uBAAuB,WAAW;EACpC,SAAS,GAAG;GACV,GAAG,OAAO,yDAAyD;IACjE;IACA,OAAO,OAAO,CAAC;GACjB,CAAC;EACH;EACA,4BAA4B,IAAI,aAAa,oBAAoB;EACjE,0BAA0B,EAAE,IAAI,GAAG;GAAC,GAAG;GAAO;EAAM,CAAC;CACvD;AACF;AAGA,IAAM,0BAA0B,eAAE,OAAO,EACvC,MAAM,eACH,OAAO;CACN,QAAQ,eAAE,QAAQ;CAClB,MAAM,eAAE,OAAO,EAAE,SAAS;AAC5B,CAAC,EACA,SAAS,EACd,CAAC;;;;;;AASD,SAAS,cAAc,IAAgB,QAAyB;CAC9D,MAAM,QAAQ,OAAO,QAAQ,GAAG;CAChC,MAAM,MAAM,OAAO,YAAY,GAAG;CAClC,IAAI,QAAQ,KAAK,MAAM,OAAO;EAC5B,GAAG,OAAO,iDAAiD;EAC3D,OAAO;CACT;CACA,IAAI;CACJ,IAAI;EACF,SAAS,MACP,KAAK,MAAM,OAAO,MAAM,OAAO,MAAM,CAAC,CAAC,GACvC,yBACA,aACF;CACF,SAAS,GAAG;EACV,GAAG,OAAO,iDAAiD,CAAC;EAC5D,OAAO;CACT;CACA,MAAM,SAAS,OAAO,MAAM,WAAW;CACvC,GAAG,OACD,SACI,6BAA6B,OAAO,MAAM,SAC1C,qCACN;CACA,OAAO;AACT;;;;;AAMA,SAAgB,wBAAwB,aAAqB;CAC3D,OAAO,KAAK,QAAQ,WAAW,GAAG,IAAI,SAAS,WAAW,EAAE,YAAY,GAAG;EACzE,WAAW;EACX,OAAO;CACT,CAAC;AACH;AAEA,SAAS,uBAAuB,aAAqB;CACnD,MAAM,uBAAuB,GAAG,YAAY;CAC5C,aAAa,oBAAoB;CAIjC,MAAM,YAAY,QAAQ,oBAAoB;CAC9C,IAAI,CAAC,WAAW,SAAS,GACvB;CAEF,MAAM,SAAS,GAAG,SAAS,oBAAoB,EAAE;CACjD,KAAK,MAAM,QAAQ,YAAY,SAAS,GAAG;EACzC,IAAI,CAAC,KAAK,WAAW,MAAM,KAAK,CAAC,KAAK,SAAS,MAAM,GACnD;EAEF,MAAM,QAAQ,KAAK,MAAM,OAAO,QAAQ,EAAc;EACtD,IAAI,yBAAyB,KAAK,KAAK,GACrC,OAAO,KAAK,WAAW,IAAI,GAAG,EAAC,OAAO,KAAI,CAAC;CAE/C;AACF;AAEA,SAAS,4BACP,IACA,aACA,OACA;CACA,MAAM,YAAY,QAAQ,WAAW;CACrC,IAAI;EACF,MAAM,QAAQ,YAAY,SAAS,EAAE,KAAK;EAC1C,MAAM,UAAU,MACb,MAAM,GAAG,oCAAoC,EAC7C,KAAI,UAAS;GAAC;GAAM,WAAW,SAAS,KAAK,WAAW,IAAI,CAAC,EAAE;EAAI,EAAE;EACxE,GAAG,OAAO,yCAAyC;GACjD;GACA;GACA;GACA,YAAY,MAAM;GAClB,mBAAmB,KAAK,IACtB,GACA,MAAM,SAAS,oCACjB;GACA;EACF,CAAC;CACH,SAAS,GAAG;EACV,GAAG,OAAO,kDAAkD;GAC1D;GACA;GACA;GACA,OAAO,OAAO,CAAC;EACjB,CAAC;CACH;AACF;;;;;;AAOA,SAAS,iBACP,IACA,SACA,aACsB;CACtB,IAAI;CACJ,IAAI;EAGF,KAAK,IAAI,SAAS,IAAI,OAAO;EAC7B,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,EAAE,CACxB;EACA,IAAI,aAAa;GACf,IAAI,mBAAmB,YAAY,gBAAgB;IACjD,GAAG,OACD,yBAAyB,eAAe,0CAA0C,YAAY,kBAC9F,WACF;IACA;GACF;GACA,IAAI,YAAY,YAAY,cAAc;IACxC,GAAG,OACD,2BAA2B,UAAU,gCAAgC,YAAY,cACnF;IACA;GACF;GACA,GAAG,OACD,4BAA4B,eAAe,iBAAiB,UAAU,iBACtE,WACF;EACF;EACA,MAAM,QAAQ;EACd,KAAK,KAAA;EACL,OAAO;CACT,SAAS,GAAG;EACV,IAAI,mBAAmB,CAAC,GACtB,+BAA+B,IAAI,oBAAoB,SAAS,CAAC;EAEnE,GAAG,QAAQ,2CAA2C,CAAC;EACvD;CACF,UAAU;EACR,IAAI,MAAM;CACZ;AACF;AAEA,SAAgB,0BACd,IACA,QACA,aACc;CACd,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,QAAQ,mCAAmC,MAAM;CACvD,GAAG,OAAO,iCAAiC,qBAAqB,MAAM,GAAG;CACzE,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,OAAO,MAAM,YAAY,CAAC,WAAW,GAAG;EAC5C;EACA,OAAO;EACP,aAAa;CACf,CAAC;CACD,IAAI,WAAW;CACf,MAAM,UAAU,WAA4C;EAC1D,IAAI,UACF;EAEF,WAAW;EACX,MAAM,SAAS;GAAC,GAAG;GAAO;EAAM;EAChC,4BAA4B,EAAE,IAAI,GAAG,MAAM;EAC3C,gCAAgC,EAAE,SAChC,YAAY,IAAI,IAAI,OACpB,MACF;CACF;CACA,KAAK,GAAG,UAAS,MAAK;EACpB,GAAG,OAAO,mCAAmC,CAAC;EAC9C,OAAO,OAAO;CAChB,CAAC;CACD,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QAAQ;GACV,GAAG,OAAO,qCAAqC,EAAC,OAAM,CAAC;GACvD,OAAO,SAAS;EAClB,OAAO,IAAI,SAAS,GAClB,OAAO,SAAS;OACX;GACL,GAAG,OAAO,8CAA8C,MAAM;GAC9D,OAAO,OAAO;EAChB;CACF,CAAC;CACD,OAAO;AACT;AAIA,IAAM,yBAAyB;AAE/B,IAAM,OAAO;;;;;;;;;;;;;;;AAgBb,eAAsB,kBACpB,IACA,QACA,aACe;CACf,MAAM,CAAC,WAAW,OAAO,MAAM,QAAQ,IAAI,CACzC,uBAAuB,IAAI,QAAQ,aAAa,WAAW,GAC3D,uBAAuB,IAAI,QAAQ,aAAa,KAAK,CACvD,CAAC;CACD,MAAM,QAAQ,CAAC,GAAG,WAAW,GAAG,GAAG;CACnC,IAAI,MAAM,WAAW,GAKnB,MAAM,IAAI,MACR,0CAA0C,qBAAqB,MAAM,GACvE;CAEF,OAAO,IAAI,KAAK,KAAK,IAAI,GAAG,MAAM,KAAI,SAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC;AAChE;;;;;;;;;;;;AAaA,eAAe,uBACb,IACA,QACA,aACA,SACiB;CACjB,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI,SAAoD;CACxD,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,OAAO,MAAM,YAAY,CAAC,SAAS,WAAW,GAAG;EACrD;EACA,OAAO;GAAC;GAAU;GAAQ;EAAS;EACnC,aAAa;CACf,CAAC;CACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAiB;CACpD,IAAI,SAAS;CACb,KAAK,OAAO,YAAY,OAAO;CAC/B,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;CACjD,KAAK,GAAG,SAAS,MAAM;CACvB,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QACF,uBAAO,IAAI,MAAM,cAAc,QAAQ,eAAe,QAAQ,CAAC;OAC1D,IAAI,SAAS,GAClB,uBAAO,IAAI,MAAM,cAAc,QAAQ,oBAAoB,MAAM,CAAC;OAElE,QAAQ,MAAM;CAElB,CAAC;CACD,MAAM,UAAU,iBAAiB;EAC/B,SAAS;EACT,uBAAO,IAAI,MAAM,8CAA8C,QAAQ,EAAE,CAAC;EAC1E,KAAK,KAAK,SAAS;CACrB,GAAG,sBAAsB;CAEzB,IAAI;EAEF,MAAM,QAAQ,wBAAwB,IAAI,SAAS,MAD9B,OACoC;EACzD,SAAS,MAAM,SAAS,YAAY;EACpC,OAAO;CACT,UAAU;EACR,aAAa,OAAO;EACpB,6BAA6B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO;GACjE,GAAG,4BAA4B,MAAM;GACrC;GACA;EACF,CAAC;CACH;AACF;;;;;;;;AASA,SAAgB,wBACd,IACA,SACA,QACQ;CACR,MAAM,QAAgB,CAAC;CACvB,KAAK,MAAM,QAAQ,OAAO,MAAM,IAAI,GAAG;EACrC,MAAM,OAAO,KAAK,KAAK,EAAE,MAAM,IAAI;EACnC,MAAM,UAAU,KAAK,GAAG,EAAE;EAC1B,IACE,KAAK,SAAS,KACd,YAAY,KAAA,KACZ,YAAY,WAEZ;EAEF,MAAM,OAAO,IAAI,KAAK,OAAO;EAC7B,IAAI,OAAO,MAAM,KAAK,QAAQ,CAAC,GAAG;GAChC,GAAG,OAAO,iCAAiC,QAAQ,WAAW,MAAM;GACpE;EACF;EACA,MAAM,KAAK,IAAI;CACjB;CACA,OAAO;AACT"}
+{"version":3,"file":"commands.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync, readdirSync, rmSync, statSync} from 'node:fs';\nimport {basename, dirname, join} from 'node:path';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../../shared/src/must.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {type LitestreamConfig} from '../../config/normalize.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {\n  isSQLiteCorruption,\n  logSQLiteCorruptionDiagnostics,\n} from '../../db/sqlite-corruption.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {deleteChangeLogDB} from '../replicator/change-log-db.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {litestreamSocketPath} from './litestream-controller.ts';\nimport {\n  litestreamBackupListDuration,\n  litestreamBackupMetricAttrs,\n  litestreamBackupProcessDuration,\n  litestreamBackupProcessMetricAttrs,\n  litestreamBackupProcessRuns,\n  litestreamRestoreAttempts,\n  litestreamRestoredDbBytes,\n  litestreamRestoreMetricAttrs,\n  litestreamRestoreProcessDuration,\n  litestreamRestoreValidationDuration,\n  type LitestreamRole,\n} from './metrics.ts';\n\nexport type ReplicaConstraints = {\n  replicaVersion: string;\n  minWatermark: string;\n};\n\nexport type RestoreResult =\n  | 'success'\n  | 'no_backup'\n  | 'invalid_replica'\n  | 'error';\n\nexport type RestoreAttempt = {\n  restored: boolean;\n  backupURL: string | undefined;\n  result: RestoreResult;\n  /**\n   * Set when the restore forked the backup (see {@link ForkBackup}), which\n   * then continues from the restored replica's WAL. The WAL is kept when the\n   * last connection closes (see `Database.persistWal()`).\n   */\n  forked?: boolean | undefined;\n};\n\n/**\n * Prepares the restored replica to continue from a forked backup\n * instead of starting a new lineage with a full snapshot.\n */\nexport type ForkBackup = {\n  /** The restore also forks its backup into the (new, empty) one at this URL. */\n  url: string;\n};\n\nconst MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES = 100;\nconst LEGACY_RESTORE_WAL_INDEX = /^[0-9a-f]{8}$/;\n\nexport class BackupNotFoundException extends Error {\n  static readonly name = 'BackupNotFoundException';\n\n  constructor(backupURL: string | undefined) {\n    super(`backup not found at ${backupURL}`);\n  }\n}\n\n/**\n * The effective backup URL: the `backupURLOverride` if present (i.e. a\n * lineage-specific location computed by the replication-manager), or\n * otherwise the configured `backupURL`.\n */\nexport function backupDestinationURL(\n  config: Pick<LitestreamConfig, 'backupURL' | 'backupURLOverride'>,\n): string | undefined {\n  return config.backupURLOverride ?? config.backupURL;\n}\n\nfunction getLitestream(\n  mode: 'restore' | 'replicate',\n  config: LitestreamConfig,\n  replicaFile: string,\n  logLevelOverride?: LogLevel,\n): {\n  litestream: string;\n  env: NodeJS.ProcessEnv;\n} {\n  const {\n    executable,\n    executableV5,\n    restoreUsingV5,\n    backupUsingV5,\n    logLevel,\n    configPath,\n    configPathV5,\n    endpoint,\n    region,\n    port,\n    checkpointThresholdMB,\n    // Note: This assumes the default page size. If page size configuration\n    // is added, this computation must be adjusted accordingly.\n    minCheckpointPageCount = checkpointThresholdMB * 256, // 1 MiB / 4 KiB page size\n    maxCheckpointPageCount = minCheckpointPageCount * 10,\n    forceCheckpointThresholdMB,\n    incrementalBackupIntervalMinutes,\n    incrementalBackupIntervalSeconds,\n    snapshotBackupIntervalHours,\n    multipartConcurrency,\n    multipartSize,\n  } = config;\n\n  const v5 =\n    (mode === 'restore' && restoreUsingV5) ||\n    (mode === 'replicate' && backupUsingV5);\n  const litestream =\n    (v5 ? executableV5 : executable) ??\n    must(executable, `Missing --litestream-executable`);\n  const litestreamConfig = v5 ? configPathV5 : configPath;\n  // Disable truncate-page-n if forced checkpoints are enabled,\n  // and otherwise use litestream's default.\n  const truncatePageN = forceCheckpointThresholdMB ? -1 : 121359;\n\n  return {\n    litestream,\n    env: {\n      ...process.env,\n      ['ZERO_REPLICA_FILE']: replicaFile,\n      ['ZERO_LITESTREAM_BACKUP_URL']: must(backupDestinationURL(config)),\n      ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n        minCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n        maxCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n        incrementalBackupIntervalMinutes, // v3 only\n      ),\n      ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_SECONDS']: String(\n        incrementalBackupIntervalSeconds, // v5 only\n      ),\n      // v5 only\n      ['ZERO_LITESTREAM_MONITOR_INTERVAL_SECONDS']: String(\n        // zero-cache explicitly requests syncs at the incremental backup\n        // interval (see LitestreamSyncRequester), and litestream's\n        // own monitor becomes a backstop that keeps backups and checkpoints\n        // progressing if zero-cache stops requesting them.\n        2 * incrementalBackupIntervalSeconds,\n      ),\n      ['ZERO_LITESTREAM_TRUNCATE_PAGE_N']: String(truncatePageN),\n      ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n      ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_HOURS']: String(\n        snapshotBackupIntervalHours,\n      ),\n      ['ZERO_LITESTREAM_SNAPSHOT_RETENTION_INTERVAL_HOURS']: String(\n        snapshotBackupIntervalHours + 6, // delete old snapshots after 6 hours\n      ),\n      ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n      ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n      ['ZERO_LOG_FORMAT']: 'json',\n      ['ZERO_LITESTREAM_SOCKET_PATH']: litestreamSocketPath(replicaFile),\n      ['LITESTREAM_CONFIG']: litestreamConfig,\n      ['LITESTREAM_PORT']: String(port),\n      ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n      ...(region ? {['ZERO_LITESTREAM_REGION']: region} : {}),\n      ['LITESTREAM_COMPACTION_SERIALIZE']: 'false',\n    },\n  };\n}\n\nexport async function tryRestore(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  replicaConstraints: ReplicaConstraints | undefined,\n  role: LitestreamRole,\n  signal?: AbortSignal,\n  forkBackup?: ForkBackup | undefined,\n): Promise<RestoreAttempt> {\n  const {reusedExisting, ...attempt} = await restoreOnce(\n    lc,\n    config,\n    replicaFile,\n    replicaConstraints,\n    role,\n    signal,\n    forkBackup,\n  );\n  if (attempt.result === 'invalid_replica' && reusedExisting) {\n    // `-if-db-not-exists` made the restore a no-op, and the existing local\n    // replica (e.g. one left behind by a failed initial sync of a different\n    // generation) was invalid. It has since been deleted, so restore from the\n    // backup.\n    const {reusedExisting: _, ...retry} = await restoreOnce(\n      lc,\n      config,\n      replicaFile,\n      replicaConstraints,\n      role,\n      signal,\n      forkBackup,\n    );\n    return retry;\n  }\n  return attempt;\n}\n\nasync function restoreOnce(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  replicaConstraints: ReplicaConstraints | undefined,\n  role: LitestreamRole,\n  signal: AbortSignal | undefined,\n  forkBackup: ForkBackup | undefined,\n): Promise<RestoreAttempt & {reusedExisting: boolean}> {\n  const {backupUsingV5} = config;\n  const backupURL = backupDestinationURL(config);\n  if (forkBackup && !backupUsingV5) {\n    forkBackup = undefined; // Ignore forkBackup if v5 is disabled.\n  }\n  const attrs = litestreamRestoreMetricAttrs(config, role, backupURL);\n  let result: RestoreResult = 'error';\n  try {\n    logRestoreDirectoryContents(lc, replicaFile, 'before-temp-cleanup');\n    deleteRestoreTempFiles(replicaFile);\n    const reusedExisting = existsSync(replicaFile);\n    if (!reusedExisting) {\n      // The local litestream state (and anything recorded in it) describes a\n      // replica that no longer exists. Discard it before the restore, which\n      // (if it seeds a backup) writes the state that belongs to the new one.\n      deleteLitestreamMetaDir(replicaFile);\n    }\n    const {litestream, env} = getLitestream(\n      'restore',\n      config,\n      replicaFile,\n      'debug', // Include all output from `litestream restore`, as it's minimal.\n    );\n    const {\n      restoreParallelism: parallelism,\n      multipartConcurrency,\n      multipartSize,\n    } = config;\n    lc.info?.(`starting litestream restore`, {\n      restoreParallelism: parallelism,\n      multipartConcurrency,\n      multipartSize,\n    });\n    // Pipe (rather than inherit) litestream's stdout/stderr so that its own\n    // `\"level\":\"ERROR\"` output on a failed restore does not go straight to the\n    // pod's stdout — where a log-scraper alert would page on it — before our\n    // code has decided how to handle the failure. The captured output is\n    // included in the thrown error so the caller can log the final outcome.\n    // See INC-961.\n    const proc = spawn(\n      litestream,\n      [\n        'restore',\n        '-if-db-not-exists',\n        '-if-replica-exists',\n        '-parallelism',\n        String(parallelism),\n        ...(forkBackup ? ['-json', '-fork-to-url', forkBackup.url] : []),\n        replicaFile,\n      ],\n      {env, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true, signal},\n    );\n    let stdout = '';\n    let stderr = '';\n    proc.stdout.setEncoding('utf-8');\n    proc.stderr.setEncoding('utf-8');\n    // Output AND capture the stdout and stderr, the former for observability\n    // and the latter for error handling.\n    proc.stdout.on('data', chunk => {\n      process.stdout.write(chunk);\n      stdout += chunk;\n    });\n    proc.stderr.on('data', chunk => {\n      process.stderr.write(chunk);\n      stderr += chunk;\n    });\n    const {promise, resolve, reject} = resolver();\n    proc.on('error', reject);\n    proc.on('close', (code, signal) => {\n      if (signal) {\n        reject(`litestream killed with ${signal}`);\n      } else if (code !== 0) {\n        reject(`litestream exited with code ${code}`);\n      } else {\n        resolve();\n      }\n    });\n    const processStart = performance.now();\n    try {\n      await promise;\n      litestreamRestoreProcessDuration().recordMs(\n        performance.now() - processStart,\n        {...attrs, result: 'success'},\n      );\n    } catch (e) {\n      litestreamRestoreProcessDuration().recordMs(\n        performance.now() - processStart,\n        {...attrs, result: 'error'},\n      );\n      const output = [stdout, stderr]\n        .map(value => value.trim())\n        .filter(Boolean)\n        .join('\\n');\n      throw new Error(output ? `${String(e)}\\n${output}` : String(e), {\n        cause: e,\n      });\n    }\n    if (!existsSync(replicaFile)) {\n      result = 'no_backup';\n      return {restored: false, backupURL, result, reusedExisting};\n    }\n    const validationStart = performance.now();\n    const db = openValidReplica(lc, replicaFile, replicaConstraints);\n    litestreamRestoreValidationDuration().recordMs(\n      performance.now() - validationStart,\n      {...attrs, result: db ? 'success' : 'invalid_replica'},\n    );\n    if (!db) {\n      result = 'invalid_replica';\n      lc.info?.(`Deleting local replica and retrying restore`);\n      deleteLiteDB(replicaFile);\n      deleteChangeLogDB(replicaFile);\n      // The litestream state (e.g. the anchor of a seeded backup) describes\n      // the deleted replica.\n      deleteLitestreamMetaDir(replicaFile);\n      return {restored: false, backupURL, result, reusedExisting};\n    }\n    try {\n      result = 'success';\n      if (!reusedExisting) {\n        // This restore materialized the replica file, so any change log beside it\n        // was written against a replica that is no longer there. Reconciliation\n        // is the safety net — a restored replica keeps its replicaVersion, so the\n        // rule falls through to truncate-or-reseed on head mismatch — but the log\n        // is a cache and deleting it here is the explicit statement that nothing\n        // in it survives the restore.\n        //\n        // A restore that reuses an existing replica (`-if-db-not-exists` made it\n        // a no-op) deliberately keeps the log: that is the process-restart path,\n        // where the log is still the one written beside this exact replica and\n        // discarding it would cost every reconnecting subscriber a `too-old`.\n        deleteChangeLogDB(replicaFile);\n        litestreamRestoredDbBytes().add(statSync(replicaFile).size, {\n          ...attrs,\n          result: 'success',\n        });\n      }\n      const forked =\n        forkBackup !== undefined &&\n        !reusedExisting &&\n        restoreForked(lc, stdout);\n      return {restored: true, backupURL, result, reusedExisting, forked};\n    } finally {\n      db.close();\n    }\n  } finally {\n    try {\n      deleteRestoreTempFiles(replicaFile);\n    } catch (e) {\n      lc.warn?.('Unable to clean up Litestream restore temporary files', {\n        replicaFile,\n        error: String(e),\n      });\n    }\n    logRestoreDirectoryContents(lc, replicaFile, 'after-temp-cleanup');\n    litestreamRestoreAttempts().add(1, {...attrs, result});\n  }\n}\n\n// From litestream/cmd/litestream/restore.go\nconst restoreForkOutputSchema = v.object({\n  fork: v\n    .object({\n      forked: v.boolean(),\n      txid: v.string().optional(),\n    })\n    .optional(),\n});\n\ntype RestoreForkOutput = v.Infer<typeof restoreForkOutputSchema>;\n\n/**\n * Whether the `-json` output of `litestream restore -fork-to-url` reports that\n * the backup was forked. A restore plan that cannot fork the backup is\n * restored without forking.\n */\nfunction restoreForked(lc: LogContext, stdout: string): boolean {\n  const start = stdout.indexOf('{');\n  const end = stdout.lastIndexOf('}');\n  if (start < 0 || end < start) {\n    lc.warn?.('litestream restore did not report a fork result');\n    return false;\n  }\n  let output: RestoreForkOutput;\n  try {\n    output = v.parse(\n      JSON.parse(stdout.slice(start, end + 1)),\n      restoreForkOutputSchema,\n      'passthrough',\n    );\n  } catch (e) {\n    lc.warn?.('unable to parse the litestream restore result', e);\n    return false;\n  }\n  const forked = output.fork?.forked === true;\n  lc.info?.(\n    forked\n      ? `forked the backup at txid ${output.fork?.txid}`\n      : 'restored without forking the backup',\n  );\n  return forked;\n}\n\n/**\n * Deletes litestream's local state for `replicaFile` (its default meta\n * directory, `.<file>-litestream`).\n */\nexport function deleteLitestreamMetaDir(replicaFile: string) {\n  rmSync(join(dirname(replicaFile), `.${basename(replicaFile)}-litestream`), {\n    recursive: true,\n    force: true,\n  });\n}\n\nfunction deleteRestoreTempFiles(replicaFile: string) {\n  const temporaryReplicaFile = `${replicaFile}.tmp`;\n  deleteLiteDB(temporaryReplicaFile);\n\n  // rocicorp/litestream zero@v0.0.9 downloads WAL indexes in parallel to\n  // `<output>.tmp-<8 lowercase hex digits>-wal` before applying them.\n  const directory = dirname(temporaryReplicaFile);\n  if (!existsSync(directory)) {\n    return;\n  }\n  const prefix = `${basename(temporaryReplicaFile)}-`;\n  for (const name of readdirSync(directory)) {\n    if (!name.startsWith(prefix) || !name.endsWith('-wal')) {\n      continue;\n    }\n    const index = name.slice(prefix.length, -'-wal'.length);\n    if (LEGACY_RESTORE_WAL_INDEX.test(index)) {\n      rmSync(join(directory, name), {force: true});\n    }\n  }\n}\n\nfunction logRestoreDirectoryContents(\n  lc: LogContext,\n  replicaFile: string,\n  phase: 'before-temp-cleanup' | 'after-temp-cleanup',\n) {\n  const directory = dirname(replicaFile);\n  try {\n    const names = readdirSync(directory).sort();\n    const entries = names\n      .slice(0, MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES)\n      .map(name => ({name, sizeBytes: statSync(join(directory, name)).size}));\n    lc.info?.('Litestream restore directory contents', {\n      phase,\n      directory,\n      replicaFile,\n      entryCount: names.length,\n      omittedEntryCount: Math.max(\n        0,\n        names.length - MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES,\n      ),\n      entries,\n    });\n  } catch (e) {\n    lc.warn?.('Unable to inspect Litestream restore directory', {\n      phase,\n      directory,\n      replicaFile,\n      error: String(e),\n    });\n  }\n}\n\n/**\n * Opens and validates the restored replica. Returns the open connection if\n * the replica is valid, which the caller must close; otherwise closes it and\n * returns `undefined`.\n */\nfunction openValidReplica(\n  lc: LogContext,\n  replica: string,\n  constraints: ReplicaConstraints | undefined,\n): Database | undefined {\n  let db: Database | undefined;\n  try {\n    // Note: Open the database and read the subscription state as a\n    // sanity / corruption check, even if there are no constraints.\n    db = new Database(lc, replica);\n    const {replicaVersion, watermark} = getSubscriptionState(\n      new StatementRunner(db),\n    );\n    if (constraints) {\n      if (replicaVersion !== constraints.replicaVersion) {\n        lc.warn?.(\n          `Local replica version ${replicaVersion} does not match expected replicaVersion ${constraints.replicaVersion}`,\n          constraints,\n        );\n        return undefined;\n      }\n      if (watermark < constraints.minWatermark) {\n        lc.warn?.(\n          `Local replica watermark ${watermark} is earlier than minWatermark ${constraints.minWatermark}`,\n        );\n        return undefined;\n      }\n      lc.info?.(\n        `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible`,\n        constraints,\n      );\n    }\n    const valid = db;\n    db = undefined; // owned by the caller\n    return valid;\n  } catch (e) {\n    if (isSQLiteCorruption(e)) {\n      logSQLiteCorruptionDiagnostics(lc, 'restored replica', replica, e);\n    }\n    lc.error?.('Error while validating restored replica', e);\n    return undefined;\n  } finally {\n    db?.close();\n  }\n}\n\nexport function startReplicaBackupProcess(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n): ChildProcess {\n  const {litestream, env} = getLitestream('replicate', config, replicaFile);\n  const attrs = litestreamBackupProcessMetricAttrs(config);\n  lc.info?.(`starting litestream backup to ${backupDestinationURL(config)}`);\n  const start = performance.now();\n  const proc = spawn(litestream, ['replicate'], {\n    env,\n    stdio: 'inherit',\n    windowsHide: true,\n  });\n  let recorded = false;\n  const record = (result: 'success' | 'error' | 'stopped') => {\n    if (recorded) {\n      return;\n    }\n    recorded = true;\n    const labels = {...attrs, result};\n    litestreamBackupProcessRuns().add(1, labels);\n    litestreamBackupProcessDuration().recordMs(\n      performance.now() - start,\n      labels,\n    );\n  };\n  proc.on('error', e => {\n    lc.warn?.(`litestream backup process error`, e);\n    record('error');\n  });\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      lc.info?.(`litestream backup process stopped`, {signal});\n      record('stopped');\n    } else if (code === 0) {\n      record('success');\n    } else {\n      lc.warn?.(`litestream backup process exited with code ${code}`);\n      record('error');\n    }\n  });\n  return proc;\n}\n\n// Listing the backup state requires a few S3 LIST requests, which should\n// normally complete well within this timeout.\nconst LIST_BACKUP_TIMEOUT_MS = 30_000;\n\nconst wsRe = /\\s+/;\n\n/**\n * Returns the time of the most recent object (snapshot or WAL segment)\n * actually uploaded to the backup replica destination, as listed by the\n * bundled litestream CLI (`litestream snapshots` / `litestream wal`).\n *\n * This queries the replica destination (e.g. S3) directly, and thus serves\n * as a source of truth for backup durability. This is in contrast to the\n * `litestream_replica_progress` metric, which is exported when litestream\n * *believes* an upload has succeeded, and has been observed to advance even\n * when nothing is actually written to the destination.\n *\n * Rejects if the backup state cannot be determined (spawn error, non-zero\n * exit, timeout, or empty/unparseable listing).\n */\nexport async function getLastBackupTime(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n): Promise<Date> {\n  const [snapshots, wal] = await Promise.all([\n    listBackupCreatedTimes(lc, config, replicaFile, 'snapshots'),\n    listBackupCreatedTimes(lc, config, replicaFile, 'wal'),\n  ]);\n  const times = [...snapshots, ...wal];\n  if (times.length === 0) {\n    // Note: the litestream CLI exits with code 0 and logs listing errors\n    // (e.g. S3 failures) to stderr, so an empty listing cannot be\n    // distinguished from a failed one. Since a valid backup always contains\n    // at least one snapshot, an empty listing is treated as a failure.\n    throw new Error(\n      `no snapshots or WAL segments listed at ${backupDestinationURL(config)}`,\n    );\n  }\n  return new Date(Math.max(...times.map(time => time.getTime())));\n}\n\n/**\n * Runs `litestream <snapshots|wal> <replica-file>` with the same config /\n * environment used by the `litestream replicate` process (so that the\n * backupURL, endpoint, region, and credentials are identical), and parses\n * the `created` column (RFC3339) of the tab-formatted output, e.g.:\n *\n * ```\n * replica  generation        index  size     created\n * s3       1862f44967b3863f  0      4546445  2026-06-10T01:11:32Z\n * ```\n */\nasync function listBackupCreatedTimes(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  command: 'snapshots' | 'wal',\n): Promise<Date[]> {\n  const start = performance.now();\n  let result: 'success' | 'empty' | 'timeout' | 'error' = 'error';\n  const {litestream, env} = getLitestream('replicate', config, replicaFile);\n  const proc = spawn(litestream, [command, replicaFile], {\n    env,\n    stdio: ['ignore', 'pipe', 'inherit'],\n    windowsHide: true,\n  });\n  const {promise, resolve, reject} = resolver<string>();\n  let stdout = '';\n  proc.stdout.setEncoding('utf-8');\n  proc.stdout.on('data', chunk => (stdout += chunk));\n  proc.on('error', reject);\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      reject(new Error(`litestream ${command} killed with ${signal}`));\n    } else if (code !== 0) {\n      reject(new Error(`litestream ${command} exited with code ${code}`));\n    } else {\n      resolve(stdout);\n    }\n  });\n  const timeout = setTimeout(() => {\n    result = 'timeout';\n    reject(new Error(`timed out listing backup state (litestream ${command})`));\n    proc.kill('SIGKILL');\n  }, LIST_BACKUP_TIMEOUT_MS);\n\n  try {\n    const output = await promise;\n    const times = parseBackupCreatedTimes(lc, command, output);\n    result = times.length ? 'success' : 'empty';\n    return times;\n  } finally {\n    clearTimeout(timeout);\n    litestreamBackupListDuration().recordMs(performance.now() - start, {\n      ...litestreamBackupMetricAttrs(config),\n      command,\n      result,\n    });\n  }\n}\n\n/**\n * Parses the `created` column (the last, RFC3339-formatted column) from the\n * tab-formatted output of `litestream snapshots` / `litestream wal`. The\n * header row and any unparseable lines are skipped.\n *\n * Exported for testing.\n */\nexport function parseBackupCreatedTimes(\n  lc: LogContext,\n  command: 'snapshots' | 'wal',\n  output: string,\n): Date[] {\n  const times: Date[] = [];\n  for (const line of output.split('\\n')) {\n    const cols = line.trim().split(wsRe);\n    const created = cols.at(-1);\n    if (\n      cols.length < 2 ||\n      created === undefined ||\n      created === 'created' /* header row */\n    ) {\n      continue;\n    }\n    const time = new Date(created);\n    if (Number.isNaN(time.getTime())) {\n      lc.warn?.(`unexpected line in litestream ${command} output: ${line}`);\n      continue;\n    }\n    times.push(time);\n  }\n  return times;\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAiEA,IAAM,uCAAuC;AAC7C,IAAM,2BAA2B;;;;;;AAejC,SAAgB,qBACd,QACoB;CACpB,OAAO,OAAO,qBAAqB,OAAO;AAC5C;AAEA,SAAS,cACP,MACA,QACA,aACA,kBAIA;CACA,MAAM,EACJ,YACA,cACA,gBACA,eACA,UACA,YACA,cACA,UACA,QACA,MACA,uBAGA,yBAAyB,wBAAwB,KACjD,yBAAyB,yBAAyB,IAClD,4BACA,kCACA,kCACA,6BACA,sBACA,kBACE;CAEJ,MAAM,KACH,SAAS,aAAa,kBACtB,SAAS,eAAe;CAC3B,MAAM,cACH,KAAK,eAAe,eACrB,KAAK,YAAY,iCAAiC;CACpD,MAAM,mBAAmB,KAAK,eAAe;CAG7C,MAAM,gBAAgB,6BAA6B,KAAK;CAExD,OAAO;EACL;EACA,KAAK;GACH,GAAG,QAAQ;IACV,sBAAsB;IACtB,+BAA+B,KAAK,qBAAqB,MAAM,CAAC;IAChE,8CAA8C,OAC7C,sBACF;IACC,8CAA8C,OAC7C,sBACF;IACC,wDAAwD,OACvD,gCACF;IACC,wDAAwD,OACvD,gCACF;IAEC,6CAA6C,OAK5C,IAAI,gCACN;IACC,oCAAoC,OAAO,aAAa;IACxD,8BAA8B,oBAAoB;IAClD,mDAAmD,OAClD,2BACF;IACC,sDAAsD,OACrD,8BAA8B,CAChC;IACC,0CAA0C,OAAO,oBAAoB;IACrE,mCAAmC,OAAO,aAAa;IACvD,oBAAoB;IACpB,gCAAgC,qBAAqB,WAAW;IAChE,sBAAsB;IACtB,oBAAoB,OAAO,IAAI;GAChC,GAAI,WAAW,GAAE,6BAA6B,SAAQ,IAAI,CAAC;GAC3D,GAAI,SAAS,GAAE,2BAA2B,OAAM,IAAI,CAAC;IACpD,oCAAoC;EACvC;CACF;AACF;AAEA,eAAsB,WACpB,IACA,QACA,aACA,oBACA,MACA,QACA,YACyB;CACzB,MAAM,EAAC,gBAAgB,GAAG,YAAW,MAAM,YACzC,IACA,QACA,aACA,oBACA,MACA,QACA,UACF;CACA,IAAI,QAAQ,WAAW,qBAAqB,gBAAgB;EAK1D,MAAM,EAAC,gBAAgB,GAAG,GAAG,UAAS,MAAM,YAC1C,IACA,QACA,aACA,oBACA,MACA,QACA,UACF;EACA,OAAO;CACT;CACA,OAAO;AACT;AAEA,eAAe,YACb,IACA,QACA,aACA,oBACA,MACA,QACA,YACqD;CACrD,MAAM,EAAC,kBAAiB;CACxB,MAAM,YAAY,qBAAqB,MAAM;CAC7C,IAAI,cAAc,CAAC,eACjB,aAAa,KAAA;CAEf,MAAM,QAAQ,6BAA6B,QAAQ,MAAM,SAAS;CAClE,IAAI,SAAwB;CAC5B,IAAI;EACF,4BAA4B,IAAI,aAAa,qBAAqB;EAClE,uBAAuB,WAAW;EAClC,MAAM,iBAAiB,WAAW,WAAW;EAC7C,IAAI,CAAC,gBAIH,wBAAwB,WAAW;EAErC,MAAM,EAAC,YAAY,QAAO,cACxB,WACA,QACA,aACA,OACF;EACA,MAAM,EACJ,oBAAoB,aACpB,sBACA,kBACE;EACJ,GAAG,OAAO,+BAA+B;GACvC,oBAAoB;GACpB;GACA;EACF,CAAC;EAOD,MAAM,OAAO,MACX,YACA;GACE;GACA;GACA;GACA;GACA,OAAO,WAAW;GAClB,GAAI,aAAa;IAAC;IAAS;IAAgB,WAAW;GAAG,IAAI,CAAC;GAC9D;EACF,GACA;GAAC;GAAK,OAAO;IAAC;IAAU;IAAQ;GAAM;GAAG,aAAa;GAAM;EAAM,CACpE;EACA,IAAI,SAAS;EACb,IAAI,SAAS;EACb,KAAK,OAAO,YAAY,OAAO;EAC/B,KAAK,OAAO,YAAY,OAAO;EAG/B,KAAK,OAAO,GAAG,SAAQ,UAAS;GAC9B,QAAQ,OAAO,MAAM,KAAK;GAC1B,UAAU;EACZ,CAAC;EACD,KAAK,OAAO,GAAG,SAAQ,UAAS;GAC9B,QAAQ,OAAO,MAAM,KAAK;GAC1B,UAAU;EACZ,CAAC;EACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAS;EAC5C,KAAK,GAAG,SAAS,MAAM;EACvB,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,IAAI,QACF,OAAO,0BAA0B,QAAQ;QACpC,IAAI,SAAS,GAClB,OAAO,+BAA+B,MAAM;QAE5C,QAAQ;EAEZ,CAAC;EACD,MAAM,eAAe,YAAY,IAAI;EACrC,IAAI;GACF,MAAM;GACN,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAS,CAC9B;EACF,SAAS,GAAG;GACV,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAO,CAC5B;GACA,MAAM,SAAS,CAAC,QAAQ,MAAM,EAC3B,KAAI,UAAS,MAAM,KAAK,CAAC,EACzB,OAAO,OAAO,EACd,KAAK,IAAI;GACZ,MAAM,IAAI,MAAM,SAAS,GAAG,OAAO,CAAC,EAAE,IAAI,WAAW,OAAO,CAAC,GAAG,EAC9D,OAAO,EACT,CAAC;EACH;EACA,IAAI,CAAC,WAAW,WAAW,GAAG;GAC5B,SAAS;GACT,OAAO;IAAC,UAAU;IAAO;IAAW;IAAQ;GAAc;EAC5D;EACA,MAAM,kBAAkB,YAAY,IAAI;EACxC,MAAM,KAAK,iBAAiB,IAAI,aAAa,kBAAkB;EAC/D,oCAAoC,EAAE,SACpC,YAAY,IAAI,IAAI,iBACpB;GAAC,GAAG;GAAO,QAAQ,KAAK,YAAY;EAAiB,CACvD;EACA,IAAI,CAAC,IAAI;GACP,SAAS;GACT,GAAG,OAAO,6CAA6C;GACvD,aAAa,WAAW;GACxB,kBAAkB,WAAW;GAG7B,wBAAwB,WAAW;GACnC,OAAO;IAAC,UAAU;IAAO;IAAW;IAAQ;GAAc;EAC5D;EACA,IAAI;GACF,SAAS;GACT,IAAI,CAAC,gBAAgB;IAYnB,kBAAkB,WAAW;IAC7B,0BAA0B,EAAE,IAAI,SAAS,WAAW,EAAE,MAAM;KAC1D,GAAG;KACH,QAAQ;IACV,CAAC;GACH;GACA,MAAM,SACJ,eAAe,KAAA,KACf,CAAC,kBACD,cAAc,IAAI,MAAM;GAC1B,OAAO;IAAC,UAAU;IAAM;IAAW;IAAQ;IAAgB;GAAM;EACnE,UAAU;GACR,GAAG,MAAM;EACX;CACF,UAAU;EACR,IAAI;GACF,uBAAuB,WAAW;EACpC,SAAS,GAAG;GACV,GAAG,OAAO,yDAAyD;IACjE;IACA,OAAO,OAAO,CAAC;GACjB,CAAC;EACH;EACA,4BAA4B,IAAI,aAAa,oBAAoB;EACjE,0BAA0B,EAAE,IAAI,GAAG;GAAC,GAAG;GAAO;EAAM,CAAC;CACvD;AACF;AAGA,IAAM,0BAA0B,eAAE,OAAO,EACvC,MAAM,eACH,OAAO;CACN,QAAQ,eAAE,QAAQ;CAClB,MAAM,eAAE,OAAO,EAAE,SAAS;AAC5B,CAAC,EACA,SAAS,EACd,CAAC;;;;;;AASD,SAAS,cAAc,IAAgB,QAAyB;CAC9D,MAAM,QAAQ,OAAO,QAAQ,GAAG;CAChC,MAAM,MAAM,OAAO,YAAY,GAAG;CAClC,IAAI,QAAQ,KAAK,MAAM,OAAO;EAC5B,GAAG,OAAO,iDAAiD;EAC3D,OAAO;CACT;CACA,IAAI;CACJ,IAAI;EACF,SAAS,MACP,KAAK,MAAM,OAAO,MAAM,OAAO,MAAM,CAAC,CAAC,GACvC,yBACA,aACF;CACF,SAAS,GAAG;EACV,GAAG,OAAO,iDAAiD,CAAC;EAC5D,OAAO;CACT;CACA,MAAM,SAAS,OAAO,MAAM,WAAW;CACvC,GAAG,OACD,SACI,6BAA6B,OAAO,MAAM,SAC1C,qCACN;CACA,OAAO;AACT;;;;;AAMA,SAAgB,wBAAwB,aAAqB;CAC3D,OAAO,KAAK,QAAQ,WAAW,GAAG,IAAI,SAAS,WAAW,EAAE,YAAY,GAAG;EACzE,WAAW;EACX,OAAO;CACT,CAAC;AACH;AAEA,SAAS,uBAAuB,aAAqB;CACnD,MAAM,uBAAuB,GAAG,YAAY;CAC5C,aAAa,oBAAoB;CAIjC,MAAM,YAAY,QAAQ,oBAAoB;CAC9C,IAAI,CAAC,WAAW,SAAS,GACvB;CAEF,MAAM,SAAS,GAAG,SAAS,oBAAoB,EAAE;CACjD,KAAK,MAAM,QAAQ,YAAY,SAAS,GAAG;EACzC,IAAI,CAAC,KAAK,WAAW,MAAM,KAAK,CAAC,KAAK,SAAS,MAAM,GACnD;EAEF,MAAM,QAAQ,KAAK,MAAM,OAAO,QAAQ,EAAc;EACtD,IAAI,yBAAyB,KAAK,KAAK,GACrC,OAAO,KAAK,WAAW,IAAI,GAAG,EAAC,OAAO,KAAI,CAAC;CAE/C;AACF;AAEA,SAAS,4BACP,IACA,aACA,OACA;CACA,MAAM,YAAY,QAAQ,WAAW;CACrC,IAAI;EACF,MAAM,QAAQ,YAAY,SAAS,EAAE,KAAK;EAC1C,MAAM,UAAU,MACb,MAAM,GAAG,oCAAoC,EAC7C,KAAI,UAAS;GAAC;GAAM,WAAW,SAAS,KAAK,WAAW,IAAI,CAAC,EAAE;EAAI,EAAE;EACxE,GAAG,OAAO,yCAAyC;GACjD;GACA;GACA;GACA,YAAY,MAAM;GAClB,mBAAmB,KAAK,IACtB,GACA,MAAM,SAAS,oCACjB;GACA;EACF,CAAC;CACH,SAAS,GAAG;EACV,GAAG,OAAO,kDAAkD;GAC1D;GACA;GACA;GACA,OAAO,OAAO,CAAC;EACjB,CAAC;CACH;AACF;;;;;;AAOA,SAAS,iBACP,IACA,SACA,aACsB;CACtB,IAAI;CACJ,IAAI;EAGF,KAAK,IAAI,SAAS,IAAI,OAAO;EAC7B,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,EAAE,CACxB;EACA,IAAI,aAAa;GACf,IAAI,mBAAmB,YAAY,gBAAgB;IACjD,GAAG,OACD,yBAAyB,eAAe,0CAA0C,YAAY,kBAC9F,WACF;IACA;GACF;GACA,IAAI,YAAY,YAAY,cAAc;IACxC,GAAG,OACD,2BAA2B,UAAU,gCAAgC,YAAY,cACnF;IACA;GACF;GACA,GAAG,OACD,4BAA4B,eAAe,iBAAiB,UAAU,iBACtE,WACF;EACF;EACA,MAAM,QAAQ;EACd,KAAK,KAAA;EACL,OAAO;CACT,SAAS,GAAG;EACV,IAAI,mBAAmB,CAAC,GACtB,+BAA+B,IAAI,oBAAoB,SAAS,CAAC;EAEnE,GAAG,QAAQ,2CAA2C,CAAC;EACvD;CACF,UAAU;EACR,IAAI,MAAM;CACZ;AACF;AAEA,SAAgB,0BACd,IACA,QACA,aACc;CACd,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,QAAQ,mCAAmC,MAAM;CACvD,GAAG,OAAO,iCAAiC,qBAAqB,MAAM,GAAG;CACzE,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,OAAO,MAAM,YAAY,CAAC,WAAW,GAAG;EAC5C;EACA,OAAO;EACP,aAAa;CACf,CAAC;CACD,IAAI,WAAW;CACf,MAAM,UAAU,WAA4C;EAC1D,IAAI,UACF;EAEF,WAAW;EACX,MAAM,SAAS;GAAC,GAAG;GAAO;EAAM;EAChC,4BAA4B,EAAE,IAAI,GAAG,MAAM;EAC3C,gCAAgC,EAAE,SAChC,YAAY,IAAI,IAAI,OACpB,MACF;CACF;CACA,KAAK,GAAG,UAAS,MAAK;EACpB,GAAG,OAAO,mCAAmC,CAAC;EAC9C,OAAO,OAAO;CAChB,CAAC;CACD,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QAAQ;GACV,GAAG,OAAO,qCAAqC,EAAC,OAAM,CAAC;GACvD,OAAO,SAAS;EAClB,OAAO,IAAI,SAAS,GAClB,OAAO,SAAS;OACX;GACL,GAAG,OAAO,8CAA8C,MAAM;GAC9D,OAAO,OAAO;EAChB;CACF,CAAC;CACD,OAAO;AACT;AAIA,IAAM,yBAAyB;AAE/B,IAAM,OAAO;;;;;;;;;;;;;;;AAgBb,eAAsB,kBACpB,IACA,QACA,aACe;CACf,MAAM,CAAC,WAAW,OAAO,MAAM,QAAQ,IAAI,CACzC,uBAAuB,IAAI,QAAQ,aAAa,WAAW,GAC3D,uBAAuB,IAAI,QAAQ,aAAa,KAAK,CACvD,CAAC;CACD,MAAM,QAAQ,CAAC,GAAG,WAAW,GAAG,GAAG;CACnC,IAAI,MAAM,WAAW,GAKnB,MAAM,IAAI,MACR,0CAA0C,qBAAqB,MAAM,GACvE;CAEF,OAAO,IAAI,KAAK,KAAK,IAAI,GAAG,MAAM,KAAI,SAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC;AAChE;;;;;;;;;;;;AAaA,eAAe,uBACb,IACA,QACA,aACA,SACiB;CACjB,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI,SAAoD;CACxD,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,OAAO,MAAM,YAAY,CAAC,SAAS,WAAW,GAAG;EACrD;EACA,OAAO;GAAC;GAAU;GAAQ;EAAS;EACnC,aAAa;CACf,CAAC;CACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAiB;CACpD,IAAI,SAAS;CACb,KAAK,OAAO,YAAY,OAAO;CAC/B,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;CACjD,KAAK,GAAG,SAAS,MAAM;CACvB,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QACF,uBAAO,IAAI,MAAM,cAAc,QAAQ,eAAe,QAAQ,CAAC;OAC1D,IAAI,SAAS,GAClB,uBAAO,IAAI,MAAM,cAAc,QAAQ,oBAAoB,MAAM,CAAC;OAElE,QAAQ,MAAM;CAElB,CAAC;CACD,MAAM,UAAU,iBAAiB;EAC/B,SAAS;EACT,uBAAO,IAAI,MAAM,8CAA8C,QAAQ,EAAE,CAAC;EAC1E,KAAK,KAAK,SAAS;CACrB,GAAG,sBAAsB;CAEzB,IAAI;EAEF,MAAM,QAAQ,wBAAwB,IAAI,SAAS,MAD9B,OACoC;EACzD,SAAS,MAAM,SAAS,YAAY;EACpC,OAAO;CACT,UAAU;EACR,aAAa,OAAO;EACpB,6BAA6B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO;GACjE,GAAG,4BAA4B,MAAM;GACrC;GACA;EACF,CAAC;CACH;AACF;;;;;;;;AASA,SAAgB,wBACd,IACA,SACA,QACQ;CACR,MAAM,QAAgB,CAAC;CACvB,KAAK,MAAM,QAAQ,OAAO,MAAM,IAAI,GAAG;EACrC,MAAM,OAAO,KAAK,KAAK,EAAE,MAAM,IAAI;EACnC,MAAM,UAAU,KAAK,GAAG,EAAE;EAC1B,IACE,KAAK,SAAS,KACd,YAAY,KAAA,KACZ,YAAY,WAEZ;EAEF,MAAM,OAAO,IAAI,KAAK,OAAO;EAC7B,IAAI,OAAO,MAAM,KAAK,QAAQ,CAAC,GAAG;GAChC,GAAG,OAAO,iCAAiC,QAAQ,WAAW,MAAM;GACpE;EACF;EACA,MAAM,KAAK,IAAI;CACjB;CACA,OAAO;AACT"}

package/out/zero-cache/src/services/view-syncer/connection-context-manager.d.ts

@@ -1,6 +1,5 @@
 import type { LogContext } from '@rocicorp/logger';
 import type { InitConnectionBody } from '../../../../zero-protocol/src/connect.ts';
-import type { ProtocolFlags } from '../../../../zero-protocol/src/protocol-flag.ts';
 import type { UpdateAuthBody } from '../../../../zero-protocol/src/update-auth.ts';
 import { type Auth, type ValidateLegacyJWT } from '../../auth/auth.ts';
 import type { ZeroConfig } from '../../config/zero-config.ts';
@@ -60,7 +59,6 @@ export type ConnectionContext = {
     readonly profileID: string | null;
     readonly baseCookie: string | null;
     readonly protocolVersion: number;
-    readonly protocolFlags: ProtocolFlags;
     readonly revision: number;
     readonly revalidateAt: number | undefined;
     readonly insertionOrder: number;