@rocicorp/zero 1.10.0-canary.10 → 1.10.0-canary.12

package/out/zero-cache/src/db/transaction-pool.js.map

@@ -1 +1 @@
-{"version":3,"file":"transaction-pool.js","names":["#mode","#init","#cleanup","#tasks","#workers","#initialWorkers","#maxWorkers","#timeoutTask","#lc","#stmtRunner","#numWorkers","#db","#addWorker","#start","#stmts","#numWorking","#failure","#done","#process","#readRunner"],"sources":["../../../../../zero-cache/src/db/transaction-pool.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {type Resolver, resolver} from '@rocicorp/resolver';\nimport type postgres from 'postgres';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport type {Enum} from '../../../shared/src/enum.ts';\nimport {Queue} from '../../../shared/src/queue.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {type PostgresDB, type PostgresTransaction} from '../types/pg.ts';\nimport type * as Mode from './mode-enum.ts';\nimport {runTx} from './run-transaction.ts';\n\ntype Mode = Enum<typeof Mode>;\n\ntype MaybePromise<T> = Promise<T> | T;\n\nexport type Statement =\n  | postgres.PendingQuery<(postgres.Row & Iterable<postgres.Row>)[]>\n  | postgres.PendingQuery<postgres.Row[]>;\n\n/**\n * A {@link Task} is logic run from within a transaction in a {@link TransactionPool}.\n * It returns a list of `Statements` that the transaction executes asynchronously and\n * awaits when it receives the 'done' signal.\n *\n */\nexport type Task = (\n  tx: PostgresTransaction,\n  lc: LogContext,\n) => MaybePromise<Statement[]>;\n\n/**\n * A {@link ReadTask} is run from within a transaction, but unlike a {@link Task},\n * the results of a ReadTask are opaque to the TransactionPool and returned to the\n * caller of {@link TransactionPool.processReadTask}.\n */\nexport type ReadTask<T> = (\n  tx: PostgresTransaction,\n  lc: LogContext,\n) => MaybePromise<T>;\n\nexport type Options = {\n  mode: Mode;\n\n  /**\n   * A {@link Task} that is run in each Transaction before it begins\n   * processing general tasks. This can be used to to set the transaction\n   * mode, export/set snapshots, etc. This will be run even if\n   * {@link fail} has been called on the pool.\n   */\n  init?: Task | undefined;\n\n  /**\n   * A {@link Task} that is run in each Transaction before it closes.\n   * This will be run even if {@link fail} has been called, or if a\n   * preceding Task threw an Error.\n   */\n  cleanup?: Task | undefined;\n\n  /**\n   * The initial number of transaction workers to process tasks.\n   * This is the steady state number of workers that will be kept\n   * alive if the TransactionPool is long lived.\n   * This must be greater than 0. Defaults to 1.\n   */\n  initialWorkers?: number;\n\n  /**\n   * When specified, allows the pool to grow to `maxWorkers`. This\n   * must be greater than or equal to `initialWorkers`. On-demand\n   * workers will be shut down after an idle timeout of 5 seconds.\n   */\n  maxWorkers?: number;\n};\n\n/**\n * A TransactionPool is a pool of one or more {@link postgres.TransactionSql}\n * objects that participate in processing a dynamic queue of tasks.\n *\n * This can be used for serializing a set of tasks that arrive asynchronously\n * to a single transaction (for writing) or performing parallel reads across\n * multiple connections at the same snapshot (e.g. read only snapshot transactions).\n */\nexport class TransactionPool {\n  #lc: LogContext;\n  readonly #mode: Mode;\n  readonly #init: TaskRunner | undefined;\n  readonly #cleanup: TaskRunner | undefined;\n  readonly #tasks = new Queue<TaskRunner | Error | 'done'>();\n  readonly #workers: Promise<unknown>[] = [];\n  readonly #initialWorkers: number;\n  readonly #maxWorkers: number;\n  readonly #timeoutTask: TimeoutTasks;\n  #numWorkers: number;\n  #numWorking = 0;\n  #db: PostgresDB | undefined; // set when running. stored to allow adaptive pool sizing.\n\n  #done = false;\n  #failure: Error | undefined;\n\n  constructor(\n    lc: LogContext,\n    opts: Options,\n    timeoutTasks = TIMEOUT_TASKS, // Overridden for tests.\n  ) {\n    const {\n      mode,\n      init,\n      cleanup,\n      initialWorkers = 1,\n      maxWorkers = initialWorkers,\n    } = opts;\n    assert(initialWorkers > 0, 'initialWorkers must be positive');\n    assert(\n      maxWorkers >= initialWorkers,\n      'maxWorkers must be >= initialWorkers',\n    );\n\n    this.#lc = lc;\n    this.#mode = mode;\n    this.#init = init ? this.#stmtRunner(init) : undefined;\n    this.#cleanup = cleanup ? this.#stmtRunner(cleanup) : undefined;\n    this.#initialWorkers = initialWorkers;\n    this.#numWorkers = initialWorkers;\n    this.#maxWorkers = maxWorkers;\n    this.#timeoutTask = timeoutTasks;\n  }\n\n  /**\n   * Starts the pool of workers to process Tasks with transactions opened from the\n   * specified {@link db}.\n   */\n  run(db: PostgresDB): this {\n    assert(!this.#db, 'already running');\n    this.#db = db;\n    for (let i = 0; i < this.#numWorkers; i++) {\n      this.#addWorker(db);\n    }\n    return this;\n  }\n\n  /**\n   * Adds context parameters to internal LogContext. This is useful for context values that\n   * are not known when the TransactionPool is constructed (e.g. determined after a database\n   * call when the pool is running).\n   *\n   * Returns an object that can be used to add more parameters.\n   */\n  addLoggingContext(key: string, value: string) {\n    this.#lc = this.#lc.withContext(key, value);\n\n    return {\n      addLoggingContext: (key: string, value: string) =>\n        this.addLoggingContext(key, value),\n    };\n  }\n\n  /**\n   * Returns a promise that:\n   *\n   * * resolves after {@link setDone} has been called (or the the pool as been {@link unref}ed\n   *   to a 0 ref count), once all added tasks have been processed and all transactions have been\n   *   committed or closed.\n   *\n   * * rejects if processing was aborted with {@link fail} or if processing any of\n   *   the tasks resulted in an error. All uncommitted transactions will have been\n   *   rolled back.\n   *\n   * Note that partial failures are possible if processing writes with multiple workers\n   * (e.g. `setDone` is called, allowing some workers to commit, after which other\n   *  workers encounter errors). Using a TransactionPool in this manner does not make\n   * sense in terms of transactional semantics, and is thus not recommended.\n   *\n   * For reads, however, multiple workers is useful for performing parallel reads\n   * at the same snapshot. See {@link synchronizedSnapshots} for an example.\n   * Resolves or rejects when all workers are done or failed.\n   */\n  async done() {\n    const numWorkers = this.#workers.length;\n    await Promise.all(this.#workers);\n\n    if (numWorkers < this.#workers.length) {\n      // If workers were added after the initial set, they must be awaited to ensure\n      // that the results (i.e. rejections) of all workers are accounted for. This only\n      // needs to be re-done once, because the fact that the first `await` completed\n      // guarantees that the pool is in a terminal state and no new workers can be added.\n      await Promise.all(this.#workers);\n    }\n    this.#lc.debug?.('transaction pool done');\n\n    const elapsed = performance.now() - this.#start;\n    if (elapsed > 60_000) {\n      if (this.#stmts > 0) {\n        this.#lc.warn?.(\n          `finished long transaction with ${this.#stmts} statements (${elapsed.toFixed(3)} ms)`,\n        );\n      } else {\n        this.#lc.warn?.(\n          `finished long read transaction (${elapsed.toFixed(3)} ms)`,\n        );\n      }\n    }\n  }\n\n  #addWorker(db: PostgresDB) {\n    const id = this.#workers.length + 1;\n    const lc = this.#lc.withContext('tx', id);\n\n    const tt: TimeoutTask =\n      this.#workers.length < this.#initialWorkers\n        ? this.#timeoutTask.forInitialWorkers\n        : this.#timeoutTask.forExtraWorkers;\n    const {timeoutMs} = tt;\n    const timeoutTask = tt.task === 'done' ? 'done' : this.#stmtRunner(tt.task);\n\n    const worker = async (tx: PostgresTransaction) => {\n      const start = performance.now();\n      try {\n        lc.debug?.('started transaction');\n\n        let last: Promise<void> = promiseVoid;\n\n        const executeTask = async (runner: TaskRunner) => {\n          runner !== this.#init && this.#numWorking++;\n          const {pending} = await runner.run(tx, lc, () => {\n            runner !== this.#init && this.#numWorking--;\n          });\n          last = pending ?? last;\n        };\n\n        let task: TaskRunner | Error | 'done' =\n          this.#init ?? (await this.#tasks.dequeue(timeoutTask, timeoutMs));\n\n        try {\n          while (task !== 'done') {\n            if (\n              task instanceof Error ||\n              (task !== this.#init && this.#failure)\n            ) {\n              throw this.#failure ?? task;\n            }\n            await executeTask(task);\n\n            // await the next task.\n            task = await this.#tasks.dequeue(timeoutTask, timeoutMs);\n          }\n        } finally {\n          // Execute the cleanup task even on failure.\n          if (this.#cleanup) {\n            await executeTask(this.#cleanup);\n          }\n        }\n\n        const elapsed = performance.now() - start;\n        lc.debug?.(`closing transaction (${elapsed.toFixed(3)} ms)`);\n        // Given the semantics of a Postgres transaction, the last statement\n        // will only succeed if all of the preceding statements succeeded.\n        return last;\n      } catch (e) {\n        if (e !== this.#failure) {\n          this.fail(e); // A failure in any worker should fail the pool.\n        }\n        throw e;\n      }\n    };\n\n    const workerTx = runTx(db, worker, {mode: this.#mode})\n      .catch(e => {\n        if (e instanceof RollbackSignal) {\n          // A RollbackSignal is used to gracefully rollback the postgres.js\n          // transaction block. It should not be thrown up to the application.\n          lc.debug?.('aborted transaction');\n        } else {\n          throw e;\n        }\n      })\n      .finally(() => this.#numWorkers--);\n\n    // Attach a rejection handler immediately to prevent unhandledRejections.\n    // The application will handle errors when it awaits processReadTask()\n    // or done().\n    workerTx.catch(() => {});\n\n    this.#workers.push(workerTx);\n\n    // After adding the worker, enqueue a terminal signal if we are in either of the\n    // terminal states (both of which prevent more tasks from being enqueued), to ensure\n    // that the added worker eventually exits.\n    if (this.#done) {\n      this.#tasks.enqueue('done');\n    }\n    if (this.#failure) {\n      this.#tasks.enqueue(this.#failure);\n    }\n  }\n\n  /**\n   * Processes the statements produced by the specified {@link Task},\n   * returning a Promise that resolves when the statements are either processed\n   * by the database or rejected.\n   *\n   * Note that statement failures will result in failing the entire\n   * TransactionPool (per transaction semantics). However, the returned Promise\n   * itself will resolve rather than reject. As such, it is fine to ignore\n   * returned Promises in order to pipeline requests to the database. It is\n   * recommended to occasionally await them (e.g. after some threshold) in\n   * order to avoid memory blowup in the case of database slowness.\n   */\n  process(task: Task): Promise<void> {\n    const r = resolver<void>();\n    this.#process(this.#stmtRunner(task, r));\n    return r.promise;\n  }\n\n  readonly #start = performance.now();\n  #stmts = 0;\n\n  /**\n   * Implements the semantics specified in {@link process()}.\n   *\n   * Specifically:\n   * * `freeWorker()` is called as soon as the statements are produced,\n   *   allowing them to be pipelined to the database.\n   * * Statement errors result in failing the transaction pool.\n   * * The client-supplied Resolver resolves on success or failure;\n   *   it is never rejected.\n   */\n  #stmtRunner(task: Task, r: {resolve: () => void} = resolver()): TaskRunner {\n    return {\n      run: async (tx, lc, freeWorker) => {\n        let stmts: Statement[];\n        try {\n          stmts = await task(tx, lc);\n        } catch (e) {\n          r.resolve();\n          throw e;\n        } finally {\n          freeWorker();\n        }\n\n        if (stmts.length === 0) {\n          r.resolve();\n          return {pending: null};\n        }\n\n        // Execute the statements (i.e. send to the db) immediately.\n        // The last result is returned for the worker to await before\n        // closing the transaction.\n        const last = stmts.reduce((_, stmt) => {\n          const query = stmt as unknown as Query;\n          const index = ++this.#stmts;\n          return stmt\n            .execute()\n            .then(() => {\n              if (index % 1000 === 0) {\n                const log = index % 10000 === 0 ? 'info' : 'debug';\n                lc[log]?.(\n                  `executed ${this.#stmts}th statement (${(performance.now() - this.#start).toFixed(3)} ms)`,\n                  {statement: query.string},\n                );\n              }\n            })\n            .catch(e => this.fail(e));\n        }, promiseVoid);\n        return {pending: last.finally(r.resolve)};\n      },\n      rejected: r.resolve,\n    };\n  }\n\n  /**\n   * Processes and returns the result of executing the {@link ReadTask} from\n   * within the transaction. An error thrown by the task will result in\n   * rejecting the returned Promise, but will not affect the transaction pool\n   * itself.\n   */\n  processReadTask<T>(readTask: ReadTask<T>): Promise<T> {\n    const r = resolver<T>();\n    this.#process(this.#readRunner(readTask, r));\n    return r.promise;\n  }\n\n  /**\n   * Implements the semantics specified in {@link processReadTask()}.\n   *\n   * Specifically:\n   * * `freeWorker()` is called as soon as the result is produced,\n   *   before resolving the client-supplied Resolver.\n   * * Errors result in rejecting the client-supplied Resolver but\n   *   do not affect transaction pool.\n   */\n  #readRunner<T>(readTask: ReadTask<T>, r: Resolver<T>): TaskRunner {\n    return {\n      run: async (tx, lc, freeWorker) => {\n        let result: T;\n        try {\n          result = await readTask(tx, lc);\n          freeWorker();\n          r.resolve(result);\n        } catch (e) {\n          freeWorker();\n          r.reject(e);\n        }\n        return {pending: null};\n      },\n      rejected: r.reject,\n    };\n  }\n\n  #process(runner: TaskRunner): void {\n    assert(!this.#done, 'already set done');\n    if (this.#failure) {\n      runner.rejected(this.#failure);\n      return;\n    }\n\n    this.#tasks.enqueue(runner);\n\n    // Check if the pool size can and should be increased.\n    if (this.#numWorkers < this.#maxWorkers) {\n      const outstanding = this.#tasks.size();\n\n      if (outstanding > this.#numWorkers - this.#numWorking) {\n        this.#db && this.#addWorker(this.#db);\n        this.#numWorkers++;\n        this.#lc.debug?.(`Increased pool size to ${this.#numWorkers}`);\n      }\n    }\n  }\n\n  /**\n   * Ends all workers with a ROLLBACK. Throws if the pool is already done\n   * or aborted.\n   */\n  abort() {\n    this.fail(new RollbackSignal());\n  }\n\n  /**\n   * Signals to all workers to end their transaction once all pending tasks have\n   * been completed. Throws if the pool is already done or aborted.\n   */\n  setDone() {\n    assert(!this.#done, 'already set done');\n    this.#done = true;\n\n    for (let i = 0; i < this.#numWorkers; i++) {\n      this.#tasks.enqueue('done');\n    }\n  }\n\n  isRunning(): boolean {\n    return this.#db !== undefined && !this.#done && this.#failure === undefined;\n  }\n\n  /**\n   * Signals all workers to fail their transactions with the given {@link err}.\n   */\n  fail(err: unknown) {\n    if (!this.#done && !this.#failure) {\n      this.#failure = ensureError(err); // Fail fast: this is checked in the worker loop.\n      // Logged for informational purposes. It is the responsibility of\n      // higher level logic to classify and handle the exception.\n      const level =\n        this.#failure instanceof ControlFlowError ? 'debug' : 'info';\n      this.#lc[level]?.(this.#failure);\n\n      for (let i = 0; i < this.#numWorkers; i++) {\n        // Enqueue the Error to terminate any workers waiting for tasks.\n        this.#tasks.enqueue(this.#failure);\n      }\n    }\n  }\n}\n\ntype SynchronizeSnapshotTasks = {\n  /**\n   * The `init` Task for the TransactionPool from which the snapshot originates.\n   * The pool must have Mode.SERIALIZABLE, and will be set to READ ONLY by the\n   * `exportSnapshot` init task. If the TransactionPool has multiple workers, the\n   * first worker will export a snapshot that the others set.\n   */\n  exportSnapshot: Task;\n\n  /**\n   * The `cleanup` Task for the TransactionPool from which the snapshot\n   * originates. This Task will wait for the follower pool to `setSnapshot`\n   * to ensure that the snapshot is successfully shared before the originating\n   * transaction is closed.\n   */\n  cleanupExport: Task;\n\n  /**\n   * The `init` Task for the TransactionPool in which workers will\n   * consequently see the same snapshot as that of the first pool. The pool\n   * must have Mode.SERIALIZABLE, and will have the ability to perform writes.\n   */\n  setSnapshot: Task;\n\n  /** The ID of the shared snapshot. */\n  snapshotID: Promise<string>;\n};\n\n/**\n * Init Tasks for Postgres snapshot synchronization across transactions.\n *\n * https://www.postgresql.org/docs/9.3/functions-admin.html#:~:text=Snapshot%20Synchronization%20Functions,identical%20content%20in%20the%20database.\n */\nexport function synchronizedSnapshots(): SynchronizeSnapshotTasks {\n  const {\n    promise: snapshotExported,\n    resolve: exportSnapshot,\n    reject: failExport,\n  } = resolver<string>();\n\n  const {\n    promise: snapshotCaptured,\n    resolve: captureSnapshot,\n    reject: failCapture,\n  } = resolver<unknown>();\n\n  // Set by the first worker to run its initTask, who becomes responsible for\n  // exporting the snapshot. TODO: Plumb the workerNum and use that instead.\n  let firstWorkerRun = false;\n\n  // Note: Neither init task should `await`, as processing in each pool can proceed\n  //       as soon as the statements have been sent to the db. However, the `cleanupExport`\n  //       task must `await` the result of `setSnapshot` to ensure that exporting transaction\n  //       does not close before the snapshot has been captured.\n  return {\n    exportSnapshot: tx => {\n      if (!firstWorkerRun) {\n        firstWorkerRun = true;\n        const stmt =\n          tx`SELECT pg_export_snapshot() AS snapshot; SET TRANSACTION READ ONLY;`.simple();\n        // Intercept the promise to propagate the information to `snapshotExported`.\n        stmt.then(result => exportSnapshot(result[0].snapshot), failExport);\n        return [stmt]; // Also return the stmt so that it gets awaited (and errors handled).\n      }\n      return snapshotExported.then(snapshotID => [\n        tx.unsafe(`SET TRANSACTION SNAPSHOT '${snapshotID}'`),\n        tx`SET TRANSACTION READ ONLY`.simple(),\n      ]);\n    },\n\n    setSnapshot: tx =>\n      snapshotExported.then(snapshotID => {\n        const stmt = tx.unsafe(`SET TRANSACTION SNAPSHOT '${snapshotID}'`);\n        // Intercept the promise to propagate the information to `cleanupExport`.\n        stmt.then(captureSnapshot, failCapture);\n        return [stmt];\n      }),\n\n    cleanupExport: async () => {\n      await snapshotCaptured;\n      return [];\n    },\n\n    snapshotID: snapshotExported,\n  };\n}\n\n/**\n * Returns `init` and `cleanup` {@link Task}s for a TransactionPool that ensure its workers\n * share a single view of the database. This is used for View Notifier and View Syncer logic\n * that allows multiple entities to perform parallel reads on the same snapshot of the database.\n */\nexport function sharedSnapshot(): {\n  init: Task;\n  cleanup: Task;\n  snapshotID: Promise<string>;\n} {\n  const {\n    promise: snapshotExported,\n    resolve: exportSnapshot,\n    reject: failExport,\n  } = resolver<string>();\n\n  // Set by the first worker to run its initTask, who becomes responsible for\n  // exporting the snapshot.\n  let firstWorkerRun = false;\n\n  // The LogContext of the exporting worker, used to identify its cleanup call.\n  // Each worker receives a unique lc instance (via withContext('tx', id)), so\n  // reference equality reliably identifies the exporting worker.\n  let exporterLc: LogContext | undefined;\n\n  // Set when the exporting worker's cleanup runs, signalling that the snapshot\n  // is no longer needed and any subsequently spawned workers should skip their\n  // initTask.\n  let firstWorkerDone = false;\n\n  return {\n    init: (tx, lc) => {\n      if (!firstWorkerRun) {\n        firstWorkerRun = true;\n        exporterLc = lc; // Remember which worker is the exporter.\n        const stmt = tx`SELECT pg_export_snapshot() AS snapshot;`.simple();\n        // Intercept the promise to propagate the information to `snapshotExported`.\n        stmt.then(result => exportSnapshot(result[0].snapshot), failExport);\n        return [stmt]; // Also return the stmt so that it gets awaited (and errors handled).\n      }\n      if (!firstWorkerDone) {\n        return snapshotExported.then(snapshotID => [\n          tx.unsafe(`SET TRANSACTION SNAPSHOT '${snapshotID}'`),\n        ]);\n      }\n      lc.debug?.('All work is done. No need to set snapshot');\n      return [];\n    },\n\n    cleanup: (_tx, lc) => {\n      // Only the exporting worker's cleanup should disable snapshot-setting.\n      // Non-exporter workers may finish early; letting them flip this flag\n      // would cause subsequently spawned workers to skip SET TRANSACTION SNAPSHOT\n      // and read a newer database view, violating snapshot isolation.\n      if (lc === exporterLc) {\n        firstWorkerDone = true;\n      }\n      return [];\n    },\n\n    snapshotID: snapshotExported,\n  };\n}\n\n/**\n * @returns An `init` Task for importing a snapshot from another transaction,\n *          as well as an `imported` Queue which resolves or rejects when each\n *          worker is initialized.\n */\nexport function importSnapshot(snapshotID: string): {\n  init: Task;\n  imported: Queue<void>;\n} {\n  const imported = new Queue<void>();\n\n  return {\n    init: tx => {\n      const stmt = tx.unsafe(`SET TRANSACTION SNAPSHOT '${snapshotID}'`);\n      stmt.then(\n        () => imported.enqueue(),\n        err => imported.enqueueRejection(err),\n      );\n      return [stmt];\n    },\n\n    imported,\n  };\n}\n\n/**\n * A superclass of Errors used for control flow that is needed to handle\n * another Error but does not constitute an error condition itself (e.g.\n * aborting transactions after a previous one fails). Subclassing this Error\n * will result in lowering the log level from `error` to `debug`.\n */\nexport class ControlFlowError extends Error {\n  constructor(cause?: unknown) {\n    super();\n    this.cause = cause;\n  }\n}\n\n/**\n * Internal error used to rollback the worker transaction. This is used\n * instead of executing a `ROLLBACK` statement because the postgres.js\n * library will otherwise try to execute an extraneous `COMMIT`, which\n * results in outputting a \"no transaction in progress\" warning to the\n * database logs.\n *\n * Throwing an exception, on the other hand, executes the postgres.js\n * codepath that calls `ROLLBACK` instead.\n */\nclass RollbackSignal extends ControlFlowError {\n  readonly name = 'RollbackSignal';\n  readonly message = 'rolling back transaction';\n}\n\nfunction ensureError(err: unknown): Error {\n  if (err instanceof Error) {\n    return err;\n  }\n  const error = new Error();\n  error.cause = err;\n  return error;\n}\n\ninterface TaskRunner {\n  /**\n   * Manages the running of a Task or ReadTask in two phases:\n   *\n   * - If the task involves blocking, this is done in the worker. Once the\n   *   blocking is done, `freeWorker()` is invoked to signal that the worker\n   *   is available to run another task. Note that this should be invoked\n   *   *before* resolving the result to the calling thread so that a\n   *   subsequent task can reuse the same worker.\n   *\n   * - Task statements are executed on the database asynchronously. The final\n   *   result of this processing is encapsulated in the returned `pending`\n   *   Promise. The worker will await the last pending Promise before closing\n   *   the transaction.\n   *\n   * @param freeWorker should be called as soon as all blocking operations are\n   *             completed in order to return the transaction to the pool.\n   * @returns A `pending` Promise indicating when the statements have been\n   *          processed by the database, allowing the transaction to be closed.\n   *          This should be `null` if there are no transaction-dependent\n   *          statements to await.\n   */\n  run(\n    tx: PostgresTransaction,\n    lc: LogContext,\n    freeWorker: () => void,\n  ): Promise<{pending: Promise<void> | null}>;\n\n  /**\n   * Invoked if the TransactionPool is already in a failed state when the task\n   * is requested.\n   */\n  rejected(reason: unknown): void;\n}\n\nconst IDLE_TIMEOUT_MS = 5_000;\n\n// The keepalive interval is settable by ZERO_TRANSACTION_POOL_KEEPALIVE_MS\n// as an emergency measure and is explicitly not made available as a server\n// option. This value is function of how the zero-cache uses transactions, and\n// should never need to be \"tuned\" or adjusted for different environments.\n//\n// Note that it must be shorter than IDLE_IN_TRANSACTION_SESSION_TIMEOUT_MS\n// with sufficient buffering to account for when the process is blocked by\n// synchronous calls (e.g. to the replica).\nconst KEEPALIVE_TIMEOUT_MS = parseInt(\n  process.env.ZERO_TRANSACTION_POOL_KEEPALIVE_MS ?? '5000',\n);\n\nconst KEEPALIVE_TASK: Task = (tx, lc) => {\n  lc.debug?.(`sending tx keepalive`);\n  return [tx`SELECT 1`.simple()];\n};\n\ntype TimeoutTask = {\n  timeoutMs: number;\n  task: Task | 'done';\n};\n\ntype TimeoutTasks = {\n  forInitialWorkers: TimeoutTask;\n  forExtraWorkers: TimeoutTask;\n};\n\n// Production timeout tasks. Overridden in tests.\nexport const TIMEOUT_TASKS: TimeoutTasks = {\n  forInitialWorkers: {\n    timeoutMs: KEEPALIVE_TIMEOUT_MS,\n    task: KEEPALIVE_TASK,\n  },\n  forExtraWorkers: {\n    timeoutMs: IDLE_TIMEOUT_MS,\n    task: 'done',\n  },\n};\n\n// The slice of information from the Query object in Postgres.js that gets logged for debugging.\n// https://github.com/porsager/postgres/blob/f58cd4f3affd3e8ce8f53e42799672d86cd2c70b/src/connection.js#L219\ntype Query = {string: string; parameters: object[]};\n"],"mappings":";;;;;;;;;;;;;;;AAkFA,IAAa,kBAAb,MAA6B;CAC3B;CACA;CACA;CACA;CACA,SAAkB,IAAI,MAAmC;CACzD,WAAwC,CAAC;CACzC;CACA;CACA;CACA;CACA,cAAc;CACd;CAEA,QAAQ;CACR;CAEA,YACE,IACA,MACA,eAAe,eACf;EACA,MAAM,EACJ,MACA,MACA,SACA,iBAAiB,GACjB,aAAa,mBACX;EACJ,OAAO,iBAAiB,GAAG,iCAAiC;EAC5D,OACE,cAAc,gBACd,sCACF;EAEA,KAAKQ,MAAM;EACX,KAAKR,QAAQ;EACb,KAAKC,QAAQ,OAAO,KAAKQ,YAAY,IAAI,IAAI,KAAA;EAC7C,KAAKP,WAAW,UAAU,KAAKO,YAAY,OAAO,IAAI,KAAA;EACtD,KAAKJ,kBAAkB;EACvB,KAAKK,cAAc;EACnB,KAAKJ,cAAc;EACnB,KAAKC,eAAe;CACtB;;;;;CAMA,IAAI,IAAsB;EACxB,OAAO,CAAC,KAAKI,KAAK,iBAAiB;EACnC,KAAKA,MAAM;EACX,KAAK,IAAI,IAAI,GAAG,IAAI,KAAKD,aAAa,KACpC,KAAKE,WAAW,EAAE;EAEpB,OAAO;CACT;;;;;;;;CASA,kBAAkB,KAAa,OAAe;EAC5C,KAAKJ,MAAM,KAAKA,IAAI,YAAY,KAAK,KAAK;EAE1C,OAAO,EACL,oBAAoB,KAAa,UAC/B,KAAK,kBAAkB,KAAK,KAAK,EACrC;CACF;;;;;;;;;;;;;;;;;;;;;CAsBA,MAAM,OAAO;EACX,MAAM,aAAa,KAAKJ,SAAS;EACjC,MAAM,QAAQ,IAAI,KAAKA,QAAQ;EAE/B,IAAI,aAAa,KAAKA,SAAS,QAK7B,MAAM,QAAQ,IAAI,KAAKA,QAAQ;EAEjC,KAAKI,IAAI,QAAQ,uBAAuB;EAExC,MAAM,UAAU,YAAY,IAAI,IAAI,KAAKK;EACzC,IAAI,UAAU,KACZ,IAAI,KAAKC,SAAS,GAChB,KAAKN,IAAI,OACP,kCAAkC,KAAKM,OAAO,eAAe,QAAQ,QAAQ,CAAC,EAAE,KAClF;OAEA,KAAKN,IAAI,OACP,mCAAmC,QAAQ,QAAQ,CAAC,EAAE,KACxD;CAGN;CAEA,WAAW,IAAgB;EACzB,MAAM,KAAK,KAAKJ,SAAS,SAAS;EAClC,MAAM,KAAK,KAAKI,IAAI,YAAY,MAAM,EAAE;EAExC,MAAM,KACJ,KAAKJ,SAAS,SAAS,KAAKC,kBACxB,KAAKE,aAAa,oBAClB,KAAKA,aAAa;EACxB,MAAM,EAAC,cAAa;EACpB,MAAM,cAAc,GAAG,SAAS,SAAS,SAAS,KAAKE,YAAY,GAAG,IAAI;EAE1E,MAAM,SAAS,OAAO,OAA4B;GAChD,MAAM,QAAQ,YAAY,IAAI;GAC9B,IAAI;IACF,GAAG,QAAQ,qBAAqB;IAEhC,IAAI,OAAsB;IAE1B,MAAM,cAAc,OAAO,WAAuB;KAChD,WAAW,KAAKR,SAAS,KAAKc;KAC9B,MAAM,EAAC,YAAW,MAAM,OAAO,IAAI,IAAI,UAAU;MAC/C,WAAW,KAAKd,SAAS,KAAKc;KAChC,CAAC;KACD,OAAO,WAAW;IACpB;IAEA,IAAI,OACF,KAAKd,SAAU,MAAM,KAAKE,OAAO,QAAQ,aAAa,SAAS;IAEjE,IAAI;KACF,OAAO,SAAS,QAAQ;MACtB,IACE,gBAAgB,SACf,SAAS,KAAKF,SAAS,KAAKe,UAE7B,MAAM,KAAKA,YAAY;MAEzB,MAAM,YAAY,IAAI;MAGtB,OAAO,MAAM,KAAKb,OAAO,QAAQ,aAAa,SAAS;KACzD;IACF,UAAU;KAER,IAAI,KAAKD,UACP,MAAM,YAAY,KAAKA,QAAQ;IAEnC;IAEA,MAAM,UAAU,YAAY,IAAI,IAAI;IACpC,GAAG,QAAQ,wBAAwB,QAAQ,QAAQ,CAAC,EAAE,KAAK;IAG3D,OAAO;GACT,SAAS,GAAG;IACV,IAAI,MAAM,KAAKc,UACb,KAAK,KAAK,CAAC;IAEb,MAAM;GACR;EACF;EAEA,MAAM,WAAW,MAAM,IAAI,QAAQ,EAAC,MAAM,KAAKhB,MAAK,CAAC,EAClD,OAAM,MAAK;GACV,IAAI,aAAa,gBAGf,GAAG,QAAQ,qBAAqB;QAEhC,MAAM;EAEV,CAAC,EACA,cAAc,KAAKU,aAAa;EAKnC,SAAS,YAAY,CAAC,CAAC;EAEvB,KAAKN,SAAS,KAAK,QAAQ;EAK3B,IAAI,KAAKa,OACP,KAAKd,OAAO,QAAQ,MAAM;EAE5B,IAAI,KAAKa,UACP,KAAKb,OAAO,QAAQ,KAAKa,QAAQ;CAErC;;;;;;;;;;;;;CAcA,QAAQ,MAA2B;EACjC,MAAM,IAAI,SAAe;EACzB,KAAKE,SAAS,KAAKT,YAAY,MAAM,CAAC,CAAC;EACvC,OAAO,EAAE;CACX;CAEA,SAAkB,YAAY,IAAI;CAClC,SAAS;;;;;;;;;;;CAYT,YAAY,MAAY,IAA2B,SAAS,GAAe;EACzE,OAAO;GACL,KAAK,OAAO,IAAI,IAAI,eAAe;IACjC,IAAI;IACJ,IAAI;KACF,QAAQ,MAAM,KAAK,IAAI,EAAE;IAC3B,SAAS,GAAG;KACV,EAAE,QAAQ;KACV,MAAM;IACR,UAAU;KACR,WAAW;IACb;IAEA,IAAI,MAAM,WAAW,GAAG;KACtB,EAAE,QAAQ;KACV,OAAO,EAAC,SAAS,KAAI;IACvB;IAqBA,OAAO,EAAC,SAhBK,MAAM,QAAQ,GAAG,SAAS;KACrC,MAAM,QAAQ;KACd,MAAM,QAAQ,EAAE,KAAKK;KACrB,OAAO,KACJ,QAAQ,EACR,WAAW;MACV,IAAI,QAAQ,QAAS,GAEnB,GADY,QAAQ,QAAU,IAAI,SAAS,WAEzC,YAAY,KAAKA,OAAO,iBAAiB,YAAY,IAAI,IAAI,KAAKD,QAAQ,QAAQ,CAAC,EAAE,OACrF,EAAC,WAAW,MAAM,OAAM,CAC1B;KAEJ,CAAC,EACA,OAAM,MAAK,KAAK,KAAK,CAAC,CAAC;IAC5B,GAAG,WACc,EAAK,QAAQ,EAAE,OAAO,EAAC;GAC1C;GACA,UAAU,EAAE;EACd;CACF;;;;;;;CAQA,gBAAmB,UAAmC;EACpD,MAAM,IAAI,SAAY;EACtB,KAAKK,SAAS,KAAKC,YAAY,UAAU,CAAC,CAAC;EAC3C,OAAO,EAAE;CACX;;;;;;;;;;CAWA,YAAe,UAAuB,GAA4B;EAChE,OAAO;GACL,KAAK,OAAO,IAAI,IAAI,eAAe;IACjC,IAAI;IACJ,IAAI;KACF,SAAS,MAAM,SAAS,IAAI,EAAE;KAC9B,WAAW;KACX,EAAE,QAAQ,MAAM;IAClB,SAAS,GAAG;KACV,WAAW;KACX,EAAE,OAAO,CAAC;IACZ;IACA,OAAO,EAAC,SAAS,KAAI;GACvB;GACA,UAAU,EAAE;EACd;CACF;CAEA,SAAS,QAA0B;EACjC,OAAO,CAAC,KAAKF,OAAO,kBAAkB;EACtC,IAAI,KAAKD,UAAU;GACjB,OAAO,SAAS,KAAKA,QAAQ;GAC7B;EACF;EAEA,KAAKb,OAAO,QAAQ,MAAM;EAG1B,IAAI,KAAKO,cAAc,KAAKJ;OACN,KAAKH,OAAO,KAE5B,IAAc,KAAKO,cAAc,KAAKK,aAAa;IACrD,KAAKJ,OAAO,KAAKC,WAAW,KAAKD,GAAG;IACpC,KAAKD;IACL,KAAKF,IAAI,QAAQ,0BAA0B,KAAKE,aAAa;GAC/D;;CAEJ;;;;;CAMA,QAAQ;EACN,KAAK,KAAK,IAAI,eAAe,CAAC;CAChC;;;;;CAMA,UAAU;EACR,OAAO,CAAC,KAAKO,OAAO,kBAAkB;EACtC,KAAKA,QAAQ;EAEb,KAAK,IAAI,IAAI,GAAG,IAAI,KAAKP,aAAa,KACpC,KAAKP,OAAO,QAAQ,MAAM;CAE9B;CAEA,YAAqB;EACnB,OAAO,KAAKQ,QAAQ,KAAA,KAAa,CAAC,KAAKM,SAAS,KAAKD,aAAa,KAAA;CACpE;;;;CAKA,KAAK,KAAc;EACjB,IAAI,CAAC,KAAKC,SAAS,CAAC,KAAKD,UAAU;GACjC,KAAKA,WAAW,YAAY,GAAG;GAG/B,MAAM,QACJ,KAAKA,oBAAoB,mBAAmB,UAAU;GACxD,KAAKR,IAAI,SAAS,KAAKQ,QAAQ;GAE/B,KAAK,IAAI,IAAI,GAAG,IAAI,KAAKN,aAAa,KAEpC,KAAKP,OAAO,QAAQ,KAAKa,QAAQ;EAErC;CACF;AACF;;;;;;AA8JA,SAAgB,eAAe,YAG7B;CACA,MAAM,WAAW,IAAI,MAAY;CAEjC,OAAO;EACL,OAAM,OAAM;GACV,MAAM,OAAO,GAAG,OAAO,6BAA6B,WAAW,EAAE;GACjE,KAAK,WACG,SAAS,QAAQ,IACvB,QAAO,SAAS,iBAAiB,GAAG,CACtC;GACA,OAAO,CAAC,IAAI;EACd;EAEA;CACF;AACF;;;;;;;AAQA,IAAa,mBAAb,cAAsC,MAAM;CAC1C,YAAY,OAAiB;EAC3B,MAAM;EACN,KAAK,QAAQ;CACf;AACF;;;;;;;;;;;AAYA,IAAM,iBAAN,cAA6B,iBAAiB;CAC5C,OAAgB;CAChB,UAAmB;AACrB;AAEA,SAAS,YAAY,KAAqB;CACxC,IAAI,eAAe,OACjB,OAAO;CAET,MAAM,wBAAQ,IAAI,MAAM;CACxB,MAAM,QAAQ;CACd,OAAO;AACT;AAqCA,IAAM,kBAAkB;AAUxB,IAAM,uBAAuB,SAC3B,QAAQ,IAAI,sCAAsC,MACpD;AAEA,IAAM,kBAAwB,IAAI,OAAO;CACvC,GAAG,QAAQ,sBAAsB;CACjC,OAAO,CAAC,EAAE,WAAW,OAAO,CAAC;AAC/B;AAaA,IAAa,gBAA8B;CACzC,mBAAmB;EACjB,WAAW;EACX,MAAM;CACR;CACA,iBAAiB;EACf,WAAW;EACX,MAAM;CACR;AACF"}
+{"version":3,"file":"transaction-pool.js","names":["#mode","#init","#cleanup","#tasks","#workers","#initialWorkers","#maxWorkers","#timeoutTask","#lc","#stmtRunner","#numWorkers","#db","#addWorker","#start","#stmts","#numWorking","#failure","#done","#process","#readRunner"],"sources":["../../../../../zero-cache/src/db/transaction-pool.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {type Resolver, resolver} from '@rocicorp/resolver';\nimport type postgres from 'postgres';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport type {Enum} from '../../../shared/src/enum.ts';\nimport {Queue} from '../../../shared/src/queue.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {type PostgresDB, type PostgresTransaction} from '../types/pg.ts';\nimport type * as Mode from './mode-enum.ts';\nimport {runTx} from './run-transaction.ts';\n\ntype Mode = Enum<typeof Mode>;\n\ntype MaybePromise<T> = Promise<T> | T;\n\nexport type Statement =\n  | postgres.PendingQuery<(postgres.Row & Iterable<postgres.Row>)[]>\n  | postgres.PendingQuery<postgres.Row[]>;\n\n/**\n * A {@link Task} is logic run from within a transaction in a {@link TransactionPool}.\n * It returns a list of `Statements` that the transaction executes asynchronously and\n * awaits when it receives the 'done' signal.\n *\n */\nexport type Task = (\n  tx: PostgresTransaction,\n  lc: LogContext,\n) => MaybePromise<Statement[]>;\n\n/**\n * A {@link ReadTask} is run from within a transaction, but unlike a {@link Task},\n * the results of a ReadTask are opaque to the TransactionPool and returned to the\n * caller of {@link TransactionPool.processReadTask}.\n */\nexport type ReadTask<T> = (\n  tx: PostgresTransaction,\n  lc: LogContext,\n) => MaybePromise<T>;\n\nexport type Options = {\n  mode: Mode;\n\n  /**\n   * A {@link Task} that is run in each Transaction before it begins\n   * processing general tasks. This can be used to to set the transaction\n   * mode, export/set snapshots, etc. This will be run even if\n   * {@link fail} has been called on the pool.\n   */\n  init?: Task | undefined;\n\n  /**\n   * A {@link Task} that is run in each Transaction before it closes.\n   * This will be run even if {@link fail} has been called, or if a\n   * preceding Task threw an Error.\n   */\n  cleanup?: Task | undefined;\n\n  /**\n   * The initial number of transaction workers to process tasks.\n   * This is the steady state number of workers that will be kept\n   * alive if the TransactionPool is long lived.\n   * This must be greater than 0. Defaults to 1.\n   */\n  initialWorkers?: number;\n\n  /**\n   * When specified, allows the pool to grow to `maxWorkers`. This\n   * must be greater than or equal to `initialWorkers`. On-demand\n   * workers will be shut down after an idle timeout of 5 seconds.\n   */\n  maxWorkers?: number;\n};\n\n/**\n * A TransactionPool is a pool of one or more {@link postgres.TransactionSql}\n * objects that participate in processing a dynamic queue of tasks.\n *\n * This can be used for serializing a set of tasks that arrive asynchronously\n * to a single transaction (for writing) or performing parallel reads across\n * multiple connections at the same snapshot (e.g. read only snapshot transactions).\n */\nexport class TransactionPool {\n  #lc: LogContext;\n  readonly #mode: Mode;\n  readonly #init: TaskRunner | undefined;\n  readonly #cleanup: TaskRunner | undefined;\n  readonly #tasks = new Queue<TaskRunner | Error | 'done'>();\n  readonly #workers: Promise<unknown>[] = [];\n  readonly #initialWorkers: number;\n  readonly #maxWorkers: number;\n  readonly #timeoutTask: TimeoutTasks;\n  #numWorkers: number;\n  #numWorking = 0;\n  #db: PostgresDB | undefined; // set when running. stored to allow adaptive pool sizing.\n\n  #done = false;\n  #failure: Error | undefined;\n\n  constructor(\n    lc: LogContext,\n    opts: Options,\n    timeoutTasks = TIMEOUT_TASKS, // Overridden for tests.\n  ) {\n    const {\n      mode,\n      init,\n      cleanup,\n      initialWorkers = 1,\n      maxWorkers = initialWorkers,\n    } = opts;\n    assert(initialWorkers > 0, 'initialWorkers must be positive');\n    assert(\n      maxWorkers >= initialWorkers,\n      'maxWorkers must be >= initialWorkers',\n    );\n\n    this.#lc = lc;\n    this.#mode = mode;\n    this.#init = init ? this.#stmtRunner(init) : undefined;\n    this.#cleanup = cleanup ? this.#stmtRunner(cleanup) : undefined;\n    this.#initialWorkers = initialWorkers;\n    this.#numWorkers = initialWorkers;\n    this.#maxWorkers = maxWorkers;\n    this.#timeoutTask = timeoutTasks;\n  }\n\n  /**\n   * Starts the pool of workers to process Tasks with transactions opened from the\n   * specified {@link db}.\n   */\n  run(db: PostgresDB): this {\n    assert(!this.#db, 'already running');\n    this.#db = db;\n    for (let i = 0; i < this.#numWorkers; i++) {\n      this.#addWorker(db);\n    }\n    return this;\n  }\n\n  /**\n   * Adds context parameters to internal LogContext. This is useful for context values that\n   * are not known when the TransactionPool is constructed (e.g. determined after a database\n   * call when the pool is running).\n   *\n   * Returns an object that can be used to add more parameters.\n   */\n  addLoggingContext(key: string, value: string) {\n    this.#lc = this.#lc.withContext(key, value);\n\n    return {\n      addLoggingContext: (key: string, value: string) =>\n        this.addLoggingContext(key, value),\n    };\n  }\n\n  /**\n   * Returns a promise that:\n   *\n   * * resolves after {@link setDone} has been called (or the the pool as been {@link unref}ed\n   *   to a 0 ref count), once all added tasks have been processed and all transactions have been\n   *   committed or closed.\n   *\n   * * rejects if processing was aborted with {@link fail} or if processing any of\n   *   the tasks resulted in an error. All uncommitted transactions will have been\n   *   rolled back.\n   *\n   * Note that partial failures are possible if processing writes with multiple workers\n   * (e.g. `setDone` is called, allowing some workers to commit, after which other\n   *  workers encounter errors). Using a TransactionPool in this manner does not make\n   * sense in terms of transactional semantics, and is thus not recommended.\n   *\n   * For reads, however, multiple workers is useful for performing parallel reads\n   * at the same snapshot. See {@link synchronizedSnapshots} for an example.\n   * Resolves or rejects when all workers are done or failed.\n   */\n  async done() {\n    const numWorkers = this.#workers.length;\n    await Promise.all(this.#workers);\n\n    if (numWorkers < this.#workers.length) {\n      // If workers were added after the initial set, they must be awaited to ensure\n      // that the results (i.e. rejections) of all workers are accounted for. This only\n      // needs to be re-done once, because the fact that the first `await` completed\n      // guarantees that the pool is in a terminal state and no new workers can be added.\n      await Promise.all(this.#workers);\n    }\n    this.#lc.debug?.('transaction pool done');\n\n    const elapsed = performance.now() - this.#start;\n    if (elapsed > 60_000) {\n      if (this.#stmts > 0) {\n        this.#lc.warn?.(\n          `finished long transaction with ${this.#stmts} statements (${elapsed.toFixed(3)} ms)`,\n        );\n      } else {\n        this.#lc.warn?.(\n          `finished long read transaction (${elapsed.toFixed(3)} ms)`,\n        );\n      }\n    }\n  }\n\n  #addWorker(db: PostgresDB) {\n    const id = this.#workers.length + 1;\n    const lc = this.#lc.withContext('tx', id);\n\n    const tt: TimeoutTask =\n      this.#workers.length < this.#initialWorkers\n        ? this.#timeoutTask.forInitialWorkers\n        : this.#timeoutTask.forExtraWorkers;\n    const {timeoutMs} = tt;\n    const timeoutTask = tt.task === 'done' ? 'done' : this.#stmtRunner(tt.task);\n\n    const worker = async (tx: PostgresTransaction) => {\n      const start = performance.now();\n      try {\n        lc.debug?.('started transaction');\n\n        let last: Promise<void> = promiseVoid;\n\n        const executeTask = async (runner: TaskRunner) => {\n          runner !== this.#init && this.#numWorking++;\n          const {pending} = await runner.run(tx, lc, () => {\n            runner !== this.#init && this.#numWorking--;\n          });\n          last = pending ?? last;\n        };\n\n        let task: TaskRunner | Error | 'done' =\n          this.#init ?? (await this.#tasks.dequeue(timeoutTask, timeoutMs));\n\n        try {\n          while (task !== 'done') {\n            if (\n              task instanceof Error ||\n              (task !== this.#init && this.#failure)\n            ) {\n              throw this.#failure ?? task;\n            }\n            await executeTask(task);\n\n            // await the next task.\n            task = await this.#tasks.dequeue(timeoutTask, timeoutMs);\n          }\n        } finally {\n          // Execute the cleanup task even on failure.\n          if (this.#cleanup) {\n            await executeTask(this.#cleanup);\n          }\n        }\n\n        const elapsed = performance.now() - start;\n        lc.debug?.(`closing transaction (${elapsed.toFixed(3)} ms)`);\n        // Given the semantics of a Postgres transaction, the last statement\n        // will only succeed if all of the preceding statements succeeded.\n        return last;\n      } catch (e) {\n        if (e !== this.#failure) {\n          this.fail(e); // A failure in any worker should fail the pool.\n        }\n        throw e;\n      }\n    };\n\n    const workerTx = runTx(db, worker, {mode: this.#mode})\n      .catch(e => {\n        if (e instanceof RollbackSignal) {\n          // A RollbackSignal is used to gracefully rollback the postgres.js\n          // transaction block. It should not be thrown up to the application.\n          lc.debug?.('aborted transaction');\n        } else {\n          throw e;\n        }\n      })\n      .finally(() => this.#numWorkers--);\n\n    // Attach a rejection handler immediately to prevent unhandledRejections.\n    // The application will handle errors when it awaits processReadTask()\n    // or done().\n    workerTx.catch(() => {});\n\n    this.#workers.push(workerTx);\n\n    // After adding the worker, enqueue a terminal signal if we are in either of the\n    // terminal states (both of which prevent more tasks from being enqueued), to ensure\n    // that the added worker eventually exits.\n    if (this.#done) {\n      this.#tasks.enqueue('done');\n    }\n    if (this.#failure) {\n      this.#tasks.enqueue(this.#failure);\n    }\n  }\n\n  /**\n   * Processes the statements produced by the specified {@link Task},\n   * returning a Promise that resolves when the statements are either processed\n   * by the database or rejected.\n   *\n   * Note that statement failures will result in failing the entire\n   * TransactionPool (per transaction semantics). However, the returned Promise\n   * itself will resolve rather than reject. As such, it is fine to ignore\n   * returned Promises in order to pipeline requests to the database. It is\n   * recommended to occasionally await them (e.g. after some threshold) in\n   * order to avoid memory blowup in the case of database slowness.\n   */\n  process(task: Task): Promise<void> {\n    const r = resolver<void>();\n    this.#process(this.#stmtRunner(task, r));\n    return r.promise;\n  }\n\n  readonly #start = performance.now();\n  #stmts = 0;\n\n  /**\n   * Implements the semantics specified in {@link process()}.\n   *\n   * Specifically:\n   * * `freeWorker()` is called as soon as the statements are produced,\n   *   allowing them to be pipelined to the database.\n   * * Statement errors result in failing the transaction pool.\n   * * The client-supplied Resolver resolves on success or failure;\n   *   it is never rejected.\n   */\n  #stmtRunner(task: Task, r: {resolve: () => void} = resolver()): TaskRunner {\n    return {\n      run: async (tx, lc, freeWorker) => {\n        let stmts: Statement[];\n        try {\n          stmts = await task(tx, lc);\n        } catch (e) {\n          r.resolve();\n          throw e;\n        } finally {\n          freeWorker();\n        }\n\n        if (stmts.length === 0) {\n          r.resolve();\n          return {pending: null};\n        }\n\n        // Execute the statements (i.e. send to the db) immediately.\n        // The last result is returned for the worker to await before\n        // closing the transaction.\n        const last = stmts.reduce((_, stmt) => {\n          const query = stmt as unknown as Query;\n          const index = ++this.#stmts;\n          return stmt\n            .execute()\n            .then(() => {\n              if (index % 1000 === 0) {\n                const log = index % 10000 === 0 ? 'info' : 'debug';\n                lc[log]?.(\n                  `executed ${this.#stmts}th statement (${(performance.now() - this.#start).toFixed(3)} ms)`,\n                  {statement: query.string},\n                );\n              }\n            })\n            .catch(e => this.fail(e));\n        }, promiseVoid);\n        return {pending: last.finally(r.resolve)};\n      },\n      rejected: r.resolve,\n    };\n  }\n\n  /**\n   * Processes and returns the result of executing the {@link ReadTask} from\n   * within the transaction. An error thrown by the task will result in\n   * rejecting the returned Promise, but will not affect the transaction pool\n   * itself.\n   */\n  processReadTask<T>(readTask: ReadTask<T>): Promise<T> {\n    const r = resolver<T>();\n    this.#process(this.#readRunner(readTask, r));\n    return r.promise;\n  }\n\n  /**\n   * Implements the semantics specified in {@link processReadTask()}.\n   *\n   * Specifically:\n   * * `freeWorker()` is called as soon as the result is produced,\n   *   before resolving the client-supplied Resolver.\n   * * Errors result in rejecting the client-supplied Resolver but\n   *   do not affect transaction pool.\n   */\n  #readRunner<T>(readTask: ReadTask<T>, r: Resolver<T>): TaskRunner {\n    return {\n      run: async (tx, lc, freeWorker) => {\n        let result: T;\n        try {\n          result = await readTask(tx, lc);\n          freeWorker();\n          r.resolve(result);\n        } catch (e) {\n          freeWorker();\n          r.reject(e);\n        }\n        return {pending: null};\n      },\n      rejected: r.reject,\n    };\n  }\n\n  #process(runner: TaskRunner): void {\n    assert(!this.#done, 'already set done');\n    if (this.#failure) {\n      runner.rejected(this.#failure);\n      return;\n    }\n\n    this.#tasks.enqueue(runner);\n\n    // Check if the pool size can and should be increased.\n    if (this.#numWorkers < this.#maxWorkers) {\n      const outstanding = this.#tasks.size();\n\n      if (outstanding > this.#numWorkers - this.#numWorking) {\n        this.#db && this.#addWorker(this.#db);\n        this.#numWorkers++;\n        this.#lc.debug?.(`Increased pool size to ${this.#numWorkers}`);\n      }\n    }\n  }\n\n  /**\n   * Ends all workers with a ROLLBACK. Throws if the pool is already done\n   * or aborted.\n   */\n  abort() {\n    this.fail(new RollbackSignal());\n  }\n\n  /**\n   * Signals to all workers to end their transaction once all pending tasks have\n   * been completed. Throws if the pool is already done or aborted.\n   */\n  setDone() {\n    assert(!this.#done, 'already set done');\n    this.#done = true;\n\n    for (let i = 0; i < this.#numWorkers; i++) {\n      this.#tasks.enqueue('done');\n    }\n  }\n\n  isRunning(): boolean {\n    return this.#db !== undefined && !this.#done && this.#failure === undefined;\n  }\n\n  /**\n   * Signals all workers to fail their transactions with the given {@link err}.\n   */\n  fail(err: unknown) {\n    if (!this.#done && !this.#failure) {\n      this.#failure = ensureError(err); // Fail fast: this is checked in the worker loop.\n      // Logged for informational purposes. It is the responsibility of\n      // higher level logic to classify and handle the exception.\n      const level =\n        this.#failure instanceof ControlFlowError ? 'debug' : 'info';\n      this.#lc[level]?.(this.#failure);\n\n      for (let i = 0; i < this.#numWorkers; i++) {\n        // Enqueue the Error to terminate any workers waiting for tasks.\n        this.#tasks.enqueue(this.#failure);\n      }\n    }\n  }\n}\n\ntype SynchronizeSnapshotTasks = {\n  /**\n   * The `init` Task for the TransactionPool from which the snapshot originates.\n   * The pool must have Mode.SERIALIZABLE, and will be set to READ ONLY by the\n   * `exportSnapshot` init task. If the TransactionPool has multiple workers, the\n   * first worker will export a snapshot that the others set.\n   */\n  exportSnapshot: Task;\n\n  /**\n   * The `cleanup` Task for the TransactionPool from which the snapshot\n   * originates. This Task will wait for the follower pool to `setSnapshot`\n   * to ensure that the snapshot is successfully shared before the originating\n   * transaction is closed.\n   */\n  cleanupExport: Task;\n\n  /**\n   * The `init` Task for the TransactionPool in which workers will\n   * consequently see the same snapshot as that of the first pool. The pool\n   * must have Mode.SERIALIZABLE, and will have the ability to perform writes.\n   */\n  setSnapshot: Task;\n\n  /** The ID of the shared snapshot. */\n  snapshotID: Promise<string>;\n};\n\n/**\n * Init Tasks for Postgres snapshot synchronization across transactions.\n *\n * https://www.postgresql.org/docs/9.3/functions-admin.html#:~:text=Snapshot%20Synchronization%20Functions,identical%20content%20in%20the%20database.\n */\nexport function synchronizedSnapshots(): SynchronizeSnapshotTasks {\n  const {\n    promise: snapshotExported,\n    resolve: exportSnapshot,\n    reject: failExport,\n  } = resolver<string>();\n\n  const {\n    promise: snapshotCaptured,\n    resolve: captureSnapshot,\n    reject: failCapture,\n  } = resolver<unknown>();\n\n  // Set by the first worker to run its initTask, who becomes responsible for\n  // exporting the snapshot. TODO: Plumb the workerNum and use that instead.\n  let firstWorkerRun = false;\n\n  // Note: Neither init task should `await`, as processing in each pool can proceed\n  //       as soon as the statements have been sent to the db. However, the `cleanupExport`\n  //       task must `await` the result of `setSnapshot` to ensure that exporting transaction\n  //       does not close before the snapshot has been captured.\n  return {\n    exportSnapshot: tx => {\n      if (!firstWorkerRun) {\n        firstWorkerRun = true;\n        const stmt =\n          tx`SELECT pg_export_snapshot() AS snapshot; SET TRANSACTION READ ONLY;`.simple();\n        // Intercept the promise to propagate the information to `snapshotExported`.\n        stmt.then(result => exportSnapshot(result[0].snapshot), failExport);\n        return [stmt]; // Also return the stmt so that it gets awaited (and errors handled).\n      }\n      return snapshotExported.then(snapshotID => [\n        tx.unsafe(`SET TRANSACTION SNAPSHOT '${snapshotID}'`),\n        tx`SET TRANSACTION READ ONLY`.simple(),\n      ]);\n    },\n\n    setSnapshot: tx =>\n      snapshotExported.then(snapshotID => {\n        const stmt = tx.unsafe(`SET TRANSACTION SNAPSHOT '${snapshotID}'`);\n        // Intercept the promise to propagate the information to `cleanupExport`.\n        stmt.then(captureSnapshot, failCapture);\n        return [stmt];\n      }),\n\n    cleanupExport: async () => {\n      await snapshotCaptured;\n      return [];\n    },\n\n    snapshotID: snapshotExported,\n  };\n}\n\n/**\n * Returns `init` and `cleanup` {@link Task}s for a TransactionPool that ensure its workers\n * share a single view of the database. This is used for View Notifier and View Syncer logic\n * that allows multiple entities to perform parallel reads on the same snapshot of the database.\n */\nexport function sharedSnapshot(): {\n  init: Task;\n  cleanup: Task;\n  snapshotID: Promise<string>;\n} {\n  const {\n    promise: snapshotExported,\n    resolve: exportSnapshot,\n    reject: failExport,\n  } = resolver<string>();\n\n  // Set by the first worker to run its initTask, who becomes responsible for\n  // exporting the snapshot.\n  let firstWorkerRun = false;\n\n  // The LogContext of the exporting worker, used to identify its cleanup call.\n  // Each worker receives a unique lc instance (via withContext('tx', id)), so\n  // reference equality reliably identifies the exporting worker.\n  let exporterLc: LogContext | undefined;\n\n  // Set when the exporting worker's cleanup runs, signalling that the snapshot\n  // is no longer needed and any subsequently spawned workers should skip their\n  // initTask.\n  let firstWorkerDone = false;\n\n  return {\n    init: (tx, lc) => {\n      if (!firstWorkerRun) {\n        firstWorkerRun = true;\n        exporterLc = lc; // Remember which worker is the exporter.\n        const stmt = tx`SELECT pg_export_snapshot() AS snapshot;`.simple();\n        // Intercept the promise to propagate the information to `snapshotExported`.\n        stmt.then(result => exportSnapshot(result[0].snapshot), failExport);\n        return [stmt]; // Also return the stmt so that it gets awaited (and errors handled).\n      }\n      if (!firstWorkerDone) {\n        return snapshotExported.then(snapshotID => [\n          tx.unsafe(`SET TRANSACTION SNAPSHOT '${snapshotID}'`),\n        ]);\n      }\n      lc.debug?.('All work is done. No need to set snapshot');\n      return [];\n    },\n\n    cleanup: (_tx, lc) => {\n      // Only the exporting worker's cleanup should disable snapshot-setting.\n      // Non-exporter workers may finish early; letting them flip this flag\n      // would cause subsequently spawned workers to skip SET TRANSACTION SNAPSHOT\n      // and read a newer database view, violating snapshot isolation.\n      if (lc === exporterLc) {\n        firstWorkerDone = true;\n      }\n      return [];\n    },\n\n    snapshotID: snapshotExported,\n  };\n}\n\n/**\n * @returns An `init` Task for importing a snapshot from another transaction,\n *          as well as an `imported` Queue which resolves or rejects when each\n *          worker is initialized.\n */\nexport function importSnapshot(snapshotID: string): {\n  init: Task;\n  imported: Queue<void>;\n} {\n  const imported = new Queue<void>();\n\n  return {\n    init: tx => {\n      const stmt = tx.unsafe(`SET TRANSACTION SNAPSHOT '${snapshotID}'`);\n      stmt.then(\n        () => imported.enqueue(),\n        err => imported.enqueueRejection(err),\n      );\n      return [stmt];\n    },\n\n    imported,\n  };\n}\n\n/**\n * A superclass of Errors used for control flow that is needed to handle\n * another Error but does not constitute an error condition itself (e.g.\n * aborting transactions after a previous one fails). Subclassing this Error\n * will result in lowering the log level from `error` to `debug`.\n */\nexport class ControlFlowError extends Error {\n  constructor(cause?: unknown) {\n    super();\n    this.cause = cause;\n  }\n}\n\n/**\n * Internal error used to rollback the worker transaction. This is used\n * instead of executing a `ROLLBACK` statement because the postgres.js\n * library will otherwise try to execute an extraneous `COMMIT`, which\n * results in outputting a \"no transaction in progress\" warning to the\n * database logs.\n *\n * Throwing an exception, on the other hand, executes the postgres.js\n * codepath that calls `ROLLBACK` instead.\n */\nclass RollbackSignal extends ControlFlowError {\n  readonly name = 'RollbackSignal';\n  readonly message = 'rolling back transaction';\n}\n\nfunction ensureError(err: unknown): Error {\n  if (err instanceof Error) {\n    return err;\n  }\n  const error = new Error();\n  error.cause = err;\n  return error;\n}\n\ninterface TaskRunner {\n  /**\n   * Manages the running of a Task or ReadTask in two phases:\n   *\n   * - If the task involves blocking, this is done in the worker. Once the\n   *   blocking is done, `freeWorker()` is invoked to signal that the worker\n   *   is available to run another task. Note that this should be invoked\n   *   *before* resolving the result to the calling thread so that a\n   *   subsequent task can reuse the same worker.\n   *\n   * - Task statements are executed on the database asynchronously. The final\n   *   result of this processing is encapsulated in the returned `pending`\n   *   Promise. The worker will await the last pending Promise before closing\n   *   the transaction.\n   *\n   * @param freeWorker should be called as soon as all blocking operations are\n   *             completed in order to return the transaction to the pool.\n   * @returns A `pending` Promise indicating when the statements have been\n   *          processed by the database, allowing the transaction to be closed.\n   *          This should be `null` if there are no transaction-dependent\n   *          statements to await.\n   */\n  run(\n    tx: PostgresTransaction,\n    lc: LogContext,\n    freeWorker: () => void,\n  ): Promise<{pending: Promise<void> | null}>;\n\n  /**\n   * Invoked if the TransactionPool is already in a failed state when the task\n   * is requested.\n   */\n  rejected(reason: unknown): void;\n}\n\nconst IDLE_TIMEOUT_MS = 5_000;\n\n// The keepalive interval is settable by ZERO_TRANSACTION_POOL_KEEPALIVE_MS\n// as an emergency measure and is explicitly not made available as a server\n// option. This value is function of how the zero-cache uses transactions, and\n// should never need to be \"tuned\" or adjusted for different environments.\n//\n// Note that it must be shorter than IDLE_IN_TRANSACTION_SESSION_TIMEOUT_MS\n// with sufficient buffering to account for when the process is blocked by\n// synchronous calls (e.g. to the replica).\nconst KEEPALIVE_TIMEOUT_MS = parseInt(\n  process.env.ZERO_TRANSACTION_POOL_KEEPALIVE_MS ?? '5000',\n);\n\nconst KEEPALIVE_TASK: Task = (tx, lc) => {\n  lc.debug?.(`sending tx keepalive`);\n  return [tx`SELECT 1`.simple()];\n};\n\ntype TimeoutTask = {\n  timeoutMs: number;\n  task: Task | 'done';\n};\n\ntype TimeoutTasks = {\n  forInitialWorkers: TimeoutTask;\n  forExtraWorkers: TimeoutTask;\n};\n\n// Production timeout tasks. Overridden in tests.\nexport const TIMEOUT_TASKS: TimeoutTasks = {\n  forInitialWorkers: {\n    timeoutMs: KEEPALIVE_TIMEOUT_MS,\n    task: KEEPALIVE_TASK,\n  },\n  forExtraWorkers: {\n    timeoutMs: IDLE_TIMEOUT_MS,\n    task: 'done',\n  },\n};\n\n// The slice of information from the Query object in Postgres.js that gets logged for debugging.\n// https://github.com/porsager/postgres/blob/f58cd4f3affd3e8ce8f53e42799672d86cd2c70b/src/connection.js#L219\ntype Query = {string: string; parameters: object[]};\n"],"mappings":";;;;;;;;;;;;;;;AAkFA,IAAa,kBAAb,MAA6B;CAC3B;CACA;CACA;CACA;CACA,SAAkB,IAAI,MAAmC;CACzD,WAAwC,CAAC;CACzC;CACA;CACA;CACA;CACA,cAAc;CACd;CAEA,QAAQ;CACR;CAEA,YACE,IACA,MACA,eAAe,eACf;EACA,MAAM,EACJ,MACA,MACA,SACA,iBAAiB,GACjB,aAAa,mBACX;EACJ,OAAO,iBAAiB,GAAG,iCAAiC;EAC5D,OACE,cAAc,gBACd,sCACF;EAEA,KAAKQ,MAAM;EACX,KAAKR,QAAQ;EACb,KAAKC,QAAQ,OAAO,KAAKQ,YAAY,IAAI,IAAI,KAAA;EAC7C,KAAKP,WAAW,UAAU,KAAKO,YAAY,OAAO,IAAI,KAAA;EACtD,KAAKJ,kBAAkB;EACvB,KAAKK,cAAc;EACnB,KAAKJ,cAAc;EACnB,KAAKC,eAAe;CACtB;;;;;CAMA,IAAI,IAAsB;EACxB,OAAO,CAAC,KAAKI,KAAK,iBAAiB;EACnC,KAAKA,MAAM;EACX,KAAK,IAAI,IAAI,GAAG,IAAI,KAAKD,aAAa,KACpC,KAAKE,WAAW,EAAE;EAEpB,OAAO;CACT;;;;;;;;CASA,kBAAkB,KAAa,OAAe;EAC5C,KAAKJ,MAAM,KAAKA,IAAI,YAAY,KAAK,KAAK;EAE1C,OAAO,EACL,oBAAoB,KAAa,UAC/B,KAAK,kBAAkB,KAAK,KAAK,EACrC;CACF;;;;;;;;;;;;;;;;;;;;;CAsBA,MAAM,OAAO;EACX,MAAM,aAAa,KAAKJ,SAAS;EACjC,MAAM,QAAQ,IAAI,KAAKA,QAAQ;EAE/B,IAAI,aAAa,KAAKA,SAAS,QAK7B,MAAM,QAAQ,IAAI,KAAKA,QAAQ;EAEjC,KAAKI,IAAI,QAAQ,uBAAuB;EAExC,MAAM,UAAU,YAAY,IAAI,IAAI,KAAKK;EACzC,IAAI,UAAU,KACZ,IAAI,KAAKC,SAAS,GAChB,KAAKN,IAAI,OACP,kCAAkC,KAAKM,OAAO,eAAe,QAAQ,QAAQ,CAAC,EAAE,KAClF;OAEA,KAAKN,IAAI,OACP,mCAAmC,QAAQ,QAAQ,CAAC,EAAE,KACxD;CAGN;CAEA,WAAW,IAAgB;EACzB,MAAM,KAAK,KAAKJ,SAAS,SAAS;EAClC,MAAM,KAAK,KAAKI,IAAI,YAAY,MAAM,EAAE;EAExC,MAAM,KACJ,KAAKJ,SAAS,SAAS,KAAKC,kBACxB,KAAKE,aAAa,oBAClB,KAAKA,aAAa;EACxB,MAAM,EAAC,cAAa;EACpB,MAAM,cAAc,GAAG,SAAS,SAAS,SAAS,KAAKE,YAAY,GAAG,IAAI;EAE1E,MAAM,SAAS,OAAO,OAA4B;GAChD,MAAM,QAAQ,YAAY,IAAI;GAC9B,IAAI;IACF,GAAG,QAAQ,qBAAqB;IAEhC,IAAI,OAAsB;IAE1B,MAAM,cAAc,OAAO,WAAuB;KAChD,WAAW,KAAKR,SAAS,KAAKc;KAC9B,MAAM,EAAC,YAAW,MAAM,OAAO,IAAI,IAAI,UAAU;MAC/C,WAAW,KAAKd,SAAS,KAAKc;KAChC,CAAC;KACD,OAAO,WAAW;IACpB;IAEA,IAAI,OACF,KAAKd,SAAU,MAAM,KAAKE,OAAO,QAAQ,aAAa,SAAS;IAEjE,IAAI;KACF,OAAO,SAAS,QAAQ;MACtB,IACE,gBAAgB,SACf,SAAS,KAAKF,SAAS,KAAKe,UAE7B,MAAM,KAAKA,YAAY;MAEzB,MAAM,YAAY,IAAI;MAGtB,OAAO,MAAM,KAAKb,OAAO,QAAQ,aAAa,SAAS;KACzD;IACF,UAAU;KAER,IAAI,KAAKD,UACP,MAAM,YAAY,KAAKA,QAAQ;IAEnC;IAEA,MAAM,UAAU,YAAY,IAAI,IAAI;IACpC,GAAG,QAAQ,wBAAwB,QAAQ,QAAQ,CAAC,EAAE,KAAK;IAG3D,OAAO;GACT,SAAS,GAAG;IACV,IAAI,MAAM,KAAKc,UACb,KAAK,KAAK,CAAC;IAEb,MAAM;GACR;EACF;EAEA,MAAM,WAAW,MAAM,IAAI,QAAQ,EAAC,MAAM,KAAKhB,MAAK,CAAC,EAClD,OAAM,MAAK;GACV,IAAI,aAAa,gBAGf,GAAG,QAAQ,qBAAqB;QAEhC,MAAM;EAEV,CAAC,EACA,cAAc,KAAKU,aAAa;EAKnC,SAAS,YAAY,CAAC,CAAC;EAEvB,KAAKN,SAAS,KAAK,QAAQ;EAK3B,IAAI,KAAKa,OACP,KAAKd,OAAO,QAAQ,MAAM;EAE5B,IAAI,KAAKa,UACP,KAAKb,OAAO,QAAQ,KAAKa,QAAQ;CAErC;;;;;;;;;;;;;CAcA,QAAQ,MAA2B;EACjC,MAAM,IAAI,SAAe;EACzB,KAAKE,SAAS,KAAKT,YAAY,MAAM,CAAC,CAAC;EACvC,OAAO,EAAE;CACX;CAEA,SAAkB,YAAY,IAAI;CAClC,SAAS;;;;;;;;;;;CAYT,YAAY,MAAY,IAA2B,SAAS,GAAe;EACzE,OAAO;GACL,KAAK,OAAO,IAAI,IAAI,eAAe;IACjC,IAAI;IACJ,IAAI;KACF,QAAQ,MAAM,KAAK,IAAI,EAAE;IAC3B,SAAS,GAAG;KACV,EAAE,QAAQ;KACV,MAAM;IACR,UAAU;KACR,WAAW;IACb;IAEA,IAAI,MAAM,WAAW,GAAG;KACtB,EAAE,QAAQ;KACV,OAAO,EAAC,SAAS,KAAI;IACvB;IAqBA,OAAO,EAAC,SAhBK,MAAM,QAAQ,GAAG,SAAS;KACrC,MAAM,QAAQ;KACd,MAAM,QAAQ,EAAE,KAAKK;KACrB,OAAO,KACJ,QAAQ,EACR,WAAW;MACV,IAAI,QAAQ,QAAS,GAEnB,GADY,QAAQ,QAAU,IAAI,SAAS,WAEzC,YAAY,KAAKA,OAAO,iBAAiB,YAAY,IAAI,IAAI,KAAKD,QAAQ,QAAQ,CAAC,EAAE,OACrF,EAAC,WAAW,MAAM,OAAM,CAC1B;KAEJ,CAAC,EACA,OAAM,MAAK,KAAK,KAAK,CAAC,CAAC;IAC5B,GAAG,WACc,EAAK,QAAQ,EAAE,OAAO,EAAC;GAC1C;GACA,UAAU,EAAE;EACd;CACF;;;;;;;CAQA,gBAAmB,UAAmC;EACpD,MAAM,IAAI,SAAY;EACtB,KAAKK,SAAS,KAAKC,YAAY,UAAU,CAAC,CAAC;EAC3C,OAAO,EAAE;CACX;;;;;;;;;;CAWA,YAAe,UAAuB,GAA4B;EAChE,OAAO;GACL,KAAK,OAAO,IAAI,IAAI,eAAe;IACjC,IAAI;IACJ,IAAI;KACF,SAAS,MAAM,SAAS,IAAI,EAAE;KAC9B,WAAW;KACX,EAAE,QAAQ,MAAM;IAClB,SAAS,GAAG;KACV,WAAW;KACX,EAAE,OAAO,CAAC;IACZ;IACA,OAAO,EAAC,SAAS,KAAI;GACvB;GACA,UAAU,EAAE;EACd;CACF;CAEA,SAAS,QAA0B;EACjC,OAAO,CAAC,KAAKF,OAAO,kBAAkB;EACtC,IAAI,KAAKD,UAAU;GACjB,OAAO,SAAS,KAAKA,QAAQ;GAC7B;EACF;EAEA,KAAKb,OAAO,QAAQ,MAAM;EAG1B,IAAI,KAAKO,cAAc,KAAKJ;OACN,KAAKH,OAAO,KAE5B,IAAc,KAAKO,cAAc,KAAKK,aAAa;IACrD,KAAKJ,OAAO,KAAKC,WAAW,KAAKD,GAAG;IACpC,KAAKD;IACL,KAAKF,IAAI,QAAQ,0BAA0B,KAAKE,aAAa;GAC/D;;CAEJ;;;;;CAMA,QAAQ;EACN,KAAK,KAAK,IAAI,eAAe,CAAC;CAChC;;;;;CAMA,UAAU;EACR,OAAO,CAAC,KAAKO,OAAO,kBAAkB;EACtC,KAAKA,QAAQ;EAEb,KAAK,IAAI,IAAI,GAAG,IAAI,KAAKP,aAAa,KACpC,KAAKP,OAAO,QAAQ,MAAM;CAE9B;CAEA,YAAqB;EACnB,OAAO,KAAKQ,QAAQ,KAAA,KAAa,CAAC,KAAKM,SAAS,KAAKD,aAAa,KAAA;CACpE;;;;CAKA,KAAK,KAAc;EACjB,IAAI,CAAC,KAAKC,SAAS,CAAC,KAAKD,UAAU;GACjC,KAAKA,WAAW,YAAY,GAAG;GAG/B,MAAM,QACJ,KAAKA,oBAAoB,mBAAmB,UAAU;GACxD,KAAKR,IAAI,SAAS,KAAKQ,QAAQ;GAE/B,KAAK,IAAI,IAAI,GAAG,IAAI,KAAKN,aAAa,KAEpC,KAAKP,OAAO,QAAQ,KAAKa,QAAQ;EAErC;CACF;AACF;;;;;;AA8FA,SAAgB,iBAId;CACA,MAAM,EACJ,SAAS,kBACT,SAAS,gBACT,QAAQ,eACN,SAAiB;CAIrB,IAAI,iBAAiB;CAKrB,IAAI;CAKJ,IAAI,kBAAkB;CAEtB,OAAO;EACL,OAAO,IAAI,OAAO;GAChB,IAAI,CAAC,gBAAgB;IACnB,iBAAiB;IACjB,aAAa;IACb,MAAM,OAAO,EAAE,2CAA2C,OAAO;IAEjE,KAAK,MAAK,WAAU,eAAe,OAAO,GAAG,QAAQ,GAAG,UAAU;IAClE,OAAO,CAAC,IAAI;GACd;GACA,IAAI,CAAC,iBACH,OAAO,iBAAiB,MAAK,eAAc,CACzC,GAAG,OAAO,6BAA6B,WAAW,EAAE,CACtD,CAAC;GAEH,GAAG,QAAQ,2CAA2C;GACtD,OAAO,CAAC;EACV;EAEA,UAAU,KAAK,OAAO;GAKpB,IAAI,OAAO,YACT,kBAAkB;GAEpB,OAAO,CAAC;EACV;EAEA,YAAY;CACd;AACF;;;;;;AAOA,SAAgB,eAAe,YAG7B;CACA,MAAM,WAAW,IAAI,MAAY;CAEjC,OAAO;EACL,OAAM,OAAM;GACV,MAAM,OAAO,GAAG,OAAO,6BAA6B,WAAW,EAAE;GACjE,KAAK,WACG,SAAS,QAAQ,IACvB,QAAO,SAAS,iBAAiB,GAAG,CACtC;GACA,OAAO,CAAC,IAAI;EACd;EAEA;CACF;AACF;;;;;;;AAQA,IAAa,mBAAb,cAAsC,MAAM;CAC1C,YAAY,OAAiB;EAC3B,MAAM;EACN,KAAK,QAAQ;CACf;AACF;;;;;;;;;;;AAYA,IAAM,iBAAN,cAA6B,iBAAiB;CAC5C,OAAgB;CAChB,UAAmB;AACrB;AAEA,SAAS,YAAY,KAAqB;CACxC,IAAI,eAAe,OACjB,OAAO;CAET,MAAM,wBAAQ,IAAI,MAAM;CACxB,MAAM,QAAQ;CACd,OAAO;AACT;AAqCA,IAAM,kBAAkB;AAUxB,IAAM,uBAAuB,SAC3B,QAAQ,IAAI,sCAAsC,MACpD;AAEA,IAAM,kBAAwB,IAAI,OAAO;CACvC,GAAG,QAAQ,sBAAsB;CACjC,OAAO,CAAC,EAAE,WAAW,OAAO,CAAC;AAC/B;AAaA,IAAa,gBAA8B;CACzC,mBAAmB;EACjB,WAAW;EACX,MAAM;CACR;CACA,iBAAiB;EACf,WAAW;EACX,MAAM;CACR;AACF"}

package/out/zero-cache/src/server/change-streamer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"names":[],"mappings":"AAsCA,OAAO,EAIL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAU/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CA4Uf"}
+{"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"names":[],"mappings":"AAsCA,OAAO,EAIL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAU/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CA8Uf"}

package/out/zero-cache/src/server/change-streamer.js

@@ -45,7 +45,7 @@ async function runWorker(parent, env, ...argv) {
 		dbPath: replica.file
 	}, sqliteCorruptionChecks);
 	initEventSink(lc, config);
-	const changeDB = await connectPgClient(lc, change.db, "change-streamer", { max: change.maxConns }, { sendStringAsJson: true });
+	const changeDB = await connectPgClient(lc, change.db, "change-streamer-init", { max: 5 }, { sendStringAsJson: true });
 	warmupConnections(lc, changeDB, "change").catch(() => {});
 	const shard = getShardConfig(config);
 	await initChangeStreamerSchema(lc, changeDB, shard);

package/out/zero-cache/src/server/change-streamer.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {DatabaseInitError} from '../../../zqlite/src/db.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink, publishCriticalEvent} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {initializeCustomChangeSource} from '../services/change-source/custom/change-source.ts';\nimport {initializePostgresChangeSource} from '../services/change-source/pg/change-source.ts';\nimport {createBackupCleanupMonitor} from '../services/change-streamer/backup-cleanup-monitor-factory.ts';\nimport {ChangeStreamerHttpServer} from '../services/change-streamer/change-streamer-http.ts';\nimport {initializeStreamer} from '../services/change-streamer/change-streamer-service.ts';\nimport type {ChangeStreamerService} from '../services/change-streamer/change-streamer.ts';\nimport {initChangeStreamerSchema} from '../services/change-streamer/schema/init.ts';\nimport {AutoResetSignal} from '../services/change-streamer/schema/tables.ts';\nimport {PurgeLocker} from '../services/change-streamer/storer.ts';\nimport {\n  exitAfter,\n  ProcessManager,\n  runUntilKilled,\n} from '../services/life-cycle.ts';\nimport {startReplicaBackupProcess} from '../services/litestream/commands.ts';\nimport {\n  changeLogFileName,\n  deleteChangeLogDB,\n} from '../services/replicator/change-log-db.ts';\nimport {\n  replicationStatusError,\n  ReplicationStatusPublisher,\n} from '../services/replicator/replication-status.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n  childWorker,\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport type {ReplicaFileMode} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {REPLICATOR_URL} from './worker-urls.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...argv: string[]\n): Promise<void> {\n  const config = getNormalizedZeroConfig({env, argv});\n  const {\n    taskID,\n    changeStreamer: {\n      port,\n      address,\n      protocol,\n      startupDelayMs,\n      backPressureLimitHeapProportion,\n      flowControlConsensusTimeoutProportion,\n      flowControlSlowSubscriberGracePeriodSeconds,\n      sqliteChangeLogMode,\n      sqliteChangeLogReadPercent,\n      sqliteChangeLogColdReadPercent,\n      sqliteChangeLogComparePercent,\n      sqliteChangeLogRetentionMs,\n      sqliteChangeLogReadBatchRows,\n      sqliteChangeLogPurgeBatchRows,\n      sqliteChangeLogBarrierTimeoutMs,\n    },\n    autoReset,\n    replicationLag,\n    litestream,\n    upstream,\n    change,\n    replica,\n    initialSync,\n    keepaliveTimeoutMs,\n    sqliteCorruptionChecks,\n  } = config;\n\n  startOtelAuto(\n    createLogContext(config, 'change-streamer', 0, false),\n    'change-streamer',\n    0,\n  );\n  lc = createLogContext(config, 'change-streamer');\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: 'change-streamer replica',\n      dbPath: replica.file,\n    },\n    sqliteCorruptionChecks,\n  );\n  initEventSink(lc, config);\n\n  // Kick off DB connection warmup in the background.\n  const changeDB = await connectPgClient(\n    lc,\n    change.db,\n    'change-streamer',\n    {max: change.maxConns},\n    {sendStringAsJson: true},\n  );\n  void warmupConnections(lc, changeDB, 'change').catch(() => {});\n\n  const shard = getShardConfig(config);\n\n  // Ensure the change DB schema is initialized/up-to-date.\n  await initChangeStreamerSchema(lc, changeDB, shard);\n\n  // When restoring from litestream, acquire a lock to prevent change-log\n  // purges. This ensures that (this) change-streamer will be able to resume\n  // from the backup.\n  let purgeLock =\n    litestream.backupURL && litestream.executable\n      ? await new PurgeLocker(lc, shard, changeDB).acquire()\n      : null;\n  const restoreOptions = {litestream, constraints: purgeLock ?? undefined};\n\n  let changeStreamer: ChangeStreamerService | undefined;\n  let backupURL: string | undefined;\n\n  const context = getServerContext(config);\n  const sqliteChangeLogEnabled = sqliteChangeLogMode !== 'off';\n  // The modes are cumulative, so `serve` implies `compare`.\n  const sqliteChangeLogComparing =\n    sqliteChangeLogMode === 'compare' || sqliteChangeLogMode === 'serve';\n  if (sqliteChangeLogEnabled) {\n    const changeLogFile = changeLogFileName(replica.file);\n    registerSQLiteCorruptionDiagnosticTarget(\n      {\n        debugName: 'change-streamer change-log',\n        dbPath: changeLogFile,\n      },\n      sqliteCorruptionChecks,\n    );\n    // The log's bytes are accounted for in the process that writes them. This\n    // is local disk only — the log is excluded from the litestream backup — and\n    // it is a whole-database total, because a table that left the replica left\n    // through its wal: the replica's footprint is what shrank and this is where\n    // it went.\n    getOrCreateGauge('replica', 'sqlite_change_log.file_bytes', {\n      description:\n        `The SQLite change log's total on-disk footprint: its main db file ` +\n        `plus its wal sidecars.`,\n      unit: 'By',\n    }).addCallback(async o =>\n      o.observe(await sqliteFileBytes(lc, changeLogFile)),\n    );\n  }\n\n  let waitForFirstBackupBeforeServing = false;\n  for (const first of [true, false]) {\n    try {\n      // Note: This performs initial sync of the replica if necessary.\n      const {\n        changeSource,\n        subscriptionState,\n        destinationBackupURL,\n        replicaID,\n        waitForBackupBeforeServing,\n      } =\n        upstream.type === 'pg'\n          ? await initializePostgresChangeSource(\n              lc,\n              upstream.db,\n              shard,\n              replica.file,\n              {\n                ...initialSync,\n                replicationSlotFailover: upstream.pgReplicationSlotFailover,\n              },\n              context,\n              replicationLag.reportIntervalMs,\n              restoreOptions,\n              {backupV5: litestream.backupUsingV5},\n              purgeLock,\n              upstream.pgStreamInboundTimeoutMs,\n            )\n          : await initializeCustomChangeSource(\n              lc,\n              upstream.db,\n              shard,\n              replica.file,\n              context,\n              restoreOptions,\n            );\n\n      const replicationStatusPublisher =\n        ReplicationStatusPublisher.forReplicaFile(replica.file);\n\n      changeStreamer = await initializeStreamer(\n        lc,\n        shard,\n        taskID,\n        address,\n        protocol,\n        changeDB,\n        changeSource,\n        replicationStatusPublisher,\n        subscriptionState,\n        destinationBackupURL\n          ? {\n              backupURL: destinationBackupURL,\n              litestreamVersion: litestream.backupUsingV5 ? 'v5' : 'legacy',\n            }\n          : null,\n        purgeLock,\n        autoReset ?? false,\n        {\n          backPressureLimitHeapProportion,\n          flowControlConsensusTimeoutProportion,\n          flowControlSlowSubscriberGracePeriodMs:\n            flowControlSlowSubscriberGracePeriodSeconds > 0\n              ? flowControlSlowSubscriberGracePeriodSeconds * 1000\n              : undefined,\n          statementTimeoutMs: change.statementTimeoutMs,\n          changeLogBatchSize: change.logBatchSize,\n          sqliteCatchup: {\n            changeLogFile: changeLogFileName(replica.file),\n            readBatchRows: sqliteChangeLogReadBatchRows,\n            barrierTimeoutMs: sqliteChangeLogBarrierTimeoutMs,\n          },\n          // The presence of these options is the writer's gate. This process\n          // performs the restore and the initial sync, so the replica's\n          // identity is in hand at the moment the log is opened.\n          sqliteChangeLogWriter: sqliteChangeLogEnabled\n            ? {\n                replicaFile: replica.file,\n                identity: {\n                  // RMv2 supplies the epoch; until then the generation and the\n                  // replica ID are the whole of the identity.\n                  epoch: null,\n                  generation: subscriptionState.replicaVersion,\n                  replicaID,\n                },\n              }\n            : undefined,\n          // The purge scheduler shares the writer's gate -- mode, not the\n          // read path -- because `write` mode, with no read selector at all,\n          // is the configuration it ships in.\n          sqliteChangeLogPurge: sqliteChangeLogEnabled\n            ? {\n                retentionMs: sqliteChangeLogRetentionMs,\n                batchRows: sqliteChangeLogPurgeBatchRows,\n              }\n            : undefined,\n          // Compare mode runs both advisory checks. Postgres remains authoritative.\n          sqliteChangeLogCompare: sqliteChangeLogComparing\n            ? {\n                replicaFile: replica.file,\n                comparePercent: sqliteChangeLogComparePercent,\n                retentionMs: sqliteChangeLogRetentionMs,\n                readBatchRows: sqliteChangeLogReadBatchRows,\n              }\n            : undefined,\n          // Slice 11 lands dark by default: serve mode constructs the stable\n          // router, while readPercent=0 keeps every catchup on PG and emits\n          // eligibility metrics before any canary traffic is enabled.\n          sqliteChangeLogServe:\n            sqliteChangeLogMode === 'serve'\n              ? {\n                  readPercent: sqliteChangeLogReadPercent,\n                  coldReadPercent: sqliteChangeLogColdReadPercent,\n                  retentionMs: sqliteChangeLogRetentionMs,\n                }\n              : undefined,\n        },\n        setTimeout,\n      );\n      backupURL = destinationBackupURL;\n      waitForFirstBackupBeforeServing = waitForBackupBeforeServing;\n      break;\n    } catch (e) {\n      if (first && e instanceof AutoResetSignal) {\n        lc.warn?.(`resetting replica ${replica.file}`, e);\n        // TODO: Make deleteLiteDB work with litestream. It will probably have to be\n        //       a semantic wipe instead of a file delete.\n        deleteLiteDB(replica.file);\n        // The change log carries the identity of the replica it was written\n        // beside, and the retry performs a fresh initial sync with a new\n        // replicaVersion. Reconciliation would catch that on its own (as\n        // 'identity-mismatch') but only after the writer opened a file that is\n        // known here to be garbage.\n        deleteChangeLogDB(replica.file);\n        // Release the purge lock before retrying. This is safe because the\n        // purge lock exists to preserve change-log entries so the new\n        // change-streamer can resume from the backup replica's watermark.\n        // An AutoResetSignal means we cant resume from the backup replica\n        // (e.g. its replication slot is gone), so the change-log entries the lock\n        // was protecting are no longer needed. The retry performs a fresh\n        // initial sync with a new replication slot, independent of the old\n        // change-log. Releasing is also necessary to avoid a\n        // self-deadlock when CHANGE_DB == UPSTREAM_DB:\n        // CREATE_REPLICATION_SLOT waits for all older transactions to\n        // finish, including this lock's open transaction.\n        await purgeLock?.release();\n        purgeLock = null;\n        continue; // execute again with a fresh initial-sync\n      }\n      if (e instanceof DatabaseInitError) {\n        throw new Error(\n          `Cannot open ZERO_REPLICA_FILE at \"${replica.file}\". Please check that the path is valid.`,\n          {cause: e},\n        );\n      }\n      throw e;\n    }\n  }\n  // impossible: upstream must have advanced in order for replication to be stuck.\n  assert(changeStreamer, `resetting replica did not advance replicaVersion`);\n\n  const processes = new ProcessManager(lc, parent);\n  if (backupURL) {\n    lc.info?.('setting up backup to', backupURL);\n    litestream.backupURL = backupURL;\n    const {promise: backupStarted, resolve} = resolver();\n\n    // Start a backup replicator and corresponding litestream backup process.\n    processes\n      .addWorker(\n        childWorker(REPLICATOR_URL, env, 'backup' satisfies ReplicaFileMode),\n        'supporting',\n        'backup-replicator',\n      )\n      // Wait for the replicator's first message (i.e. \"ready\") before starting\n      // litestream backup in order to avoid contending on the sqlite lock\n      // when the replicator first prepares the db file.\n      .once('message', () => {\n        processes.addSubprocess(\n          startReplicaBackupProcess(lc, litestream, replica.file),\n          'supporting',\n          'litestream',\n        );\n        resolve();\n      });\n    await backupStarted;\n  }\n\n  const backupMonitor = createBackupCleanupMonitor({\n    lc,\n    config,\n    replicaFile: replica.file,\n    changeStreamer,\n  });\n\n  let readinessGate = promiseVoid;\n  if (waitForFirstBackupBeforeServing) {\n    const start = performance.now();\n    lc.info?.(`awaiting initial backup ...`);\n\n    readinessGate = backupMonitor.firstBackupReceived().then(() => {\n      const elapsed = performance.now() - start;\n      lc.info?.(`initial backup confirmed after ${elapsed.toFixed(2)}ms`);\n    });\n  }\n\n  const changeStreamerWebServer = new ChangeStreamerHttpServer(\n    lc,\n    {port, keepaliveTimeoutMs, startupDelayMs, readinessGate},\n    parent,\n    changeStreamer,\n  );\n\n  void readinessGate.then(() => parent.send(['ready', {ready: true}]));\n\n  // Note: The changeStreamer itself is not started here; it is started by the\n  //       changeStreamerWebServer after a delay to ensure that routing\n  //       routing elements have registered the server before the\n  //       change-streamer takes over the replication slot.\n  // TODO: Remove this delay and start the changeStreamer normally here once\n  //       transitioned to RMv2, since changeStreamer startup will no longer\n  //       disrupt an existing task.\n  try {\n    await runUntilKilled(lc, parent, changeStreamerWebServer, backupMonitor);\n  } catch (err) {\n    processes.logErrorAndExit(err, 'change-streamer');\n  } finally {\n    await processes.shutdown();\n  }\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () =>\n      runWorker(\n        must(parentWorker),\n        process.env,\n        ...process.argv.slice(2),\n      ).catch(async e => {\n        await publishCriticalEvent(\n          lc,\n          replicationStatusError(lc, 'Initializing', e),\n        );\n        throw e;\n      }),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAmDA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAClD,MAAM,EACJ,QACA,gBAAgB,EACd,MACA,SACA,UACA,gBACA,iCACA,uCACA,6CACA,qBACA,4BACA,gCACA,+BACA,4BACA,8BACA,+BACA,mCAEF,WACA,gBACA,YACA,UACA,QACA,SACA,aACA,oBACA,2BACE;CAEJ,cACE,iBAAiB,QAAQ,mBAAmB,GAAG,KAAK,GACpD,mBACA,CACF;CACA,KAAK,iBAAiB,QAAQ,iBAAiB;CAC/C,yCACE;EACE,WAAW;EACX,QAAQ,QAAQ;CAClB,GACA,sBACF;CACA,cAAc,IAAI,MAAM;CAGxB,MAAM,WAAW,MAAM,gBACrB,IACA,OAAO,IACP,mBACA,EAAC,KAAK,OAAO,SAAQ,GACrB,EAAC,kBAAkB,KAAI,CACzB;CACA,kBAAuB,IAAI,UAAU,QAAQ,EAAE,YAAY,CAAC,CAAC;CAE7D,MAAM,QAAQ,eAAe,MAAM;CAGnC,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAKlD,IAAI,YACF,WAAW,aAAa,WAAW,aAC/B,MAAM,IAAI,YAAY,IAAI,OAAO,QAAQ,EAAE,QAAQ,IACnD;CACN,MAAM,iBAAiB;EAAC;EAAY,aAAa,aAAa,KAAA;CAAS;CAEvE,IAAI;CACJ,IAAI;CAEJ,MAAM,UAAU,iBAAiB,MAAM;CACvC,MAAM,yBAAyB,wBAAwB;CAEvD,MAAM,2BACJ,wBAAwB,aAAa,wBAAwB;CAC/D,IAAI,wBAAwB;EAC1B,MAAM,gBAAgB,kBAAkB,QAAQ,IAAI;EACpD,yCACE;GACE,WAAW;GACX,QAAQ;EACV,GACA,sBACF;EAMA,iBAAiB,WAAW,gCAAgC;GAC1D,aACE;GAEF,MAAM;EACR,CAAC,EAAE,YAAY,OAAM,MACnB,EAAE,QAAQ,MAAM,gBAAgB,IAAI,aAAa,CAAC,CACpD;CACF;CAEA,IAAI,kCAAkC;CACtC,KAAK,MAAM,SAAS,CAAC,MAAM,KAAK,GAC9B,IAAI;EAEF,MAAM,EACJ,cACA,mBACA,sBACA,WACA,+BAEA,SAAS,SAAS,OACd,MAAM,+BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR;GACE,GAAG;GACH,yBAAyB,SAAS;EACpC,GACA,SACA,eAAe,kBACf,gBACA,EAAC,UAAU,WAAW,cAAa,GACnC,WACA,SAAS,wBACX,IACA,MAAM,6BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR,SACA,cACF;EAEN,MAAM,6BACJ,2BAA2B,eAAe,QAAQ,IAAI;EAExD,iBAAiB,MAAM,mBACrB,IACA,OACA,QACA,SACA,UACA,UACA,cACA,4BACA,mBACA,uBACI;GACE,WAAW;GACX,mBAAmB,WAAW,gBAAgB,OAAO;EACvD,IACA,MACJ,WACA,aAAa,OACb;GACE;GACA;GACA,wCACE,8CAA8C,IAC1C,8CAA8C,MAC9C,KAAA;GACN,oBAAoB,OAAO;GAC3B,oBAAoB,OAAO;GAC3B,eAAe;IACb,eAAe,kBAAkB,QAAQ,IAAI;IAC7C,eAAe;IACf,kBAAkB;GACpB;GAIA,uBAAuB,yBACnB;IACE,aAAa,QAAQ;IACrB,UAAU;KAGR,OAAO;KACP,YAAY,kBAAkB;KAC9B;IACF;GACF,IACA,KAAA;GAIJ,sBAAsB,yBAClB;IACE,aAAa;IACb,WAAW;GACb,IACA,KAAA;GAEJ,wBAAwB,2BACpB;IACE,aAAa,QAAQ;IACrB,gBAAgB;IAChB,aAAa;IACb,eAAe;GACjB,IACA,KAAA;GAIJ,sBACE,wBAAwB,UACpB;IACE,aAAa;IACb,iBAAiB;IACjB,aAAa;GACf,IACA,KAAA;EACR,GACA,UACF;EACA,YAAY;EACZ,kCAAkC;EAClC;CACF,SAAS,GAAG;EACV,IAAI,SAAS,aAAa,iBAAiB;GACzC,GAAG,OAAO,qBAAqB,QAAQ,QAAQ,CAAC;GAGhD,aAAa,QAAQ,IAAI;GAMzB,kBAAkB,QAAQ,IAAI;GAY9B,MAAM,WAAW,QAAQ;GACzB,YAAY;GACZ;EACF;EACA,IAAI,aAAa,mBACf,MAAM,IAAI,MACR,qCAAqC,QAAQ,KAAK,0CAClD,EAAC,OAAO,EAAC,CACX;EAEF,MAAM;CACR;CAGF,OAAO,gBAAgB,kDAAkD;CAEzE,MAAM,YAAY,IAAI,eAAe,IAAI,MAAM;CAC/C,IAAI,WAAW;EACb,GAAG,OAAO,wBAAwB,SAAS;EAC3C,WAAW,YAAY;EACvB,MAAM,EAAC,SAAS,eAAe,YAAW,SAAS;EAGnD,UACG,UACC,YAAY,gBAAgB,KAAK,QAAkC,GACnE,cACA,mBACF,EAIC,KAAK,iBAAiB;GACrB,UAAU,cACR,0BAA0B,IAAI,YAAY,QAAQ,IAAI,GACtD,cACA,YACF;GACA,QAAQ;EACV,CAAC;EACH,MAAM;CACR;CAEA,MAAM,gBAAgB,2BAA2B;EAC/C;EACA;EACA,aAAa,QAAQ;EACrB;CACF,CAAC;CAED,IAAI,gBAAgB;CACpB,IAAI,iCAAiC;EACnC,MAAM,QAAQ,YAAY,IAAI;EAC9B,GAAG,OAAO,6BAA6B;EAEvC,gBAAgB,cAAc,oBAAoB,EAAE,WAAW;GAC7D,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,GAAG,OAAO,kCAAkC,QAAQ,QAAQ,CAAC,EAAE,GAAG;EACpE,CAAC;CACH;CAEA,MAAM,0BAA0B,IAAI,yBAClC,IACA;EAAC;EAAM;EAAoB;EAAgB;CAAa,GACxD,QACA,cACF;CAEA,cAAmB,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC,CAAC;CASnE,IAAI;EACF,MAAM,eAAe,IAAI,QAAQ,yBAAyB,aAAa;CACzE,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,iBAAiB;CAClD,UAAU;EACR,MAAM,UAAU,SAAS;CAC3B;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UAEJ,UACE,KAAK,YAAY,GACjB,QAAQ,KACR,GAAG,QAAQ,KAAK,MAAM,CAAC,CACzB,EAAE,MAAM,OAAM,MAAK;CACjB,MAAM,qBACJ,IACA,uBAAuB,IAAI,gBAAgB,CAAC,CAC9C;CACA,MAAM;AACR,CAAC,CACL"}
+{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {DatabaseInitError} from '../../../zqlite/src/db.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink, publishCriticalEvent} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {initializeCustomChangeSource} from '../services/change-source/custom/change-source.ts';\nimport {initializePostgresChangeSource} from '../services/change-source/pg/change-source.ts';\nimport {createBackupCleanupMonitor} from '../services/change-streamer/backup-cleanup-monitor-factory.ts';\nimport {ChangeStreamerHttpServer} from '../services/change-streamer/change-streamer-http.ts';\nimport {initializeStreamer} from '../services/change-streamer/change-streamer-service.ts';\nimport type {ChangeStreamerService} from '../services/change-streamer/change-streamer.ts';\nimport {initChangeStreamerSchema} from '../services/change-streamer/schema/init.ts';\nimport {AutoResetSignal} from '../services/change-streamer/schema/tables.ts';\nimport {PurgeLocker} from '../services/change-streamer/storer.ts';\nimport {\n  exitAfter,\n  ProcessManager,\n  runUntilKilled,\n} from '../services/life-cycle.ts';\nimport {startReplicaBackupProcess} from '../services/litestream/commands.ts';\nimport {\n  changeLogFileName,\n  deleteChangeLogDB,\n} from '../services/replicator/change-log-db.ts';\nimport {\n  replicationStatusError,\n  ReplicationStatusPublisher,\n} from '../services/replicator/replication-status.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n  childWorker,\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport type {ReplicaFileMode} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {REPLICATOR_URL} from './worker-urls.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...argv: string[]\n): Promise<void> {\n  const config = getNormalizedZeroConfig({env, argv});\n  const {\n    taskID,\n    changeStreamer: {\n      port,\n      address,\n      protocol,\n      startupDelayMs,\n      backPressureLimitHeapProportion,\n      flowControlConsensusTimeoutProportion,\n      flowControlSlowSubscriberGracePeriodSeconds,\n      sqliteChangeLogMode,\n      sqliteChangeLogReadPercent,\n      sqliteChangeLogColdReadPercent,\n      sqliteChangeLogComparePercent,\n      sqliteChangeLogRetentionMs,\n      sqliteChangeLogReadBatchRows,\n      sqliteChangeLogPurgeBatchRows,\n      sqliteChangeLogBarrierTimeoutMs,\n    },\n    autoReset,\n    replicationLag,\n    litestream,\n    upstream,\n    change,\n    replica,\n    initialSync,\n    keepaliveTimeoutMs,\n    sqliteCorruptionChecks,\n  } = config;\n\n  startOtelAuto(\n    createLogContext(config, 'change-streamer', 0, false),\n    'change-streamer',\n    0,\n  );\n  lc = createLogContext(config, 'change-streamer');\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: 'change-streamer replica',\n      dbPath: replica.file,\n    },\n    sqliteCorruptionChecks,\n  );\n  initEventSink(lc, config);\n\n  // Startup-time client used for for change-streamer initialization\n  // and handoff / takeover. Steady-state clients are managed by the\n  // change-streamer (Storer) implementation.\n  const changeDB = await connectPgClient(\n    lc,\n    change.db,\n    'change-streamer-init',\n    {max: 5},\n    {sendStringAsJson: true},\n  );\n  void warmupConnections(lc, changeDB, 'change').catch(() => {});\n\n  const shard = getShardConfig(config);\n\n  // Ensure the change DB schema is initialized/up-to-date.\n  await initChangeStreamerSchema(lc, changeDB, shard);\n\n  // When restoring from litestream, acquire a lock to prevent change-log\n  // purges. This ensures that (this) change-streamer will be able to resume\n  // from the backup.\n  let purgeLock =\n    litestream.backupURL && litestream.executable\n      ? await new PurgeLocker(lc, shard, changeDB).acquire()\n      : null;\n  const restoreOptions = {litestream, constraints: purgeLock ?? undefined};\n\n  let changeStreamer: ChangeStreamerService | undefined;\n  let backupURL: string | undefined;\n\n  const context = getServerContext(config);\n  const sqliteChangeLogEnabled = sqliteChangeLogMode !== 'off';\n  // The modes are cumulative, so `serve` implies `compare`.\n  const sqliteChangeLogComparing =\n    sqliteChangeLogMode === 'compare' || sqliteChangeLogMode === 'serve';\n  if (sqliteChangeLogEnabled) {\n    const changeLogFile = changeLogFileName(replica.file);\n    registerSQLiteCorruptionDiagnosticTarget(\n      {\n        debugName: 'change-streamer change-log',\n        dbPath: changeLogFile,\n      },\n      sqliteCorruptionChecks,\n    );\n    // The log's bytes are accounted for in the process that writes them. This\n    // is local disk only — the log is excluded from the litestream backup — and\n    // it is a whole-database total, because a table that left the replica left\n    // through its wal: the replica's footprint is what shrank and this is where\n    // it went.\n    getOrCreateGauge('replica', 'sqlite_change_log.file_bytes', {\n      description:\n        `The SQLite change log's total on-disk footprint: its main db file ` +\n        `plus its wal sidecars.`,\n      unit: 'By',\n    }).addCallback(async o =>\n      o.observe(await sqliteFileBytes(lc, changeLogFile)),\n    );\n  }\n\n  let waitForFirstBackupBeforeServing = false;\n  for (const first of [true, false]) {\n    try {\n      // Note: This performs initial sync of the replica if necessary.\n      const {\n        changeSource,\n        subscriptionState,\n        destinationBackupURL,\n        replicaID,\n        waitForBackupBeforeServing,\n      } =\n        upstream.type === 'pg'\n          ? await initializePostgresChangeSource(\n              lc,\n              upstream.db,\n              shard,\n              replica.file,\n              {\n                ...initialSync,\n                replicationSlotFailover: upstream.pgReplicationSlotFailover,\n              },\n              context,\n              replicationLag.reportIntervalMs,\n              restoreOptions,\n              {backupV5: litestream.backupUsingV5},\n              purgeLock,\n              upstream.pgStreamInboundTimeoutMs,\n            )\n          : await initializeCustomChangeSource(\n              lc,\n              upstream.db,\n              shard,\n              replica.file,\n              context,\n              restoreOptions,\n            );\n\n      const replicationStatusPublisher =\n        ReplicationStatusPublisher.forReplicaFile(replica.file);\n\n      changeStreamer = await initializeStreamer(\n        lc,\n        shard,\n        taskID,\n        address,\n        protocol,\n        changeDB,\n        changeSource,\n        replicationStatusPublisher,\n        subscriptionState,\n        destinationBackupURL\n          ? {\n              backupURL: destinationBackupURL,\n              litestreamVersion: litestream.backupUsingV5 ? 'v5' : 'legacy',\n            }\n          : null,\n        purgeLock,\n        autoReset ?? false,\n        {\n          backPressureLimitHeapProportion,\n          flowControlConsensusTimeoutProportion,\n          flowControlSlowSubscriberGracePeriodMs:\n            flowControlSlowSubscriberGracePeriodSeconds > 0\n              ? flowControlSlowSubscriberGracePeriodSeconds * 1000\n              : undefined,\n          statementTimeoutMs: change.statementTimeoutMs,\n          changeLogBatchSize: change.logBatchSize,\n          sqliteCatchup: {\n            changeLogFile: changeLogFileName(replica.file),\n            readBatchRows: sqliteChangeLogReadBatchRows,\n            barrierTimeoutMs: sqliteChangeLogBarrierTimeoutMs,\n          },\n          // The presence of these options is the writer's gate. This process\n          // performs the restore and the initial sync, so the replica's\n          // identity is in hand at the moment the log is opened.\n          sqliteChangeLogWriter: sqliteChangeLogEnabled\n            ? {\n                replicaFile: replica.file,\n                identity: {\n                  // RMv2 supplies the epoch; until then the generation and the\n                  // replica ID are the whole of the identity.\n                  epoch: null,\n                  generation: subscriptionState.replicaVersion,\n                  replicaID,\n                },\n              }\n            : undefined,\n          // The purge scheduler shares the writer's gate -- mode, not the\n          // read path -- because `write` mode, with no read selector at all,\n          // is the configuration it ships in.\n          sqliteChangeLogPurge: sqliteChangeLogEnabled\n            ? {\n                retentionMs: sqliteChangeLogRetentionMs,\n                batchRows: sqliteChangeLogPurgeBatchRows,\n              }\n            : undefined,\n          // Compare mode runs both advisory checks. Postgres remains authoritative.\n          sqliteChangeLogCompare: sqliteChangeLogComparing\n            ? {\n                replicaFile: replica.file,\n                comparePercent: sqliteChangeLogComparePercent,\n                retentionMs: sqliteChangeLogRetentionMs,\n                readBatchRows: sqliteChangeLogReadBatchRows,\n              }\n            : undefined,\n          // Slice 11 lands dark by default: serve mode constructs the stable\n          // router, while readPercent=0 keeps every catchup on PG and emits\n          // eligibility metrics before any canary traffic is enabled.\n          sqliteChangeLogServe:\n            sqliteChangeLogMode === 'serve'\n              ? {\n                  readPercent: sqliteChangeLogReadPercent,\n                  coldReadPercent: sqliteChangeLogColdReadPercent,\n                  retentionMs: sqliteChangeLogRetentionMs,\n                }\n              : undefined,\n        },\n        setTimeout,\n      );\n      backupURL = destinationBackupURL;\n      waitForFirstBackupBeforeServing = waitForBackupBeforeServing;\n      break;\n    } catch (e) {\n      if (first && e instanceof AutoResetSignal) {\n        lc.warn?.(`resetting replica ${replica.file}`, e);\n        // TODO: Make deleteLiteDB work with litestream. It will probably have to be\n        //       a semantic wipe instead of a file delete.\n        deleteLiteDB(replica.file);\n        // The change log carries the identity of the replica it was written\n        // beside, and the retry performs a fresh initial sync with a new\n        // replicaVersion. Reconciliation would catch that on its own (as\n        // 'identity-mismatch') but only after the writer opened a file that is\n        // known here to be garbage.\n        deleteChangeLogDB(replica.file);\n        // Release the purge lock before retrying. This is safe because the\n        // purge lock exists to preserve change-log entries so the new\n        // change-streamer can resume from the backup replica's watermark.\n        // An AutoResetSignal means we cant resume from the backup replica\n        // (e.g. its replication slot is gone), so the change-log entries the lock\n        // was protecting are no longer needed. The retry performs a fresh\n        // initial sync with a new replication slot, independent of the old\n        // change-log. Releasing is also necessary to avoid a\n        // self-deadlock when CHANGE_DB == UPSTREAM_DB:\n        // CREATE_REPLICATION_SLOT waits for all older transactions to\n        // finish, including this lock's open transaction.\n        await purgeLock?.release();\n        purgeLock = null;\n        continue; // execute again with a fresh initial-sync\n      }\n      if (e instanceof DatabaseInitError) {\n        throw new Error(\n          `Cannot open ZERO_REPLICA_FILE at \"${replica.file}\". Please check that the path is valid.`,\n          {cause: e},\n        );\n      }\n      throw e;\n    }\n  }\n  // impossible: upstream must have advanced in order for replication to be stuck.\n  assert(changeStreamer, `resetting replica did not advance replicaVersion`);\n\n  const processes = new ProcessManager(lc, parent);\n  if (backupURL) {\n    lc.info?.('setting up backup to', backupURL);\n    litestream.backupURL = backupURL;\n    const {promise: backupStarted, resolve} = resolver();\n\n    // Start a backup replicator and corresponding litestream backup process.\n    processes\n      .addWorker(\n        childWorker(REPLICATOR_URL, env, 'backup' satisfies ReplicaFileMode),\n        'supporting',\n        'backup-replicator',\n      )\n      // Wait for the replicator's first message (i.e. \"ready\") before starting\n      // litestream backup in order to avoid contending on the sqlite lock\n      // when the replicator first prepares the db file.\n      .once('message', () => {\n        processes.addSubprocess(\n          startReplicaBackupProcess(lc, litestream, replica.file),\n          'supporting',\n          'litestream',\n        );\n        resolve();\n      });\n    await backupStarted;\n  }\n\n  const backupMonitor = createBackupCleanupMonitor({\n    lc,\n    config,\n    replicaFile: replica.file,\n    changeStreamer,\n  });\n\n  let readinessGate = promiseVoid;\n  if (waitForFirstBackupBeforeServing) {\n    const start = performance.now();\n    lc.info?.(`awaiting initial backup ...`);\n\n    readinessGate = backupMonitor.firstBackupReceived().then(() => {\n      const elapsed = performance.now() - start;\n      lc.info?.(`initial backup confirmed after ${elapsed.toFixed(2)}ms`);\n    });\n  }\n\n  const changeStreamerWebServer = new ChangeStreamerHttpServer(\n    lc,\n    {port, keepaliveTimeoutMs, startupDelayMs, readinessGate},\n    parent,\n    changeStreamer,\n  );\n\n  void readinessGate.then(() => parent.send(['ready', {ready: true}]));\n\n  // Note: The changeStreamer itself is not started here; it is started by the\n  //       changeStreamerWebServer after a delay to ensure that routing\n  //       routing elements have registered the server before the\n  //       change-streamer takes over the replication slot.\n  // TODO: Remove this delay and start the changeStreamer normally here once\n  //       transitioned to RMv2, since changeStreamer startup will no longer\n  //       disrupt an existing task.\n  try {\n    await runUntilKilled(lc, parent, changeStreamerWebServer, backupMonitor);\n  } catch (err) {\n    processes.logErrorAndExit(err, 'change-streamer');\n  } finally {\n    await processes.shutdown();\n  }\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () =>\n      runWorker(\n        must(parentWorker),\n        process.env,\n        ...process.argv.slice(2),\n      ).catch(async e => {\n        await publishCriticalEvent(\n          lc,\n          replicationStatusError(lc, 'Initializing', e),\n        );\n        throw e;\n      }),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAmDA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAClD,MAAM,EACJ,QACA,gBAAgB,EACd,MACA,SACA,UACA,gBACA,iCACA,uCACA,6CACA,qBACA,4BACA,gCACA,+BACA,4BACA,8BACA,+BACA,mCAEF,WACA,gBACA,YACA,UACA,QACA,SACA,aACA,oBACA,2BACE;CAEJ,cACE,iBAAiB,QAAQ,mBAAmB,GAAG,KAAK,GACpD,mBACA,CACF;CACA,KAAK,iBAAiB,QAAQ,iBAAiB;CAC/C,yCACE;EACE,WAAW;EACX,QAAQ,QAAQ;CAClB,GACA,sBACF;CACA,cAAc,IAAI,MAAM;CAKxB,MAAM,WAAW,MAAM,gBACrB,IACA,OAAO,IACP,wBACA,EAAC,KAAK,EAAC,GACP,EAAC,kBAAkB,KAAI,CACzB;CACA,kBAAuB,IAAI,UAAU,QAAQ,EAAE,YAAY,CAAC,CAAC;CAE7D,MAAM,QAAQ,eAAe,MAAM;CAGnC,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAKlD,IAAI,YACF,WAAW,aAAa,WAAW,aAC/B,MAAM,IAAI,YAAY,IAAI,OAAO,QAAQ,EAAE,QAAQ,IACnD;CACN,MAAM,iBAAiB;EAAC;EAAY,aAAa,aAAa,KAAA;CAAS;CAEvE,IAAI;CACJ,IAAI;CAEJ,MAAM,UAAU,iBAAiB,MAAM;CACvC,MAAM,yBAAyB,wBAAwB;CAEvD,MAAM,2BACJ,wBAAwB,aAAa,wBAAwB;CAC/D,IAAI,wBAAwB;EAC1B,MAAM,gBAAgB,kBAAkB,QAAQ,IAAI;EACpD,yCACE;GACE,WAAW;GACX,QAAQ;EACV,GACA,sBACF;EAMA,iBAAiB,WAAW,gCAAgC;GAC1D,aACE;GAEF,MAAM;EACR,CAAC,EAAE,YAAY,OAAM,MACnB,EAAE,QAAQ,MAAM,gBAAgB,IAAI,aAAa,CAAC,CACpD;CACF;CAEA,IAAI,kCAAkC;CACtC,KAAK,MAAM,SAAS,CAAC,MAAM,KAAK,GAC9B,IAAI;EAEF,MAAM,EACJ,cACA,mBACA,sBACA,WACA,+BAEA,SAAS,SAAS,OACd,MAAM,+BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR;GACE,GAAG;GACH,yBAAyB,SAAS;EACpC,GACA,SACA,eAAe,kBACf,gBACA,EAAC,UAAU,WAAW,cAAa,GACnC,WACA,SAAS,wBACX,IACA,MAAM,6BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR,SACA,cACF;EAEN,MAAM,6BACJ,2BAA2B,eAAe,QAAQ,IAAI;EAExD,iBAAiB,MAAM,mBACrB,IACA,OACA,QACA,SACA,UACA,UACA,cACA,4BACA,mBACA,uBACI;GACE,WAAW;GACX,mBAAmB,WAAW,gBAAgB,OAAO;EACvD,IACA,MACJ,WACA,aAAa,OACb;GACE;GACA;GACA,wCACE,8CAA8C,IAC1C,8CAA8C,MAC9C,KAAA;GACN,oBAAoB,OAAO;GAC3B,oBAAoB,OAAO;GAC3B,eAAe;IACb,eAAe,kBAAkB,QAAQ,IAAI;IAC7C,eAAe;IACf,kBAAkB;GACpB;GAIA,uBAAuB,yBACnB;IACE,aAAa,QAAQ;IACrB,UAAU;KAGR,OAAO;KACP,YAAY,kBAAkB;KAC9B;IACF;GACF,IACA,KAAA;GAIJ,sBAAsB,yBAClB;IACE,aAAa;IACb,WAAW;GACb,IACA,KAAA;GAEJ,wBAAwB,2BACpB;IACE,aAAa,QAAQ;IACrB,gBAAgB;IAChB,aAAa;IACb,eAAe;GACjB,IACA,KAAA;GAIJ,sBACE,wBAAwB,UACpB;IACE,aAAa;IACb,iBAAiB;IACjB,aAAa;GACf,IACA,KAAA;EACR,GACA,UACF;EACA,YAAY;EACZ,kCAAkC;EAClC;CACF,SAAS,GAAG;EACV,IAAI,SAAS,aAAa,iBAAiB;GACzC,GAAG,OAAO,qBAAqB,QAAQ,QAAQ,CAAC;GAGhD,aAAa,QAAQ,IAAI;GAMzB,kBAAkB,QAAQ,IAAI;GAY9B,MAAM,WAAW,QAAQ;GACzB,YAAY;GACZ;EACF;EACA,IAAI,aAAa,mBACf,MAAM,IAAI,MACR,qCAAqC,QAAQ,KAAK,0CAClD,EAAC,OAAO,EAAC,CACX;EAEF,MAAM;CACR;CAGF,OAAO,gBAAgB,kDAAkD;CAEzE,MAAM,YAAY,IAAI,eAAe,IAAI,MAAM;CAC/C,IAAI,WAAW;EACb,GAAG,OAAO,wBAAwB,SAAS;EAC3C,WAAW,YAAY;EACvB,MAAM,EAAC,SAAS,eAAe,YAAW,SAAS;EAGnD,UACG,UACC,YAAY,gBAAgB,KAAK,QAAkC,GACnE,cACA,mBACF,EAIC,KAAK,iBAAiB;GACrB,UAAU,cACR,0BAA0B,IAAI,YAAY,QAAQ,IAAI,GACtD,cACA,YACF;GACA,QAAQ;EACV,CAAC;EACH,MAAM;CACR;CAEA,MAAM,gBAAgB,2BAA2B;EAC/C;EACA;EACA,aAAa,QAAQ;EACrB;CACF,CAAC;CAED,IAAI,gBAAgB;CACpB,IAAI,iCAAiC;EACnC,MAAM,QAAQ,YAAY,IAAI;EAC9B,GAAG,OAAO,6BAA6B;EAEvC,gBAAgB,cAAc,oBAAoB,EAAE,WAAW;GAC7D,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,GAAG,OAAO,kCAAkC,QAAQ,QAAQ,CAAC,EAAE,GAAG;EACpE,CAAC;CACH;CAEA,MAAM,0BAA0B,IAAI,yBAClC,IACA;EAAC;EAAM;EAAoB;EAAgB;CAAa,GACxD,QACA,cACF;CAEA,cAAmB,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC,CAAC;CASnE,IAAI;EACF,MAAM,eAAe,IAAI,QAAQ,yBAAyB,aAAa;CACzE,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,iBAAiB;CAClD,UAAU;EACR,MAAM,UAAU,SAAS;CAC3B;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UAEJ,UACE,KAAK,YAAY,GACjB,QAAQ,KACR,GAAG,QAAQ,KAAK,MAAM,CAAC,CACzB,EAAE,MAAM,OAAM,MAAK;CACjB,MAAM,qBACJ,IACA,uBAAuB,IAAI,gBAAgB,CAAC,CAC9C;CACA,MAAM;AACR,CAAC,CACL"}

package/out/zero-cache/src/server/replicator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"replicator.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/replicator.ts"],"names":[],"mappings":"AAmCA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAgB/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CAqGf"}
+{"version":3,"file":"replicator.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/replicator.ts"],"names":[],"mappings":"AAmCA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAgB/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CAoGf"}

package/out/zero-cache/src/server/replicator.js

@@ -57,7 +57,7 @@ async function runWorker(parent, env, ...args) {
 	const shard = getShardConfig(config);
 	const { taskID, change, changeStreamer: { port, uri: changeStreamerURI = runningLocalChangeStreamer ? `http://localhost:${port}/` : void 0 } } = config;
 	const changeStreamer = new ChangeStreamerHttpClient(lc, shard, change.db, changeStreamerURI);
-	const replicator = new ReplicatorService(lc, taskID, `${workerName}-${pid}`, mode, dbPath, changeStreamer, workerClient, runningLocalChangeStreamer ? ReplicationStatusPublisher.forReplicaFile(dbPath) : null);
+	const replicator = new ReplicatorService(lc, taskID, `${workerName}-${pid}`, mode, changeStreamer, workerClient, runningLocalChangeStreamer ? ReplicationStatusPublisher.forReplicaFile(dbPath) : null);
 	setUpMessageHandlers(lc, replicator, parent);
 	const running = runUntilKilled(lc, parent, replicator);
 	for await (const _ of replicator.subscribe()) {

package/out/zero-cache/src/server/replicator.js.map

@@ -1 +1 @@
-{"version":3,"file":"replicator.js","names":[],"sources":["../../../../../zero-cache/src/server/replicator.ts"],"sourcesContent":["import {stat} from 'node:fs/promises';\nimport {pid} from 'node:process';\nimport type {ObservableCallback} from '@opentelemetry/api';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport type {\n  LitestreamConfig,\n  NormalizedZeroConfig,\n} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {ChangeStreamerHttpClient} from '../services/change-streamer/change-streamer-http.ts';\nimport {reserveAndGetSnapshotStatus} from '../services/change-streamer/snapshot.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {\n  tryRestore,\n  type RestoreResult,\n} from '../services/litestream/commands.ts';\nimport {\n  litestreamRestoreDuration,\n  litestreamRestoreMetricAttrs,\n  litestreamRestoreRuns,\n} from '../services/litestream/metrics.ts';\nimport {ReplicationStatusPublisher} from '../services/replicator/replication-status.ts';\nimport {\n  ReplicatorService,\n  type ReplicatorMode,\n} from '../services/replicator/replicator.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {ThreadWriteWorkerClient} from '../services/replicator/write-worker-client.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {\n  deleteStaleChangeLog,\n  getPragmaConfig,\n  replicaFileModeSchema,\n  setUpMessageHandlers,\n  setupReplica,\n  type WalMode,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  assert(args.length > 0, `replicator mode not specified`);\n  const fileMode = v.parse(args[0], replicaFileModeSchema);\n\n  const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n  const mode: ReplicatorMode = fileMode === 'backup' ? 'backup' : 'serving';\n  const runningLocalChangeStreamer =\n    config.changeStreamer.mode === 'dedicated' && !config.changeStreamer.uri;\n  const workerName = `${mode}-replicator`;\n  startOtelAuto(createLogContext(config, workerName, 0, false), workerName, 0);\n  lc = createLogContext(config, workerName);\n  const unregisterInitialCorruptionDiagnosticTarget =\n    registerSQLiteCorruptionDiagnosticTarget(\n      {\n        debugName: `${workerName} replica`,\n        dbPath: config.replica.file,\n      },\n      config.sqliteCorruptionChecks,\n    );\n  initEventSink(lc, config);\n\n  // Remote view-syncers restore from the replication-manager, which supplies\n  // the backup URL. A local change-streamer owns the canonical replica and\n  // must not infer backup configuration from bundled executable paths.\n  if (\n    fileMode === 'serving' &&\n    !runningLocalChangeStreamer &&\n    (config.litestream.executable || config.litestream.executableV5)\n  ) {\n    await restoreReplica(lc, config);\n  }\n\n  const {file: dbPath, walMode} = await setupReplica(\n    lc,\n    fileMode,\n    config.replica,\n  );\n  unregisterInitialCorruptionDiagnosticTarget();\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: `${workerName} replica`,\n      dbPath,\n    },\n    config.sqliteCorruptionChecks,\n  );\n\n  // A file left behind by a run with the writer enabled would otherwise hold\n  // local disk indefinitely. See `replicatorDeletesStaleChangeLog` for why\n  // this is keyed on the config flag and nothing else.\n  deleteStaleChangeLog(config.changeStreamer.sqliteChangeLogMode, dbPath);\n\n  setupMetrics(lc, dbPath, walMode);\n\n  // Create the write worker for async SQLite writes.\n  const pragmas = getPragmaConfig(fileMode);\n  const workerClient = new ThreadWriteWorkerClient();\n  await workerClient.init(dbPath, mode, pragmas, config.log);\n\n  const shard = getShardConfig(config);\n  const {\n    taskID,\n    change,\n    changeStreamer: {\n      port,\n      uri: changeStreamerURI = runningLocalChangeStreamer\n        ? `http://localhost:${port}/`\n        : undefined,\n    },\n  } = config;\n  const changeStreamer = new ChangeStreamerHttpClient(\n    lc,\n    shard,\n    change.db,\n    changeStreamerURI,\n  );\n\n  const replicator = new ReplicatorService(\n    lc,\n    taskID,\n    `${workerName}-${pid}`,\n    mode,\n    dbPath,\n    changeStreamer,\n    workerClient,\n    runningLocalChangeStreamer\n      ? // publish ReplicationStatusEvents from backup-replicator only\n        ReplicationStatusPublisher.forReplicaFile(dbPath)\n      : null,\n  );\n\n  setUpMessageHandlers(lc, replicator, parent);\n\n  const running = runUntilKilled(lc, parent, replicator);\n\n  // Signal readiness once the first ReplicaVersionReady notification is received.\n  for await (const _ of replicator.subscribe()) {\n    parent.send(['ready', {ready: true}]);\n    break;\n  }\n\n  return running;\n}\n\nfunction setupMetrics(lc: LogContext, file: string, walMode: WalMode) {\n  getOrCreateGauge('replica', 'db_size', {\n    description:\n      `The size of the replica's main db file, ` +\n      `which does not include the wal file(s)`,\n    unit: 'bytes',\n  }).addCallback(observeFileSize(lc, file));\n\n  getOrCreateGauge('replica', 'wal_size', {\n    description: `The size of the replica's wal file`,\n    unit: 'bytes',\n  }).addCallback(observeFileSize(lc, `${file}-wal`));\n\n  if (walMode === 'wal2') {\n    getOrCreateGauge('replica', 'wal2_size', {\n      description: `The size of the replica's wal2 file`,\n      unit: 'bytes',\n    }).addCallback(observeFileSize(lc, `${file}-wal2`));\n  }\n\n  // A whole-database total rather than a per-file one. Its counterpart,\n  // `sqlite_change_log.file_bytes`, is emitted by the change-streamer, which is\n  // the process that writes the log.\n  getOrCreateGauge('replica', 'file_bytes', {\n    description:\n      `The replica's total on-disk footprint: its main db file plus its ` +\n      `wal sidecars.`,\n    unit: 'By',\n  }).addCallback(observeSQLiteFileBytes(lc, file));\n}\n\nfunction observeFileSize(lc: LogContext, file: string): ObservableCallback {\n  return async o => {\n    try {\n      const stats = await stat(file);\n      o.observe(stats.size);\n    } catch (e) {\n      lc.warn?.(`unable to stat ${file} for size metrics`, e);\n    }\n  };\n}\n\nfunction observeSQLiteFileBytes(\n  lc: LogContext,\n  file: string,\n): ObservableCallback {\n  return async o => o.observe(await sqliteFileBytes(lc, file));\n}\n\nconst RETRY_INTERVAL_MS = 3000;\n\n// View-syncers (no replicaConstraints) wait indefinitely for the\n// replication-manager to publish a restorable backup. On a fresh stack the\n// first backup is not durable until the initial sync completes and litestream\n// uploads the initial snapshot, which can take many minutes for a large\n// replica. The platform's startup probe budget (which scales with replica\n// size) is the backstop, so restoreReplica must not impose its own shorter\n// cap and self-terminate while the backup is still being produced.\nasync function restoreReplica(lc: LogContext, config: NormalizedZeroConfig) {\n  const start = performance.now();\n  let backupURL: string | undefined;\n  let result: RestoreResult | undefined;\n  try {\n    for (;;) {\n      const snapshotStatus = await reserveAndGetSnapshotStatus(lc, config);\n      // The backupURL comes from the replication-manager's snapshot response.\n      ({backupURL} = snapshotStatus);\n      const litestream: LitestreamConfig = {...config.litestream, backupURL};\n      const attempt = await tryRestore(\n        lc,\n        litestream,\n        config.replica.file,\n        snapshotStatus,\n        'view_syncer',\n      );\n      if (attempt.restored) {\n        result = attempt.result;\n        return;\n      }\n      lc.info?.(\n        `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n      );\n      await sleep(RETRY_INTERVAL_MS);\n    }\n  } finally {\n    const attrs = litestreamRestoreMetricAttrs(\n      config.litestream,\n      'view_syncer',\n      backupURL,\n    );\n    const labels = {...attrs, result: result ?? 'error'};\n    litestreamRestoreRuns().add(1, labels);\n    litestreamRestoreDuration().recordMs(performance.now() - start, labels);\n  }\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAqDA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,OAAO,KAAK,SAAS,GAAG,+BAA+B;CACvD,MAAM,WAAW,MAAQ,KAAK,IAAI,qBAAqB;CAEvD,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CACjE,MAAM,OAAuB,aAAa,WAAW,WAAW;CAChE,MAAM,6BACJ,OAAO,eAAe,SAAS,eAAe,CAAC,OAAO,eAAe;CACvE,MAAM,aAAa,GAAG,KAAK;CAC3B,cAAc,iBAAiB,QAAQ,YAAY,GAAG,KAAK,GAAG,YAAY,CAAC;CAC3E,KAAK,iBAAiB,QAAQ,UAAU;CACxC,MAAM,8CACJ,yCACE;EACE,WAAW,GAAG,WAAW;EACzB,QAAQ,OAAO,QAAQ;CACzB,GACA,OAAO,sBACT;CACF,cAAc,IAAI,MAAM;CAKxB,IACE,aAAa,aACb,CAAC,+BACA,OAAO,WAAW,cAAc,OAAO,WAAW,eAEnD,MAAM,eAAe,IAAI,MAAM;CAGjC,MAAM,EAAC,MAAM,QAAQ,YAAW,MAAM,aACpC,IACA,UACA,OAAO,OACT;CACA,4CAA4C;CAC5C,yCACE;EACE,WAAW,GAAG,WAAW;EACzB;CACF,GACA,OAAO,sBACT;CAKA,qBAAqB,OAAO,eAAe,qBAAqB,MAAM;CAEtE,aAAa,IAAI,QAAQ,OAAO;CAGhC,MAAM,UAAU,gBAAgB,QAAQ;CACxC,MAAM,eAAe,IAAI,wBAAwB;CACjD,MAAM,aAAa,KAAK,QAAQ,MAAM,SAAS,OAAO,GAAG;CAEzD,MAAM,QAAQ,eAAe,MAAM;CACnC,MAAM,EACJ,QACA,QACA,gBAAgB,EACd,MACA,KAAK,oBAAoB,6BACrB,oBAAoB,KAAK,KACzB,KAAA,QAEJ;CACJ,MAAM,iBAAiB,IAAI,yBACzB,IACA,OACA,OAAO,IACP,iBACF;CAEA,MAAM,aAAa,IAAI,kBACrB,IACA,QACA,GAAG,WAAW,GAAG,OACjB,MACA,QACA,gBACA,cACA,6BAEI,2BAA2B,eAAe,MAAM,IAChD,IACN;CAEA,qBAAqB,IAAI,YAAY,MAAM;CAE3C,MAAM,UAAU,eAAe,IAAI,QAAQ,UAAU;CAGrD,WAAW,MAAM,KAAK,WAAW,UAAU,GAAG;EAC5C,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;EACpC;CACF;CAEA,OAAO;AACT;AAEA,SAAS,aAAa,IAAgB,MAAc,SAAkB;CACpE,iBAAiB,WAAW,WAAW;EACrC,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,IAAI,CAAC;CAExC,iBAAiB,WAAW,YAAY;EACtC,aAAa;EACb,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,GAAG,KAAK,KAAK,CAAC;CAEjD,IAAI,YAAY,QACd,iBAAiB,WAAW,aAAa;EACvC,aAAa;EACb,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,GAAG,KAAK,MAAM,CAAC;CAMpD,iBAAiB,WAAW,cAAc;EACxC,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,uBAAuB,IAAI,IAAI,CAAC;AACjD;AAEA,SAAS,gBAAgB,IAAgB,MAAkC;CACzE,OAAO,OAAM,MAAK;EAChB,IAAI;GACF,MAAM,QAAQ,MAAM,KAAK,IAAI;GAC7B,EAAE,QAAQ,MAAM,IAAI;EACtB,SAAS,GAAG;GACV,GAAG,OAAO,kBAAkB,KAAK,oBAAoB,CAAC;EACxD;CACF;AACF;AAEA,SAAS,uBACP,IACA,MACoB;CACpB,OAAO,OAAM,MAAK,EAAE,QAAQ,MAAM,gBAAgB,IAAI,IAAI,CAAC;AAC7D;AAEA,IAAM,oBAAoB;AAS1B,eAAe,eAAe,IAAgB,QAA8B;CAC1E,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI;CACJ,IAAI;CACJ,IAAI;EACF,SAAS;GACP,MAAM,iBAAiB,MAAM,4BAA4B,IAAI,MAAM;GAEnE,CAAC,CAAC,aAAa;GAEf,MAAM,UAAU,MAAM,WACpB,IACA;IAHoC,GAAG,OAAO;IAAY;GAG1D,GACA,OAAO,QAAQ,MACf,gBACA,aACF;GACA,IAAI,QAAQ,UAAU;IACpB,SAAS,QAAQ;IACjB;GACF;GACA,GAAG,OACD,kCAAkC,oBAAoB,IAAK,SAC7D;GACA,MAAM,MAAM,iBAAiB;EAC/B;CACF,UAAU;EAMR,MAAM,SAAS;GAAC,GALF,6BACZ,OAAO,YACP,eACA,SAEiB;GAAO,QAAQ,UAAU;EAAO;EACnD,sBAAsB,EAAE,IAAI,GAAG,MAAM;EACrC,0BAA0B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO,MAAM;CACxE;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}
+{"version":3,"file":"replicator.js","names":[],"sources":["../../../../../zero-cache/src/server/replicator.ts"],"sourcesContent":["import {stat} from 'node:fs/promises';\nimport {pid} from 'node:process';\nimport type {ObservableCallback} from '@opentelemetry/api';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport type {\n  LitestreamConfig,\n  NormalizedZeroConfig,\n} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {ChangeStreamerHttpClient} from '../services/change-streamer/change-streamer-http.ts';\nimport {reserveAndGetSnapshotStatus} from '../services/change-streamer/snapshot.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {\n  tryRestore,\n  type RestoreResult,\n} from '../services/litestream/commands.ts';\nimport {\n  litestreamRestoreDuration,\n  litestreamRestoreMetricAttrs,\n  litestreamRestoreRuns,\n} from '../services/litestream/metrics.ts';\nimport {ReplicationStatusPublisher} from '../services/replicator/replication-status.ts';\nimport {\n  ReplicatorService,\n  type ReplicatorMode,\n} from '../services/replicator/replicator.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {ThreadWriteWorkerClient} from '../services/replicator/write-worker-client.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {\n  deleteStaleChangeLog,\n  getPragmaConfig,\n  replicaFileModeSchema,\n  setUpMessageHandlers,\n  setupReplica,\n  type WalMode,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  assert(args.length > 0, `replicator mode not specified`);\n  const fileMode = v.parse(args[0], replicaFileModeSchema);\n\n  const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n  const mode: ReplicatorMode = fileMode === 'backup' ? 'backup' : 'serving';\n  const runningLocalChangeStreamer =\n    config.changeStreamer.mode === 'dedicated' && !config.changeStreamer.uri;\n  const workerName = `${mode}-replicator`;\n  startOtelAuto(createLogContext(config, workerName, 0, false), workerName, 0);\n  lc = createLogContext(config, workerName);\n  const unregisterInitialCorruptionDiagnosticTarget =\n    registerSQLiteCorruptionDiagnosticTarget(\n      {\n        debugName: `${workerName} replica`,\n        dbPath: config.replica.file,\n      },\n      config.sqliteCorruptionChecks,\n    );\n  initEventSink(lc, config);\n\n  // Remote view-syncers restore from the replication-manager, which supplies\n  // the backup URL. A local change-streamer owns the canonical replica and\n  // must not infer backup configuration from bundled executable paths.\n  if (\n    fileMode === 'serving' &&\n    !runningLocalChangeStreamer &&\n    (config.litestream.executable || config.litestream.executableV5)\n  ) {\n    await restoreReplica(lc, config);\n  }\n\n  const {file: dbPath, walMode} = await setupReplica(\n    lc,\n    fileMode,\n    config.replica,\n  );\n  unregisterInitialCorruptionDiagnosticTarget();\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: `${workerName} replica`,\n      dbPath,\n    },\n    config.sqliteCorruptionChecks,\n  );\n\n  // A file left behind by a run with the writer enabled would otherwise hold\n  // local disk indefinitely. See `replicatorDeletesStaleChangeLog` for why\n  // this is keyed on the config flag and nothing else.\n  deleteStaleChangeLog(config.changeStreamer.sqliteChangeLogMode, dbPath);\n\n  setupMetrics(lc, dbPath, walMode);\n\n  // Create the write worker for async SQLite writes.\n  const pragmas = getPragmaConfig(fileMode);\n  const workerClient = new ThreadWriteWorkerClient();\n  await workerClient.init(dbPath, mode, pragmas, config.log);\n\n  const shard = getShardConfig(config);\n  const {\n    taskID,\n    change,\n    changeStreamer: {\n      port,\n      uri: changeStreamerURI = runningLocalChangeStreamer\n        ? `http://localhost:${port}/`\n        : undefined,\n    },\n  } = config;\n  const changeStreamer = new ChangeStreamerHttpClient(\n    lc,\n    shard,\n    change.db,\n    changeStreamerURI,\n  );\n\n  const replicator = new ReplicatorService(\n    lc,\n    taskID,\n    `${workerName}-${pid}`,\n    mode,\n    changeStreamer,\n    workerClient,\n    runningLocalChangeStreamer\n      ? // publish ReplicationStatusEvents from backup-replicator only\n        ReplicationStatusPublisher.forReplicaFile(dbPath)\n      : null,\n  );\n\n  setUpMessageHandlers(lc, replicator, parent);\n\n  const running = runUntilKilled(lc, parent, replicator);\n\n  // Signal readiness once the first ReplicaVersionReady notification is received.\n  for await (const _ of replicator.subscribe()) {\n    parent.send(['ready', {ready: true}]);\n    break;\n  }\n\n  return running;\n}\n\nfunction setupMetrics(lc: LogContext, file: string, walMode: WalMode) {\n  getOrCreateGauge('replica', 'db_size', {\n    description:\n      `The size of the replica's main db file, ` +\n      `which does not include the wal file(s)`,\n    unit: 'bytes',\n  }).addCallback(observeFileSize(lc, file));\n\n  getOrCreateGauge('replica', 'wal_size', {\n    description: `The size of the replica's wal file`,\n    unit: 'bytes',\n  }).addCallback(observeFileSize(lc, `${file}-wal`));\n\n  if (walMode === 'wal2') {\n    getOrCreateGauge('replica', 'wal2_size', {\n      description: `The size of the replica's wal2 file`,\n      unit: 'bytes',\n    }).addCallback(observeFileSize(lc, `${file}-wal2`));\n  }\n\n  // A whole-database total rather than a per-file one. Its counterpart,\n  // `sqlite_change_log.file_bytes`, is emitted by the change-streamer, which is\n  // the process that writes the log.\n  getOrCreateGauge('replica', 'file_bytes', {\n    description:\n      `The replica's total on-disk footprint: its main db file plus its ` +\n      `wal sidecars.`,\n    unit: 'By',\n  }).addCallback(observeSQLiteFileBytes(lc, file));\n}\n\nfunction observeFileSize(lc: LogContext, file: string): ObservableCallback {\n  return async o => {\n    try {\n      const stats = await stat(file);\n      o.observe(stats.size);\n    } catch (e) {\n      lc.warn?.(`unable to stat ${file} for size metrics`, e);\n    }\n  };\n}\n\nfunction observeSQLiteFileBytes(\n  lc: LogContext,\n  file: string,\n): ObservableCallback {\n  return async o => o.observe(await sqliteFileBytes(lc, file));\n}\n\nconst RETRY_INTERVAL_MS = 3000;\n\n// View-syncers (no replicaConstraints) wait indefinitely for the\n// replication-manager to publish a restorable backup. On a fresh stack the\n// first backup is not durable until the initial sync completes and litestream\n// uploads the initial snapshot, which can take many minutes for a large\n// replica. The platform's startup probe budget (which scales with replica\n// size) is the backstop, so restoreReplica must not impose its own shorter\n// cap and self-terminate while the backup is still being produced.\nasync function restoreReplica(lc: LogContext, config: NormalizedZeroConfig) {\n  const start = performance.now();\n  let backupURL: string | undefined;\n  let result: RestoreResult | undefined;\n  try {\n    for (;;) {\n      const snapshotStatus = await reserveAndGetSnapshotStatus(lc, config);\n      // The backupURL comes from the replication-manager's snapshot response.\n      ({backupURL} = snapshotStatus);\n      const litestream: LitestreamConfig = {...config.litestream, backupURL};\n      const attempt = await tryRestore(\n        lc,\n        litestream,\n        config.replica.file,\n        snapshotStatus,\n        'view_syncer',\n      );\n      if (attempt.restored) {\n        result = attempt.result;\n        return;\n      }\n      lc.info?.(\n        `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n      );\n      await sleep(RETRY_INTERVAL_MS);\n    }\n  } finally {\n    const attrs = litestreamRestoreMetricAttrs(\n      config.litestream,\n      'view_syncer',\n      backupURL,\n    );\n    const labels = {...attrs, result: result ?? 'error'};\n    litestreamRestoreRuns().add(1, labels);\n    litestreamRestoreDuration().recordMs(performance.now() - start, labels);\n  }\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAqDA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,OAAO,KAAK,SAAS,GAAG,+BAA+B;CACvD,MAAM,WAAW,MAAQ,KAAK,IAAI,qBAAqB;CAEvD,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CACjE,MAAM,OAAuB,aAAa,WAAW,WAAW;CAChE,MAAM,6BACJ,OAAO,eAAe,SAAS,eAAe,CAAC,OAAO,eAAe;CACvE,MAAM,aAAa,GAAG,KAAK;CAC3B,cAAc,iBAAiB,QAAQ,YAAY,GAAG,KAAK,GAAG,YAAY,CAAC;CAC3E,KAAK,iBAAiB,QAAQ,UAAU;CACxC,MAAM,8CACJ,yCACE;EACE,WAAW,GAAG,WAAW;EACzB,QAAQ,OAAO,QAAQ;CACzB,GACA,OAAO,sBACT;CACF,cAAc,IAAI,MAAM;CAKxB,IACE,aAAa,aACb,CAAC,+BACA,OAAO,WAAW,cAAc,OAAO,WAAW,eAEnD,MAAM,eAAe,IAAI,MAAM;CAGjC,MAAM,EAAC,MAAM,QAAQ,YAAW,MAAM,aACpC,IACA,UACA,OAAO,OACT;CACA,4CAA4C;CAC5C,yCACE;EACE,WAAW,GAAG,WAAW;EACzB;CACF,GACA,OAAO,sBACT;CAKA,qBAAqB,OAAO,eAAe,qBAAqB,MAAM;CAEtE,aAAa,IAAI,QAAQ,OAAO;CAGhC,MAAM,UAAU,gBAAgB,QAAQ;CACxC,MAAM,eAAe,IAAI,wBAAwB;CACjD,MAAM,aAAa,KAAK,QAAQ,MAAM,SAAS,OAAO,GAAG;CAEzD,MAAM,QAAQ,eAAe,MAAM;CACnC,MAAM,EACJ,QACA,QACA,gBAAgB,EACd,MACA,KAAK,oBAAoB,6BACrB,oBAAoB,KAAK,KACzB,KAAA,QAEJ;CACJ,MAAM,iBAAiB,IAAI,yBACzB,IACA,OACA,OAAO,IACP,iBACF;CAEA,MAAM,aAAa,IAAI,kBACrB,IACA,QACA,GAAG,WAAW,GAAG,OACjB,MACA,gBACA,cACA,6BAEI,2BAA2B,eAAe,MAAM,IAChD,IACN;CAEA,qBAAqB,IAAI,YAAY,MAAM;CAE3C,MAAM,UAAU,eAAe,IAAI,QAAQ,UAAU;CAGrD,WAAW,MAAM,KAAK,WAAW,UAAU,GAAG;EAC5C,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;EACpC;CACF;CAEA,OAAO;AACT;AAEA,SAAS,aAAa,IAAgB,MAAc,SAAkB;CACpE,iBAAiB,WAAW,WAAW;EACrC,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,IAAI,CAAC;CAExC,iBAAiB,WAAW,YAAY;EACtC,aAAa;EACb,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,GAAG,KAAK,KAAK,CAAC;CAEjD,IAAI,YAAY,QACd,iBAAiB,WAAW,aAAa;EACvC,aAAa;EACb,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,GAAG,KAAK,MAAM,CAAC;CAMpD,iBAAiB,WAAW,cAAc;EACxC,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,uBAAuB,IAAI,IAAI,CAAC;AACjD;AAEA,SAAS,gBAAgB,IAAgB,MAAkC;CACzE,OAAO,OAAM,MAAK;EAChB,IAAI;GACF,MAAM,QAAQ,MAAM,KAAK,IAAI;GAC7B,EAAE,QAAQ,MAAM,IAAI;EACtB,SAAS,GAAG;GACV,GAAG,OAAO,kBAAkB,KAAK,oBAAoB,CAAC;EACxD;CACF;AACF;AAEA,SAAS,uBACP,IACA,MACoB;CACpB,OAAO,OAAM,MAAK,EAAE,QAAQ,MAAM,gBAAgB,IAAI,IAAI,CAAC;AAC7D;AAEA,IAAM,oBAAoB;AAS1B,eAAe,eAAe,IAAgB,QAA8B;CAC1E,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI;CACJ,IAAI;CACJ,IAAI;EACF,SAAS;GACP,MAAM,iBAAiB,MAAM,4BAA4B,IAAI,MAAM;GAEnE,CAAC,CAAC,aAAa;GAEf,MAAM,UAAU,MAAM,WACpB,IACA;IAHoC,GAAG,OAAO;IAAY;GAG1D,GACA,OAAO,QAAQ,MACf,gBACA,aACF;GACA,IAAI,QAAQ,UAAU;IACpB,SAAS,QAAQ;IACjB;GACF;GACA,GAAG,OACD,kCAAkC,oBAAoB,IAAK,SAC7D;GACA,MAAM,MAAM,iBAAiB;EAC/B;CACF,UAAU;EAMR,MAAM,SAAS;GAAC,GALF,6BACZ,OAAO,YACP,eACA,SAEiB;GAAO,QAAQ,UAAU;EAAO;EACnD,sBAAsB,EAAE,IAAI,GAAG,MAAM;EACrC,0BAA0B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO,MAAM;CACxE;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}

package/out/zero-cache/src/services/analyze.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"analyze.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/analyze.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,oDAAoD,CAAC;AAC3F,OAAO,KAAK,EAAC,GAAG,EAAC,MAAM,mCAAmC,CAAC;AAC3D,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6CAA6C,CAAC;AAC9E,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,kDAAkD,CAAC;AAWxF,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,iBAAiB,CAAC;AAC7C,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,wBAAwB,CAAC;AAQjE,wBAAsB,YAAY,CAChC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,oBAAoB,EAC5B,YAAY,EAAE,YAAY,EAC1B,GAAG,EAAE,GAAG,EACR,UAAU,UAAO,EACjB,UAAU,UAAQ,EAClB,WAAW,CAAC,EAAE,iBAAiB,EAC/B,IAAI,CAAC,EAAE,OAAO,EACd,SAAS,UAAQ,GAChB,OAAO,CAAC,kBAAkB,CAAC,CA2F7B"}
+{"version":3,"file":"analyze.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/analyze.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,oDAAoD,CAAC;AAC3F,OAAO,KAAK,EAAC,GAAG,EAAC,MAAM,mCAAmC,CAAC;AAC3D,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6CAA6C,CAAC;AAC9E,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,kDAAkD,CAAC;AAWxF,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,iBAAiB,CAAC;AAC7C,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,wBAAwB,CAAC;AAQjE,wBAAsB,YAAY,CAChC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,oBAAoB,EAC5B,YAAY,EAAE,YAAY,EAC1B,GAAG,EAAE,GAAG,EACR,UAAU,UAAO,EACjB,UAAU,UAAQ,EAClB,WAAW,CAAC,EAAE,iBAAiB,EAC/B,IAAI,CAAC,EAAE,OAAO,EACd,SAAS,UAAQ,GAChB,OAAO,CAAC,kBAAkB,CAAC,CA4F7B"}

package/out/zero-cache/src/services/analyze.js

@@ -36,6 +36,7 @@ async function analyzeQuery(lc, config, clientSchema, ast, syncedRows = true, ve
 			planDebugger,
 			host: {
 				debug: new Debug(),
+				enableNotExists: true,
 				getSource(tableName) {
 					let source = tables.get(tableName);
 					if (source) return source;

package/out/zero-cache/src/services/analyze.js.map

@@ -1 +1 @@
-{"version":3,"file":"analyze.js","names":[],"sources":["../../../../../zero-cache/src/services/analyze.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {AnalyzeQueryResult} from '../../../zero-protocol/src/analyze-query-result.ts';\nimport type {AST} from '../../../zero-protocol/src/ast.ts';\nimport type {ClientSchema} from '../../../zero-protocol/src/client-schema.ts';\nimport type {PermissionsConfig} from '../../../zero-schema/src/compiled-permissions.ts';\nimport {Debug} from '../../../zql/src/builder/debug-delegate.ts';\nimport {MemoryStorage} from '../../../zql/src/ivm/memory-storage.ts';\nimport {\n  AccumulatorDebugger,\n  serializePlanDebugEvents,\n} from '../../../zql/src/planner/planner-debug.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport {explainQueries} from '../../../zqlite/src/explain-queries.ts';\nimport {createSQLiteCostModel} from '../../../zqlite/src/sqlite-cost-model.ts';\nimport {TableSource} from '../../../zqlite/src/table-source.ts';\nimport type {JWTAuth} from '../auth/auth.ts';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {computeZqlSpecs, mustGetTableSpec} from '../db/lite-tables.ts';\nimport type {LiteAndZqlSpec, LiteTableSpec} from '../db/specs.ts';\nimport {runAst} from './run-ast.ts';\nimport {TimeSliceTimer} from './view-syncer/view-syncer.ts';\n\nconst TIME_SLICE_LAP_THRESHOLD_MS = 200;\n\nexport async function analyzeQuery(\n  lc: LogContext,\n  config: NormalizedZeroConfig,\n  clientSchema: ClientSchema,\n  ast: AST,\n  syncedRows = true,\n  vendedRows = false,\n  permissions?: PermissionsConfig,\n  auth?: JWTAuth,\n  joinPlans = false,\n): Promise<AnalyzeQueryResult> {\n  using db = new Database(lc, config.replica.file);\n  const fullTables = new Map<string, LiteTableSpec>();\n  const tableSpecs = new Map<string, LiteAndZqlSpec>();\n  const tables = new Map<string, TableSource>();\n\n  computeZqlSpecs(\n    lc,\n    db,\n    {includeBackfillingColumns: false},\n    tableSpecs,\n    fullTables,\n  );\n\n  // Mirror production: the planner runs iff ZERO_ENABLE_QUERY_PLANNER is set\n  // on the server, so the analysis reflects what actually executes. Diagnostic\n  // event collection is orthogonal and opt-in via `joinPlans`.\n  const costModel = config.enableQueryPlanner\n    ? createSQLiteCostModel(db, tableSpecs)\n    : undefined;\n  const planDebugger = joinPlans ? new AccumulatorDebugger() : undefined;\n  const timer = await new TimeSliceTimer(lc).start();\n  const shouldYield = () => timer.elapsedLap() > TIME_SLICE_LAP_THRESHOLD_MS;\n  const yieldProcess = () => timer.yieldProcess();\n  const result = await runAst(\n    lc,\n    clientSchema,\n    ast,\n    true,\n    {\n      applyPermissions: permissions !== undefined,\n      syncedRows,\n      vendedRows,\n      auth,\n      db,\n      tableSpecs,\n      permissions,\n      costModel,\n      planDebugger,\n      host: {\n        debug: new Debug(),\n        getSource(tableName: string) {\n          let source = tables.get(tableName);\n          if (source) {\n            return source;\n          }\n\n          const tableSpec = mustGetTableSpec(tableSpecs, tableName);\n          const {primaryKey} = tableSpec.tableSpec;\n\n          source = new TableSource(\n            lc,\n            config.log,\n            db,\n            tableName,\n            tableSpec.zqlSpec,\n            primaryKey,\n            shouldYield,\n          );\n          tables.set(tableName, source);\n          return source;\n        },\n        createStorage() {\n          return new MemoryStorage();\n        },\n        decorateSourceInput: input => input,\n        decorateInput: input => input,\n        addEdge() {},\n        decorateFilterInput: input => input,\n      },\n    },\n    yieldProcess,\n  );\n\n  // Fill in plans for any queries SQLite did not actually execute (and thus\n  // did not populate scanStatus EXPLAIN for) using the substituted-binding\n  // fallback. Plans captured at execution time use the real bindings and win.\n  const fallback = explainQueries(result.readRowCountsByQuery ?? {}, db);\n  const captured = result.sqlitePlans ?? {};\n  for (const [query, plan] of Object.entries(fallback)) {\n    if (!captured[query]) {\n      captured[query] = plan;\n    }\n  }\n  result.sqlitePlans = captured;\n\n  if (planDebugger) {\n    result.joinPlans = serializePlanDebugEvents(planDebugger.events);\n  }\n\n  return result;\n}\n"],"mappings":";;;;;;;;;;;;AAsBA,IAAM,8BAA8B;AAEpC,eAAsB,aACpB,IACA,QACA,cACA,KACA,aAAa,MACb,aAAa,OACb,aACA,MACA,YAAY,OACiB;;;EAC7B,MAAM,KAAA,YAAA,EAAK,IAAI,SAAS,IAAI,OAAO,QAAQ,IAAI,CAAA;EAC/C,MAAM,6BAAa,IAAI,IAA2B;EAClD,MAAM,6BAAa,IAAI,IAA4B;EACnD,MAAM,yBAAS,IAAI,IAAyB;EAE5C,gBACE,IACA,IACA,EAAC,2BAA2B,MAAK,GACjC,YACA,UACF;EAKA,MAAM,YAAY,OAAO,qBACrB,sBAAsB,IAAI,UAAU,IACpC,KAAA;EACJ,MAAM,eAAe,YAAY,IAAI,oBAAoB,IAAI,KAAA;EAC7D,MAAM,QAAQ,MAAM,IAAI,eAAe,EAAE,EAAE,MAAM;EACjD,MAAM,oBAAoB,MAAM,WAAW,IAAI;EAC/C,MAAM,qBAAqB,MAAM,aAAa;EAC9C,MAAM,SAAS,MAAM,OACnB,IACA,cACA,KACA,MACA;GACE,kBAAkB,gBAAgB,KAAA;GAClC;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA,MAAM;IACJ,OAAO,IAAI,MAAM;IACjB,UAAU,WAAmB;KAC3B,IAAI,SAAS,OAAO,IAAI,SAAS;KACjC,IAAI,QACF,OAAO;KAGT,MAAM,YAAY,iBAAiB,YAAY,SAAS;KACxD,MAAM,EAAC,eAAc,UAAU;KAE/B,SAAS,IAAI,YACX,IACA,OAAO,KACP,IACA,WACA,UAAU,SACV,YACA,WACF;KACA,OAAO,IAAI,WAAW,MAAM;KAC5B,OAAO;IACT;IACA,gBAAgB;KACd,OAAO,IAAI,cAAc;IAC3B;IACA,sBAAqB,UAAS;IAC9B,gBAAe,UAAS;IACxB,UAAU,CAAC;IACX,sBAAqB,UAAS;GAChC;EACF,GACA,YACF;EAKA,MAAM,WAAW,eAAe,OAAO,wBAAwB,CAAC,GAAG,EAAE;EACrE,MAAM,WAAW,OAAO,eAAe,CAAC;EACxC,KAAK,MAAM,CAAC,OAAO,SAAS,OAAO,QAAQ,QAAQ,GACjD,IAAI,CAAC,SAAS,QACZ,SAAS,SAAS;EAGtB,OAAO,cAAc;EAErB,IAAI,cACF,OAAO,YAAY,yBAAyB,aAAa,MAAM;EAGjE,OAAO;;;;;;AACT"}
+{"version":3,"file":"analyze.js","names":[],"sources":["../../../../../zero-cache/src/services/analyze.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {AnalyzeQueryResult} from '../../../zero-protocol/src/analyze-query-result.ts';\nimport type {AST} from '../../../zero-protocol/src/ast.ts';\nimport type {ClientSchema} from '../../../zero-protocol/src/client-schema.ts';\nimport type {PermissionsConfig} from '../../../zero-schema/src/compiled-permissions.ts';\nimport {Debug} from '../../../zql/src/builder/debug-delegate.ts';\nimport {MemoryStorage} from '../../../zql/src/ivm/memory-storage.ts';\nimport {\n  AccumulatorDebugger,\n  serializePlanDebugEvents,\n} from '../../../zql/src/planner/planner-debug.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport {explainQueries} from '../../../zqlite/src/explain-queries.ts';\nimport {createSQLiteCostModel} from '../../../zqlite/src/sqlite-cost-model.ts';\nimport {TableSource} from '../../../zqlite/src/table-source.ts';\nimport type {JWTAuth} from '../auth/auth.ts';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {computeZqlSpecs, mustGetTableSpec} from '../db/lite-tables.ts';\nimport type {LiteAndZqlSpec, LiteTableSpec} from '../db/specs.ts';\nimport {runAst} from './run-ast.ts';\nimport {TimeSliceTimer} from './view-syncer/view-syncer.ts';\n\nconst TIME_SLICE_LAP_THRESHOLD_MS = 200;\n\nexport async function analyzeQuery(\n  lc: LogContext,\n  config: NormalizedZeroConfig,\n  clientSchema: ClientSchema,\n  ast: AST,\n  syncedRows = true,\n  vendedRows = false,\n  permissions?: PermissionsConfig,\n  auth?: JWTAuth,\n  joinPlans = false,\n): Promise<AnalyzeQueryResult> {\n  using db = new Database(lc, config.replica.file);\n  const fullTables = new Map<string, LiteTableSpec>();\n  const tableSpecs = new Map<string, LiteAndZqlSpec>();\n  const tables = new Map<string, TableSource>();\n\n  computeZqlSpecs(\n    lc,\n    db,\n    {includeBackfillingColumns: false},\n    tableSpecs,\n    fullTables,\n  );\n\n  // Mirror production: the planner runs iff ZERO_ENABLE_QUERY_PLANNER is set\n  // on the server, so the analysis reflects what actually executes. Diagnostic\n  // event collection is orthogonal and opt-in via `joinPlans`.\n  const costModel = config.enableQueryPlanner\n    ? createSQLiteCostModel(db, tableSpecs)\n    : undefined;\n  const planDebugger = joinPlans ? new AccumulatorDebugger() : undefined;\n  const timer = await new TimeSliceTimer(lc).start();\n  const shouldYield = () => timer.elapsedLap() > TIME_SLICE_LAP_THRESHOLD_MS;\n  const yieldProcess = () => timer.yieldProcess();\n  const result = await runAst(\n    lc,\n    clientSchema,\n    ast,\n    true,\n    {\n      applyPermissions: permissions !== undefined,\n      syncedRows,\n      vendedRows,\n      auth,\n      db,\n      tableSpecs,\n      permissions,\n      costModel,\n      planDebugger,\n      host: {\n        debug: new Debug(),\n        enableNotExists: true,\n        getSource(tableName: string) {\n          let source = tables.get(tableName);\n          if (source) {\n            return source;\n          }\n\n          const tableSpec = mustGetTableSpec(tableSpecs, tableName);\n          const {primaryKey} = tableSpec.tableSpec;\n\n          source = new TableSource(\n            lc,\n            config.log,\n            db,\n            tableName,\n            tableSpec.zqlSpec,\n            primaryKey,\n            shouldYield,\n          );\n          tables.set(tableName, source);\n          return source;\n        },\n        createStorage() {\n          return new MemoryStorage();\n        },\n        decorateSourceInput: input => input,\n        decorateInput: input => input,\n        addEdge() {},\n        decorateFilterInput: input => input,\n      },\n    },\n    yieldProcess,\n  );\n\n  // Fill in plans for any queries SQLite did not actually execute (and thus\n  // did not populate scanStatus EXPLAIN for) using the substituted-binding\n  // fallback. Plans captured at execution time use the real bindings and win.\n  const fallback = explainQueries(result.readRowCountsByQuery ?? {}, db);\n  const captured = result.sqlitePlans ?? {};\n  for (const [query, plan] of Object.entries(fallback)) {\n    if (!captured[query]) {\n      captured[query] = plan;\n    }\n  }\n  result.sqlitePlans = captured;\n\n  if (planDebugger) {\n    result.joinPlans = serializePlanDebugEvents(planDebugger.events);\n  }\n\n  return result;\n}\n"],"mappings":";;;;;;;;;;;;AAsBA,IAAM,8BAA8B;AAEpC,eAAsB,aACpB,IACA,QACA,cACA,KACA,aAAa,MACb,aAAa,OACb,aACA,MACA,YAAY,OACiB;;;EAC7B,MAAM,KAAA,YAAA,EAAK,IAAI,SAAS,IAAI,OAAO,QAAQ,IAAI,CAAA;EAC/C,MAAM,6BAAa,IAAI,IAA2B;EAClD,MAAM,6BAAa,IAAI,IAA4B;EACnD,MAAM,yBAAS,IAAI,IAAyB;EAE5C,gBACE,IACA,IACA,EAAC,2BAA2B,MAAK,GACjC,YACA,UACF;EAKA,MAAM,YAAY,OAAO,qBACrB,sBAAsB,IAAI,UAAU,IACpC,KAAA;EACJ,MAAM,eAAe,YAAY,IAAI,oBAAoB,IAAI,KAAA;EAC7D,MAAM,QAAQ,MAAM,IAAI,eAAe,EAAE,EAAE,MAAM;EACjD,MAAM,oBAAoB,MAAM,WAAW,IAAI;EAC/C,MAAM,qBAAqB,MAAM,aAAa;EAC9C,MAAM,SAAS,MAAM,OACnB,IACA,cACA,KACA,MACA;GACE,kBAAkB,gBAAgB,KAAA;GAClC;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA,MAAM;IACJ,OAAO,IAAI,MAAM;IACjB,iBAAiB;IACjB,UAAU,WAAmB;KAC3B,IAAI,SAAS,OAAO,IAAI,SAAS;KACjC,IAAI,QACF,OAAO;KAGT,MAAM,YAAY,iBAAiB,YAAY,SAAS;KACxD,MAAM,EAAC,eAAc,UAAU;KAE/B,SAAS,IAAI,YACX,IACA,OAAO,KACP,IACA,WACA,UAAU,SACV,YACA,WACF;KACA,OAAO,IAAI,WAAW,MAAM;KAC5B,OAAO;IACT;IACA,gBAAgB;KACd,OAAO,IAAI,cAAc;IAC3B;IACA,sBAAqB,UAAS;IAC9B,gBAAe,UAAS;IACxB,UAAU,CAAC;IACX,sBAAqB,UAAS;GAChC;EACF,GACA,YACF;EAKA,MAAM,WAAW,eAAe,OAAO,wBAAwB,CAAC,GAAG,EAAE;EACrE,MAAM,WAAW,OAAO,eAAe,CAAC;EACxC,KAAK,MAAM,CAAC,OAAO,SAAS,OAAO,QAAQ,QAAQ,GACjD,IAAI,CAAC,SAAS,QACZ,SAAS,SAAS;EAGtB,OAAO,cAAc;EAErB,IAAI,cACF,OAAO,YAAY,yBAAyB,aAAa,MAAM;EAGjE,OAAO;;;;;;AACT"}

package/out/zero-cache/src/services/change-source/common/backfill-manager.d.ts

@@ -2,7 +2,17 @@ import type { LogContext } from '@rocicorp/logger';
 import { type JSONFormat } from '../../../types/lite.ts';
 import type { BackfillCompleted, BackfillRequest, ChangeStreamMessage, MessageBackfill } from '../protocol/current.ts';
 import type { Cancelable, ChangeStreamMultiplexer, Listener } from './change-stream-multiplexer.ts';
-type BackfillStreamer = (req: BackfillRequest) => AsyncGenerator<MessageBackfill | BackfillCompleted>;
+/**
+ * A {@link MessageBackfill} or {@link BackfillCompleted} paired with the
+ * approximate number of upstream bytes it represents. The BackfillManager uses
+ * `byteSize` to bound the size of each replica transaction, committing and
+ * reopening once {@link COMMIT_THRESHOLD_BYTES} is reached.
+ */
+export type BackfillMessage = {
+    message: MessageBackfill | BackfillCompleted;
+    byteSize: number;
+};
+type BackfillStreamer = (req: BackfillRequest) => AsyncGenerator<BackfillMessage>;
 /**
  * The BackfillManager initiates backfills for BackfillRequests from the
  * change-streamer (i.e. unfinished backfills from previous sessions)
@@ -21,7 +31,7 @@ type BackfillStreamer = (req: BackfillRequest) => AsyncGenerator<MessageBackfill
  */
 export declare class BackfillManager implements Cancelable, Listener {
     #private;
-    constructor(lc: LogContext, changeStreamer: ChangeStreamMultiplexer, backfillStreamer: BackfillStreamer, jsonFormat?: JSONFormat, minBackoffMs?: number, maxBackoffMs?: number);
+    constructor(lc: LogContext, changeStreamer: ChangeStreamMultiplexer, backfillStreamer: BackfillStreamer, jsonFormat?: JSONFormat, minBackoffMs?: number, maxBackoffMs?: number, commitThresholdBytes?: number);
     run(lastWatermark: string, initialRequests: BackfillRequest[]): void;
     /**
      * Implements {@link Listener.onChange()}, invoked by the

package/out/zero-cache/src/services/change-source/common/backfill-manager.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"backfill-manager.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/common/backfill-manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAOjD,OAAO,EAAmB,KAAK,UAAU,EAAC,MAAM,wBAAwB,CAAC;AAKzE,OAAO,KAAK,EACV,iBAAiB,EACjB,eAAe,EACf,mBAAmB,EAEnB,eAAe,EAChB,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EACV,UAAU,EACV,uBAAuB,EACvB,QAAQ,EACT,MAAM,gCAAgC,CAAC;AAMxC,KAAK,gBAAgB,GAAG,CACtB,GAAG,EAAE,eAAe,KACjB,cAAc,CAAC,eAAe,GAAG,iBAAiB,CAAC,CAAC;AAgBzD;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,eAAgB,YAAW,UAAU,EAAE,QAAQ;;gBA+BxD,EAAE,EAAE,UAAU,EACd,cAAc,EAAE,uBAAuB,EACvC,gBAAgB,EAAE,gBAAgB,EAClC,UAAU,GAAE,UAA6B,EACzC,YAAY,SAA0B,EACtC,YAAY,SAA0B;IAWxC,GAAG,CAAC,aAAa,EAAE,MAAM,EAAE,eAAe,EAAE,eAAe,EAAE;IAsP7D;;;OAGG;IACH,QAAQ,CAAC,OAAO,EAAE,mBAAmB,GAAG,IAAI;IA6M5C,MAAM,IAAI,IAAI;CAIf;AAED,uBAAe,mBAAoB,SAAQ,KAAK;gBAClC,EAAE,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO;CAO9D;AA6BD;;;;;GAKG;AACH,qBAAa,0BAA2B,SAAQ,mBAAmB;IACjE,QAAQ,CAAC,IAAI,gCAAgC;gBAEjC,EAAE,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO;CAG9D"}
+{"version":3,"file":"backfill-manager.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/common/backfill-manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAOjD,OAAO,EAAmB,KAAK,UAAU,EAAC,MAAM,wBAAwB,CAAC;AAKzE,OAAO,KAAK,EACV,iBAAiB,EACjB,eAAe,EACf,mBAAmB,EAEnB,eAAe,EAChB,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EACV,UAAU,EACV,uBAAuB,EACvB,QAAQ,EACT,MAAM,gCAAgC,CAAC;AAMxC;;;;;GAKG;AACH,MAAM,MAAM,eAAe,GAAG;IAC5B,OAAO,EAAE,eAAe,GAAG,iBAAiB,CAAC;IAC7C,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,KAAK,gBAAgB,GAAG,CACtB,GAAG,EAAE,eAAe,KACjB,cAAc,CAAC,eAAe,CAAC,CAAC;AAuBrC;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,eAAgB,YAAW,UAAU,EAAE,QAAQ;;gBAiCxD,EAAE,EAAE,UAAU,EACd,cAAc,EAAE,uBAAuB,EACvC,gBAAgB,EAAE,gBAAgB,EAClC,UAAU,GAAE,UAA6B,EACzC,YAAY,SAA0B,EACtC,YAAY,SAA0B,EACtC,oBAAoB,SAAyB;IAY/C,GAAG,CAAC,aAAa,EAAE,MAAM,EAAE,eAAe,EAAE,eAAe,EAAE;IA+P7D;;;OAGG;IACH,QAAQ,CAAC,OAAO,EAAE,mBAAmB,GAAG,IAAI;IA6M5C,MAAM,IAAI,IAAI;CAIf;AAED,uBAAe,mBAAoB,SAAQ,KAAK;gBAClC,EAAE,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO;CAO9D;AA6BD;;;;;GAKG;AACH,qBAAa,0BAA2B,SAAQ,mBAAmB;IACjE,QAAQ,CAAC,IAAI,gCAAgC;gBAEjC,EAAE,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO;CAG9D"}

package/out/zero-cache/src/services/change-source/common/backfill-manager.js

@@ -10,6 +10,12 @@ import { resolver } from "@rocicorp/resolver";
 function tableKey({ schema, name }) {
 	return `${schema}.${name}`;
 }
+/**
+* Commit (and reopen) the current backfill transaction once this many bytes of
+* backfill data have accumulated in it. Bounds the size of a single replica
+* COMMIT/checkpoint so it can't monopolize the replica.
+*/
+var COMMIT_THRESHOLD_BYTES = 8 * 1024 * 1024;
 var MIN_BACKOFF_INTERVAL_MS = 2e3;
 var MAX_BACKOFF_INTERVAL_MS = 6e4;
 /**
@@ -50,7 +56,8 @@ var BackfillManager = class {
 	#awaitingStatusWatermarks = [];
 	/** The watermark of the current transaction in the change stream. */
 	#currentTxWatermark = null;
-	constructor(lc, changeStreamer, backfillStreamer, jsonFormat = "s", minBackoffMs = MIN_BACKOFF_INTERVAL_MS, maxBackoffMs = MAX_BACKOFF_INTERVAL_MS) {
+	#commitThresholdBytes;
+	constructor(lc, changeStreamer, backfillStreamer, jsonFormat = "s", minBackoffMs = MIN_BACKOFF_INTERVAL_MS, maxBackoffMs = MAX_BACKOFF_INTERVAL_MS, commitThresholdBytes = COMMIT_THRESHOLD_BYTES) {
 		this.#lc = lc.withContext("component", "backfill-manager");
 		this.#changeStreamer = changeStreamer;
 		this.#backfillStreamer = backfillStreamer;
@@ -58,6 +65,7 @@ var BackfillManager = class {
 		this.#minBackoffMs = minBackoffMs;
 		this.#maxBackoffMs = maxBackoffMs;
 		this.#retryDelayMs = minBackoffMs;
+		this.#commitThresholdBytes = commitThresholdBytes;
 	}
 	run(lastWatermark, initialRequests) {
 		this.#lc.info?.(`starting backfill manager with ${initialRequests.length} initial requests`, { requests: initialRequests });
@@ -124,6 +132,7 @@ var BackfillManager = class {
 	async #runBackfill(lc, state) {
 		const changeStream = this.#changeStreamer;
 		let backfillTx = null;
+		let uncommittedBytes = 0;
 		/**
 		* @returns the new tx watermark, or null if backfill was cancelled
 		*/
@@ -165,10 +174,11 @@ var BackfillManager = class {
 				changeStream.release(backfillTx);
 			}
 			backfillTx = null;
+			uncommittedBytes = 0;
 		};
-		for await (const msg of this.#backfillStreamer(state.request)) {
+		for await (const { message: msg, byteSize } of this.#backfillStreamer(state.request)) {
 			const mustWaitBeforeFlush = msg.tag === "backfill-completed" && (this.#changeStreamReached(lc, msg.watermark) || backfillTx !== null && backfillTx < msg.watermark);
-			if (backfillTx && (changeStream.waiterDelay() > 0 || mustWaitBeforeFlush)) commitTx();
+			if (backfillTx && (changeStream.waiterDelay() > 0 || mustWaitBeforeFlush || uncommittedBytes >= this.#commitThresholdBytes)) commitTx();
 			mustWaitBeforeFlush && await mustWaitBeforeFlush;
 			if (msg.tag === "backfill" && msg.rowValues.length > 0 && msg.relation.rowKey.columns.length === 0) throw new MissingRowKeyError(state.request);
 			if ((backfillTx ??= await beginTxFor(msg)) === null) {
@@ -177,6 +187,7 @@ var BackfillManager = class {
 				return;
 			}
 			await changeStream.push(["data", msg]);
+			uncommittedBytes += byteSize;
 		}
 		backfillTx && commitTx();
 		lc.debug?.(`backfill stream exited`, state.canceledReason ?? "");