@rocicorp/zero 1.3.0-canary.0 → 1.3.0-canary.2

package/out/zero-cache/src/services/life-cycle.js.map

@@ -1 +1 @@
-{"version":3,"file":"life-cycle.js","names":["#lc","#userFacing","#all","#exitImpl","#start","#ready","#startDrain","#kill","#exit","#runningState","#drainStart","#onExit","#initializing","#nextID","#stopInterval","#lastHeartbeat","#checkIntervalTimer","#checkStopInterval","#checkImmediateTimer"],"sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {IncomingHttpHeaders} from 'node:http';\nimport {pid} from 'node:process';\nimport type {EventEmitter} from 'stream';\nimport {\n  singleProcessMode,\n  type Subprocess,\n  type Worker,\n} from '../types/processes.ts';\nimport {RunningState} from './running-state.ts';\nimport type {SingletonService} from './service.ts';\n\n/**\n * * `user-facing` workers serve external requests and are the first to\n *   receive a `SIGTERM` or `SIGINT` signal for graceful shutdown.\n *\n * * `supporting` workers support `user-facing` workers and are sent\n *   the `SIGTERM` signal only after all `user-facing` workers have\n *   exited.\n *\n * For other kill signals, such as `SIGQUIT` and `SIGABRT`, all workers\n * are stopped without draining. `SIGQUIT` is used to represent an\n * intentional shutdown (for which draining is not beneficial), whereas\n * `SIGABRT` is used for unexpected process exits.\n */\nexport type WorkerType = 'user-facing' | 'supporting';\n\nexport const GRACEFUL_SHUTDOWN = ['SIGTERM', 'SIGINT'] as const;\nexport const FORCEFUL_SHUTDOWN = ['SIGQUIT', 'SIGABRT'] as const;\n\ntype GracefulShutdownSignal = (typeof GRACEFUL_SHUTDOWN)[number];\n\n// An internal error code used to indicate that a message has already been\n// logged at level ERRROR. When a process exits with this error code, the\n// parent process logs the exit at level WARN instead of ERROR.\nexport const UNHANDLED_EXCEPTION_ERROR_CODE = 13;\n\n// An internal error code used to indicate that the server should exit\n// without draining (e.g. due to a supporting worker get a signal to shut\n// down), but the exit is otherwise intentional.\nconst INTENTIONAL_SHUTDOWN_ERROR_CODE = 14;\n\n/**\n * Handles readiness, termination signals, and coordination of graceful\n * shutdown.\n */\nexport class ProcessManager {\n  readonly #lc: LogContext;\n  readonly #userFacing = new Set<Subprocess>();\n  readonly #all = new Set<Subprocess>();\n  readonly #exitImpl: (code: number) => never;\n  readonly #start = Date.now();\n  readonly #ready: Promise<void>[] = [];\n\n  #runningState = new RunningState('process-manager');\n  #drainStart = 0;\n\n  constructor(lc: LogContext, proc: EventEmitter) {\n    this.#lc = lc.withContext('component', 'process-manager');\n\n    // Propagate `SIGTERM` and `SIGINT` to all user-facing workers,\n    // initiating a graceful shutdown. The parent process will\n    // exit once all user-facing workers have exited ...\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.on(signal, () => this.#startDrain(signal));\n    }\n\n    // ... which will result in sending `SIGTERM` to the remaining workers.\n    proc.on('exit', code =>\n      this.#kill(\n        this.#all,\n        code === 0\n          ? 'SIGTERM' // graceful, drained shutdown\n          : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n            ? 'SIGQUIT' // intentional abort without drain\n            : 'SIGABRT', // unintentional shutdown, alertable error\n      ),\n    );\n\n    // For other (catchable) kill signals, exit with a non-zero error code\n    // to send a `SIGQUIT` (intentional shutdown) or `SIGABRT` (unexpected\n    // shutdown) to all workers. For these signals, workers are stopped\n    // immediately without draining, since there is no merit to slowly draining\n    // when supporting workers have stopped.\n    //\n    // The logic for handling these signals is in `runUntilKilled()`.\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      proc.on(signal, () =>\n        this.#exit(signal === 'SIGQUIT' ? INTENTIONAL_SHUTDOWN_ERROR_CODE : -1),\n      );\n    }\n\n    this.#exitImpl = (code: number) => {\n      if (singleProcessMode()) {\n        return proc.emit('exit', code) as never; // For unit / integration tests.\n      }\n      process.exit(code);\n    };\n  }\n\n  done() {\n    return this.#runningState.stopped();\n  }\n\n  #exit(code: number) {\n    this.#lc.info?.('exiting with code', code);\n    this.#runningState.stop(this.#lc);\n    void this.#lc.flush().finally(() => this.#exitImpl(code));\n  }\n\n  #startDrain(signal: GracefulShutdownSignal) {\n    this.#lc.info?.(`initiating drain (${signal})`);\n    this.#drainStart = Date.now();\n    if (this.#userFacing.size) {\n      this.#kill(this.#userFacing, signal);\n    } else {\n      this.#kill(this.#all, signal);\n    }\n  }\n\n  addSubprocess(proc: Subprocess, type: WorkerType, name: string) {\n    if (type === 'user-facing') {\n      this.#userFacing.add(proc);\n    }\n    this.#all.add(proc);\n\n    let isOpen = true;\n    proc.on('close', (code, signal) => {\n      isOpen = false;\n      this.#onExit(code, signal, null, type, name, proc);\n    });\n\n    // As per https://nodejs.org/api/child_process.html#event-error\n    // 'error' events can happen when sending a message to a child process\n    // fails. This is not really an error when the server is shutting down,\n    // so log any post-close errors at 'warn'.\n    proc.on('error', err =>\n      this.#lc[isOpen ? 'error' : 'warn']?.(\n        `error from ${name} ${proc.pid}`,\n        err,\n      ),\n    );\n  }\n\n  readonly #initializing = new Map<number, string>();\n  #nextID = 0;\n\n  addWorker(worker: Worker, type: WorkerType, name: string): Worker {\n    this.addSubprocess(worker, type, name);\n\n    const id = ++this.#nextID;\n    this.#initializing.set(id, name);\n    const {promise, resolve} = resolver();\n    this.#ready.push(promise);\n\n    worker.onceMessageType('ready', () => {\n      this.#lc.debug?.(`${name} ready (${Date.now() - this.#start} ms)`);\n      this.#initializing.delete(id);\n      resolve();\n    });\n\n    return worker;\n  }\n\n  initializing(): string[] {\n    return [...this.#initializing.values()];\n  }\n\n  async allWorkersReady() {\n    await Promise.all(this.#ready);\n  }\n\n  logErrorAndExit(err: unknown, name: string) {\n    // only accessible by the main (i.e. user-facing) process.\n    this.#onExit(-1, null, err, 'user-facing', name, undefined);\n  }\n\n  #onExit(\n    code: number,\n    sig: NodeJS.Signals | null,\n    err: unknown,\n    type: WorkerType,\n    name: string,\n    worker: Subprocess | undefined,\n  ) {\n    // Remove the worker from maps to avoid attempting to send more signals to it.\n    if (worker) {\n      this.#userFacing.delete(worker);\n      this.#all.delete(worker);\n    }\n\n    const pid = worker?.pid ?? process.pid;\n\n    if (type === 'supporting') {\n      // Supporting workers like the replication-manager shut down without a\n      // drain signal when receiving protocol-specific instructions (like auto\n      // reset). In this case, a special error code is used to signal that the\n      // server should be shut down without draining, but it is otherwise not\n      // considered an unexpected/alertable error.\n      if (code === 0 && (this.#drainStart === 0 || this.#userFacing.size > 0)) {\n        code = INTENTIONAL_SHUTDOWN_ERROR_CODE;\n      }\n      const log =\n        code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n          ? 'info'\n          : 'warn';\n      this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n      return this.#exit(code);\n    }\n\n    const log =\n      code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n        ? 'info'\n        : this.#drainStart > 0 || code === UNHANDLED_EXCEPTION_ERROR_CODE\n          ? 'warn'\n          : 'error';\n    this.#lc[log]?.(\n      sig\n        ? `${name} (${pid}) killed with (${sig})`\n        : `${name} (${pid}) exited with code (${code})`,\n      err ?? '',\n    );\n\n    // user-facing workers exited or finished draining.\n    if (this.#userFacing.size === 0) {\n      this.#lc.info?.(\n        this.#drainStart\n          ? `all user-facing workers drained (${\n              Date.now() - this.#drainStart\n            } ms)`\n          : `all user-facing workers exited`,\n      );\n      return this.#exit(0);\n    }\n\n    if (this.#drainStart === 0) {\n      // If a user-facing worker exits without receiving a drain signal,\n      // shutdown the server.\n      return this.#exit(code || -1);\n    }\n\n    return undefined;\n  }\n\n  #kill(workers: Iterable<Subprocess>, signal: NodeJS.Signals) {\n    for (const worker of workers) {\n      try {\n        worker.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\n/**\n * Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with\n * an optional {@link SingletonService.drain drain()}, or stopping them\n * without draining for `SIGQUIT`.\n *\n * @returns a Promise that resolves/rejects when any of the services stops/throws.\n */\n\nexport async function runUntilKilled(\n  lc: LogContext,\n  parent: EventEmitter,\n  ...services: SingletonService[]\n): Promise<void> {\n  if (services.length === 0) {\n    return;\n  }\n  for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) {\n    parent.once(signal, () => {\n      const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN as readonly NodeJS.Signals[];\n\n      services.forEach(async svc => {\n        if (GRACEFUL_SIGNALS.includes(signal) && svc.drain) {\n          lc.info?.(`draining ${svc.constructor.name} ${svc.id} (${signal})`);\n          await svc.drain();\n        }\n        lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${signal})`);\n        await svc.stop();\n      });\n    });\n  }\n\n  try {\n    // Run all services and resolve when any of them stops.\n    const svc = await Promise.race(\n      services.map(svc => svc.run().then(() => svc)),\n    );\n    lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);\n  } catch (e) {\n    lc.error?.(`exiting on error`, e);\n    throw e;\n  }\n}\n\nexport async function exitAfter(run: () => Promise<void>) {\n  try {\n    await run();\n    // oxlint-disable-next-line no-console\n    console.info(`pid ${pid} exiting normally`);\n    process.exit(0);\n  } catch (e) {\n    // oxlint-disable-next-line no-console\n    console.error(`pid ${pid} exiting with error`, e);\n    process.exit(-1);\n  }\n}\n\nconst DEFAULT_STOP_INTERVAL_MS = 20_000;\n\n/**\n * The HeartbeatMonitor monitors the cadence heartbeats (e.g. \"/keepalive\"\n * health checks made to HttpServices) that signal that the server\n * should continue processing requests. When a configurable `stopInterval`\n * elapses without receiving these heartbeats, the monitor initiates a\n * graceful shutdown of the server. This works with common load balancing\n * frameworks such as AWS Elastic Load Balancing.\n *\n * The HeartbeatMonitor is **opt-in** in that it only kicks in after it\n * starts receiving keepalives.\n */\nexport class HeartbeatMonitor {\n  readonly #stopInterval: number;\n\n  #lc: LogContext;\n  #checkIntervalTimer: NodeJS.Timeout | undefined;\n  #checkImmediateTimer: NodeJS.Immediate | undefined;\n  #lastHeartbeat = 0;\n\n  constructor(lc: LogContext, stopInterval = DEFAULT_STOP_INTERVAL_MS) {\n    this.#lc = lc;\n    this.#stopInterval = stopInterval;\n  }\n\n  onHeartbeat(reqHeaders: IncomingHttpHeaders) {\n    this.#lastHeartbeat = Date.now();\n    if (this.#checkIntervalTimer === undefined) {\n      this.#lc.info?.(\n        `starting heartbeat monitor at ${\n          this.#stopInterval / 1000\n        } second interval`,\n        reqHeaders,\n      );\n      // e.g. check every 5 seconds to see if it's been over 20 seconds\n      //      since the last heartbeat.\n      this.#checkIntervalTimer = setInterval(\n        this.#checkStopInterval,\n        this.#stopInterval / 4,\n      );\n    }\n  }\n\n  #checkStopInterval = () => {\n    // In the Node.js event loop, timers like setInterval and setTimeout\n    // run *before* I/O events coming from network sockets or file reads/writes.\n    // When this process gets starved of CPU resources for long periods of time,\n    // for example when other processes are monopolizing all available cores,\n    // pathological behavior can emerge:\n    // - keepalive network request comes in, but is queued in Node internals waiting\n    //   for time on the event loop\n    // - CPU is starved/monopolized by other processes for longer than the time\n    //   configured via this.#stopInterval\n    // - When CPU becomes available and the event loop wakes up, this stop interval\n    //   check is run *before* the keepalive request is processed. The value of\n    //   this.#lastHeartbeat is now very stale, and erroneously triggers a shutdown\n    //   even though keepalive requests were about to be processed and update\n    //   this.#lastHeartbeat. Downtime ensues.\n    //\n    // To avoid this, we push the check out to a phase of the event loop *after*\n    // I/O events are processed, using setImmediate():\n    // https://nodejs.org/en/learn/asynchronous-work/event-loop-timers-and-nexttick#setimmediate-vs-settimeout\n    //\n    // This ensures we see a value for this.#lastHeartbeat that reflects\n    // any keepalive requests that came in during the current event loop turn.\n    this.#checkImmediateTimer = setImmediate(() => {\n      this.#checkImmediateTimer = undefined;\n      const timeSinceLastHeartbeat = Date.now() - this.#lastHeartbeat;\n      if (timeSinceLastHeartbeat >= this.#stopInterval) {\n        this.#lc.info?.(\n          `last heartbeat received ${\n            timeSinceLastHeartbeat / 1000\n          } seconds ago. draining.`,\n        );\n        process.kill(process.pid, GRACEFUL_SHUTDOWN[0]);\n      }\n    });\n  };\n\n  stop() {\n    clearTimeout(this.#checkIntervalTimer);\n    if (this.#checkImmediateTimer) {\n      clearImmediate(this.#checkImmediateTimer);\n    }\n  }\n}\n"],"mappings":";;;;;AA4BA,IAAa,oBAAoB,CAAC,WAAW,SAAS;AACtD,IAAa,oBAAoB,CAAC,WAAW,UAAU;AAYvD,IAAM,kCAAkC;;;;;AAMxC,IAAa,iBAAb,MAA4B;CAC1B;CACA,8BAAuB,IAAI,KAAiB;CAC5C,uBAAgB,IAAI,KAAiB;CACrC;CACA,SAAkB,KAAK,KAAK;CAC5B,SAAmC,EAAE;CAErC,gBAAgB,IAAI,aAAa,kBAAkB;CACnD,cAAc;CAEd,YAAY,IAAgB,MAAoB;AAC9C,QAAA,KAAW,GAAG,YAAY,aAAa,kBAAkB;AAKzD,OAAK,MAAM,UAAU,kBACnB,MAAK,GAAG,cAAc,MAAA,WAAiB,OAAO,CAAC;AAIjD,OAAK,GAAG,SAAQ,SACd,MAAA,KACE,MAAA,KACA,SAAS,IACL,YACA,SAAS,kCACP,YACA,UACP,CACF;AASD,OAAK,MAAM,UAAU,kBACnB,MAAK,GAAG,cACN,MAAA,KAAW,WAAW,YAAY,kCAAkC,GAAG,CACxE;AAGH,QAAA,YAAkB,SAAiB;AACjC,OAAI,mBAAmB,CACrB,QAAO,KAAK,KAAK,QAAQ,KAAK;AAEhC,WAAQ,KAAK,KAAK;;;CAItB,OAAO;AACL,SAAO,MAAA,aAAmB,SAAS;;CAGrC,MAAM,MAAc;AAClB,QAAA,GAAS,OAAO,qBAAqB,KAAK;AAC1C,QAAA,aAAmB,KAAK,MAAA,GAAS;AAC5B,QAAA,GAAS,OAAO,CAAC,cAAc,MAAA,SAAe,KAAK,CAAC;;CAG3D,YAAY,QAAgC;AAC1C,QAAA,GAAS,OAAO,qBAAqB,OAAO,GAAG;AAC/C,QAAA,aAAmB,KAAK,KAAK;AAC7B,MAAI,MAAA,WAAiB,KACnB,OAAA,KAAW,MAAA,YAAkB,OAAO;MAEpC,OAAA,KAAW,MAAA,KAAW,OAAO;;CAIjC,cAAc,MAAkB,MAAkB,MAAc;AAC9D,MAAI,SAAS,cACX,OAAA,WAAiB,IAAI,KAAK;AAE5B,QAAA,IAAU,IAAI,KAAK;EAEnB,IAAI,SAAS;AACb,OAAK,GAAG,UAAU,MAAM,WAAW;AACjC,YAAS;AACT,SAAA,OAAa,MAAM,QAAQ,MAAM,MAAM,MAAM,KAAK;IAClD;AAMF,OAAK,GAAG,UAAS,QACf,MAAA,GAAS,SAAS,UAAU,UAC1B,cAAc,KAAK,GAAG,KAAK,OAC3B,IACD,CACF;;CAGH,gCAAyB,IAAI,KAAqB;CAClD,UAAU;CAEV,UAAU,QAAgB,MAAkB,MAAsB;AAChE,OAAK,cAAc,QAAQ,MAAM,KAAK;EAEtC,MAAM,KAAK,EAAE,MAAA;AACb,QAAA,aAAmB,IAAI,IAAI,KAAK;EAChC,MAAM,EAAC,SAAS,YAAW,UAAU;AACrC,QAAA,MAAY,KAAK,QAAQ;AAEzB,SAAO,gBAAgB,eAAe;AACpC,SAAA,GAAS,QAAQ,GAAG,KAAK,UAAU,KAAK,KAAK,GAAG,MAAA,MAAY,MAAM;AAClE,SAAA,aAAmB,OAAO,GAAG;AAC7B,YAAS;IACT;AAEF,SAAO;;CAGT,eAAyB;AACvB,SAAO,CAAC,GAAG,MAAA,aAAmB,QAAQ,CAAC;;CAGzC,MAAM,kBAAkB;AACtB,QAAM,QAAQ,IAAI,MAAA,MAAY;;CAGhC,gBAAgB,KAAc,MAAc;AAE1C,QAAA,OAAa,IAAI,MAAM,KAAK,eAAe,MAAM,KAAA,EAAU;;CAG7D,QACE,MACA,KACA,KACA,MACA,MACA,QACA;AAEA,MAAI,QAAQ;AACV,SAAA,WAAiB,OAAO,OAAO;AAC/B,SAAA,IAAU,OAAO,OAAO;;EAG1B,MAAM,MAAM,QAAQ,OAAO,QAAQ;AAEnC,MAAI,SAAS,cAAc;AAMzB,OAAI,SAAS,MAAM,MAAA,eAAqB,KAAK,MAAA,WAAiB,OAAO,GACnE,QAAO;GAET,MAAM,MACJ,SAAS,KAAK,SAAS,kCACnB,SACA;AACN,SAAA,GAAS,OAAO,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAAI,OAAO,GAAG;AACzE,UAAO,MAAA,KAAW,KAAK;;EAGzB,MAAM,MACJ,SAAS,KAAK,SAAS,kCACnB,SACA,MAAA,aAAmB,KAAK,SAAA,KACtB,SACA;AACR,QAAA,GAAS,OACP,MACI,GAAG,KAAK,IAAI,IAAI,iBAAiB,IAAI,KACrC,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAC/C,OAAO,GACR;AAGD,MAAI,MAAA,WAAiB,SAAS,GAAG;AAC/B,SAAA,GAAS,OACP,MAAA,aACI,oCACE,KAAK,KAAK,GAAG,MAAA,WACd,QACD,iCACL;AACD,UAAO,MAAA,KAAW,EAAE;;AAGtB,MAAI,MAAA,eAAqB,EAGvB,QAAO,MAAA,KAAW,QAAQ,GAAG;;CAMjC,MAAM,SAA+B,QAAwB;AAC3D,OAAK,MAAM,UAAU,QACnB,KAAI;AACF,UAAO,KAAK,OAAO;WACZ,GAAG;AACV,SAAA,GAAS,QAAQ,EAAE;;;;;;;;;;;AAc3B,eAAsB,eACpB,IACA,QACA,GAAG,UACY;AACf,KAAI,SAAS,WAAW,EACtB;AAEF,MAAK,MAAM,UAAU,CAAC,GAAG,mBAAmB,GAAG,kBAAkB,CAC/D,QAAO,KAAK,cAAc;EACxB,MAAM,mBAAmB;AAEzB,WAAS,QAAQ,OAAM,QAAO;AAC5B,OAAI,iBAAiB,SAAS,OAAO,IAAI,IAAI,OAAO;AAClD,OAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,GAAG;AACnE,UAAM,IAAI,OAAO;;AAEnB,MAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,GAAG;AACnE,SAAM,IAAI,MAAM;IAChB;GACF;AAGJ,KAAI;EAEF,MAAM,MAAM,MAAM,QAAQ,KACxB,SAAS,KAAI,QAAO,IAAI,KAAK,CAAC,WAAW,IAAI,CAAC,CAC/C;AACD,KAAG,OAAO,GAAG,IAAI,YAAY,KAAK,IAAI,IAAI,GAAG,WAAW;UACjD,GAAG;AACV,KAAG,QAAQ,oBAAoB,EAAE;AACjC,QAAM;;;AAIV,eAAsB,UAAU,KAA0B;AACxD,KAAI;AACF,QAAM,KAAK;AAEX,UAAQ,KAAK,OAAO,IAAI,mBAAmB;AAC3C,UAAQ,KAAK,EAAE;UACR,GAAG;AAEV,UAAQ,MAAM,OAAO,IAAI,sBAAsB,EAAE;AACjD,UAAQ,KAAK,GAAG;;;AAIpB,IAAM,2BAA2B;;;;;;;;;;;;AAajC,IAAa,mBAAb,MAA8B;CAC5B;CAEA;CACA;CACA;CACA,iBAAiB;CAEjB,YAAY,IAAgB,eAAe,0BAA0B;AACnE,QAAA,KAAW;AACX,QAAA,eAAqB;;CAGvB,YAAY,YAAiC;AAC3C,QAAA,gBAAsB,KAAK,KAAK;AAChC,MAAI,MAAA,uBAA6B,KAAA,GAAW;AAC1C,SAAA,GAAS,OACP,iCACE,MAAA,eAAqB,IACtB,mBACD,WACD;AAGD,SAAA,qBAA2B,YACzB,MAAA,mBACA,MAAA,eAAqB,EACtB;;;CAIL,2BAA2B;AAsBzB,QAAA,sBAA4B,mBAAmB;AAC7C,SAAA,sBAA4B,KAAA;GAC5B,MAAM,yBAAyB,KAAK,KAAK,GAAG,MAAA;AAC5C,OAAI,0BAA0B,MAAA,cAAoB;AAChD,UAAA,GAAS,OACP,2BACE,yBAAyB,IAC1B,yBACF;AACD,YAAQ,KAAK,QAAQ,KAAK,kBAAkB,GAAG;;IAEjD;;CAGJ,OAAO;AACL,eAAa,MAAA,mBAAyB;AACtC,MAAI,MAAA,oBACF,gBAAe,MAAA,oBAA0B"}
+{"version":3,"file":"life-cycle.js","names":["#lc","#userFacing","#all","#exitImpl","#start","#ready","#startDrain","#kill","#exit","#runningState","#drainStart","#onExit","#initializing","#nextID","#stopInterval","#lastHeartbeat","#checkIntervalTimer","#checkStopInterval","#checkImmediateTimer"],"sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"sourcesContent":["import type {IncomingHttpHeaders} from 'node:http';\nimport {pid} from 'node:process';\nimport type {EventEmitter} from 'stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {\n  singleProcessMode,\n  type Subprocess,\n  type Worker,\n} from '../types/processes.ts';\nimport {RunningState} from './running-state.ts';\nimport type {SingletonService} from './service.ts';\n\n/**\n * * `user-facing` workers serve external requests and are the first to\n *   receive a `SIGTERM` or `SIGINT` signal for graceful shutdown.\n *\n * * `supporting` workers support `user-facing` workers and are sent\n *   the `SIGTERM` signal only after all `user-facing` workers have\n *   exited.\n *\n * For other kill signals, such as `SIGQUIT` and `SIGABRT`, all workers\n * are stopped without draining. `SIGQUIT` is used to represent an\n * intentional shutdown (for which draining is not beneficial), whereas\n * `SIGABRT` is used for unexpected process exits.\n */\nexport type WorkerType = 'user-facing' | 'supporting';\n\nexport const GRACEFUL_SHUTDOWN = ['SIGTERM', 'SIGINT'] as const;\nexport const FORCEFUL_SHUTDOWN = ['SIGQUIT', 'SIGABRT'] as const;\n\ntype GracefulShutdownSignal = (typeof GRACEFUL_SHUTDOWN)[number];\n\n// An internal error code used to indicate that a message has already been\n// logged at level ERRROR. When a process exits with this error code, the\n// parent process logs the exit at level WARN instead of ERROR.\nexport const UNHANDLED_EXCEPTION_ERROR_CODE = 13;\n\n// An internal error code used to indicate that the server should exit\n// without draining (e.g. due to a supporting worker get a signal to shut\n// down), but the exit is otherwise intentional.\nexport const INTENTIONAL_SHUTDOWN_ERROR_CODE = 14;\n\n/**\n * Handles readiness, termination signals, and coordination of graceful\n * shutdown.\n */\nexport class ProcessManager {\n  readonly #lc: LogContext;\n  readonly #userFacing = new Set<Subprocess>();\n  readonly #all = new Set<Subprocess>();\n  readonly #exitImpl: (code: number) => never;\n  readonly #start = Date.now();\n  readonly #ready: Promise<void>[] = [];\n\n  #runningState = new RunningState('process-manager');\n  #drainStart = 0;\n\n  constructor(lc: LogContext, proc: EventEmitter) {\n    this.#lc = lc.withContext('component', 'process-manager');\n\n    // Propagate `SIGTERM` and `SIGINT` to all user-facing workers,\n    // initiating a graceful shutdown. The parent process will\n    // exit once all user-facing workers have exited ...\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.on(signal, () => this.#startDrain(signal));\n    }\n\n    // ... which will result in sending `SIGTERM` to the remaining workers.\n    proc.on('exit', code =>\n      this.#kill(\n        this.#all,\n        code === 0\n          ? 'SIGTERM' // graceful, drained shutdown\n          : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n            ? 'SIGQUIT' // intentional abort without drain\n            : 'SIGABRT', // unintentional shutdown, alertable error\n      ),\n    );\n\n    // For other (catchable) kill signals, exit with a non-zero error code\n    // to send a `SIGQUIT` (intentional shutdown) or `SIGABRT` (unexpected\n    // shutdown) to all workers. For these signals, workers are stopped\n    // immediately without draining, since there is no merit to slowly draining\n    // when supporting workers have stopped.\n    //\n    // The logic for handling these signals is in `runUntilKilled()`.\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      proc.on(signal, () =>\n        this.#exit(signal === 'SIGQUIT' ? INTENTIONAL_SHUTDOWN_ERROR_CODE : -1),\n      );\n    }\n\n    this.#exitImpl = (code: number) => {\n      if (singleProcessMode()) {\n        return proc.emit('exit', code) as never; // For unit / integration tests.\n      }\n      process.exit(code);\n    };\n  }\n\n  done() {\n    return this.#runningState.stopped();\n  }\n\n  #exit(code: number) {\n    this.#lc.info?.('exiting with code', code);\n    this.#runningState.stop(this.#lc);\n    void this.#lc.flush().finally(() => this.#exitImpl(code));\n  }\n\n  #startDrain(signal: GracefulShutdownSignal) {\n    if (this.#all.size === 0) {\n      // Shutdown if a signal is received before any subprocesses are added.\n      this.#lc.info?.(`exiting on ${signal}`);\n      this.#exit(0);\n    }\n    this.#lc.info?.(`initiating drain (${signal})`);\n    this.#drainStart = Date.now();\n    if (this.#userFacing.size) {\n      this.#kill(this.#userFacing, signal);\n    } else {\n      this.#kill(this.#all, signal);\n    }\n  }\n\n  addSubprocess(proc: Subprocess, type: WorkerType, name: string) {\n    if (type === 'user-facing') {\n      this.#userFacing.add(proc);\n    }\n    this.#all.add(proc);\n\n    let isOpen = true;\n    proc.on('close', (code, signal) => {\n      isOpen = false;\n      this.#onExit(code, signal, null, type, name, proc);\n    });\n\n    // As per https://nodejs.org/api/child_process.html#event-error\n    // 'error' events can happen when sending a message to a child process\n    // fails. This is not really an error when the server is shutting down,\n    // so log any post-close errors at 'warn'.\n    proc.on('error', err =>\n      this.#lc[isOpen ? 'error' : 'warn']?.(\n        `error from ${name} ${proc.pid}`,\n        err,\n      ),\n    );\n  }\n\n  readonly #initializing = new Map<number, string>();\n  #nextID = 0;\n\n  addWorker(worker: Worker, type: WorkerType, name: string): Worker {\n    this.addSubprocess(worker, type, name);\n\n    const id = ++this.#nextID;\n    this.#initializing.set(id, name);\n    const {promise, resolve} = resolver();\n    this.#ready.push(promise);\n\n    worker.onceMessageType('ready', () => {\n      this.#lc.debug?.(`${name} ready (${Date.now() - this.#start} ms)`);\n      this.#initializing.delete(id);\n      resolve();\n    });\n\n    return worker;\n  }\n\n  initializing(): string[] {\n    return [...this.#initializing.values()];\n  }\n\n  async allWorkersReady() {\n    await Promise.all(this.#ready);\n  }\n\n  logErrorAndExit(err: unknown, name: string) {\n    // only accessible by the main (i.e. user-facing) process.\n    this.#onExit(-1, null, err, 'user-facing', name, undefined);\n  }\n\n  #onExit(\n    code: number,\n    sig: NodeJS.Signals | null,\n    err: unknown,\n    type: WorkerType,\n    name: string,\n    worker: Subprocess | undefined,\n  ) {\n    // Remove the worker from maps to avoid attempting to send more signals to it.\n    if (worker) {\n      this.#userFacing.delete(worker);\n      this.#all.delete(worker);\n    }\n\n    const pid = worker?.pid ?? process.pid;\n\n    if (type === 'supporting') {\n      // Supporting workers like the replication-manager shut down without a\n      // drain signal when receiving protocol-specific instructions (like auto\n      // reset). In this case, a special error code is used to signal that the\n      // server should be shut down without draining, but it is otherwise not\n      // considered an unexpected/alertable error.\n      if (code === 0 && (this.#drainStart === 0 || this.#userFacing.size > 0)) {\n        code = INTENTIONAL_SHUTDOWN_ERROR_CODE;\n      }\n      const log =\n        code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n          ? 'info'\n          : 'warn';\n      this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n      return this.#exit(code);\n    }\n\n    const log =\n      code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n        ? 'info'\n        : this.#drainStart > 0 || code === UNHANDLED_EXCEPTION_ERROR_CODE\n          ? 'warn'\n          : 'error';\n    this.#lc[log]?.(\n      sig\n        ? `${name} (${pid}) killed with (${sig})`\n        : `${name} (${pid}) exited with code (${code})`,\n      err ?? '',\n    );\n\n    // user-facing workers exited or finished draining.\n    if (this.#userFacing.size === 0) {\n      this.#lc.info?.(\n        this.#drainStart\n          ? `all user-facing workers drained (${\n              Date.now() - this.#drainStart\n            } ms)`\n          : `all user-facing workers exited`,\n      );\n      return this.#exit(0);\n    }\n\n    if (this.#drainStart === 0) {\n      // If a user-facing worker exits without receiving a drain signal,\n      // shutdown the server.\n      return this.#exit(code || -1);\n    }\n\n    return undefined;\n  }\n\n  #kill(workers: Iterable<Subprocess>, signal: NodeJS.Signals) {\n    for (const worker of workers) {\n      try {\n        worker.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\n/**\n * Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with\n * an optional {@link SingletonService.drain drain()}, or stopping them\n * without draining for `SIGQUIT`.\n *\n * @returns a Promise that resolves/rejects when any of the services stops/throws.\n */\n\nexport async function runUntilKilled(\n  lc: LogContext,\n  parent: EventEmitter,\n  ...services: SingletonService[]\n): Promise<void> {\n  if (services.length === 0) {\n    return;\n  }\n  for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) {\n    parent.once(signal, () => {\n      const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN as readonly NodeJS.Signals[];\n\n      services.forEach(async svc => {\n        if (GRACEFUL_SIGNALS.includes(signal) && svc.drain) {\n          lc.info?.(`draining ${svc.constructor.name} ${svc.id} (${signal})`);\n          await svc.drain();\n        }\n        lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${signal})`);\n        await svc.stop();\n      });\n    });\n  }\n\n  try {\n    // Run all services and resolve when any of them stops.\n    const svc = await Promise.race(\n      services.map(svc => svc.run().then(() => svc)),\n    );\n    lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);\n  } catch (e) {\n    lc.error?.(`exiting on error`, e);\n    throw e;\n  }\n}\n\nexport async function exitAfter(run: () => Promise<void>) {\n  try {\n    await run();\n    // oxlint-disable-next-line no-console\n    console.info(`pid ${pid} exiting normally`);\n    process.exit(0);\n  } catch (e) {\n    // oxlint-disable-next-line no-console\n    console.error(`pid ${pid} exiting with error`, e);\n    process.exit(-1);\n  }\n}\n\nconst DEFAULT_STOP_INTERVAL_MS = 20_000;\n\n/**\n * The HeartbeatMonitor monitors the cadence heartbeats (e.g. \"/keepalive\"\n * health checks made to HttpServices) that signal that the server\n * should continue processing requests. When a configurable `stopInterval`\n * elapses without receiving these heartbeats, the monitor initiates a\n * graceful shutdown of the server. This works with common load balancing\n * frameworks such as AWS Elastic Load Balancing.\n *\n * The HeartbeatMonitor is **opt-in** in that it only kicks in after it\n * starts receiving keepalives.\n */\nexport class HeartbeatMonitor {\n  readonly #stopInterval: number;\n\n  #lc: LogContext;\n  #checkIntervalTimer: NodeJS.Timeout | undefined;\n  #checkImmediateTimer: NodeJS.Immediate | undefined;\n  #lastHeartbeat = 0;\n\n  constructor(lc: LogContext, stopInterval = DEFAULT_STOP_INTERVAL_MS) {\n    this.#lc = lc;\n    this.#stopInterval = stopInterval;\n  }\n\n  onHeartbeat(reqHeaders: IncomingHttpHeaders) {\n    this.#lastHeartbeat = Date.now();\n    if (this.#checkIntervalTimer === undefined) {\n      this.#lc.info?.(\n        `starting heartbeat monitor at ${\n          this.#stopInterval / 1000\n        } second interval`,\n        reqHeaders,\n      );\n      // e.g. check every 5 seconds to see if it's been over 20 seconds\n      //      since the last heartbeat.\n      this.#checkIntervalTimer = setInterval(\n        this.#checkStopInterval,\n        this.#stopInterval / 4,\n      );\n    }\n  }\n\n  #checkStopInterval = () => {\n    // In the Node.js event loop, timers like setInterval and setTimeout\n    // run *before* I/O events coming from network sockets or file reads/writes.\n    // When this process gets starved of CPU resources for long periods of time,\n    // for example when other processes are monopolizing all available cores,\n    // pathological behavior can emerge:\n    // - keepalive network request comes in, but is queued in Node internals waiting\n    //   for time on the event loop\n    // - CPU is starved/monopolized by other processes for longer than the time\n    //   configured via this.#stopInterval\n    // - When CPU becomes available and the event loop wakes up, this stop interval\n    //   check is run *before* the keepalive request is processed. The value of\n    //   this.#lastHeartbeat is now very stale, and erroneously triggers a shutdown\n    //   even though keepalive requests were about to be processed and update\n    //   this.#lastHeartbeat. Downtime ensues.\n    //\n    // To avoid this, we push the check out to a phase of the event loop *after*\n    // I/O events are processed, using setImmediate():\n    // https://nodejs.org/en/learn/asynchronous-work/event-loop-timers-and-nexttick#setimmediate-vs-settimeout\n    //\n    // This ensures we see a value for this.#lastHeartbeat that reflects\n    // any keepalive requests that came in during the current event loop turn.\n    this.#checkImmediateTimer = setImmediate(() => {\n      this.#checkImmediateTimer = undefined;\n      const timeSinceLastHeartbeat = Date.now() - this.#lastHeartbeat;\n      if (timeSinceLastHeartbeat >= this.#stopInterval) {\n        this.#lc.info?.(\n          `last heartbeat received ${\n            timeSinceLastHeartbeat / 1000\n          } seconds ago. draining.`,\n        );\n        process.kill(process.pid, GRACEFUL_SHUTDOWN[0]);\n      }\n    });\n  };\n\n  stop() {\n    clearTimeout(this.#checkIntervalTimer);\n    if (this.#checkImmediateTimer) {\n      clearImmediate(this.#checkImmediateTimer);\n    }\n  }\n}\n"],"mappings":";;;;;AA4BA,IAAa,oBAAoB,CAAC,WAAW,SAAS;AACtD,IAAa,oBAAoB,CAAC,WAAW,UAAU;;;;;AAkBvD,IAAa,iBAAb,MAA4B;CAC1B;CACA,8BAAuB,IAAI,KAAiB;CAC5C,uBAAgB,IAAI,KAAiB;CACrC;CACA,SAAkB,KAAK,KAAK;CAC5B,SAAmC,EAAE;CAErC,gBAAgB,IAAI,aAAa,kBAAkB;CACnD,cAAc;CAEd,YAAY,IAAgB,MAAoB;AAC9C,QAAA,KAAW,GAAG,YAAY,aAAa,kBAAkB;AAKzD,OAAK,MAAM,UAAU,kBACnB,MAAK,GAAG,cAAc,MAAA,WAAiB,OAAO,CAAC;AAIjD,OAAK,GAAG,SAAQ,SACd,MAAA,KACE,MAAA,KACA,SAAS,IACL,YACA,SAAA,KACE,YACA,UACP,CACF;AASD,OAAK,MAAM,UAAU,kBACnB,MAAK,GAAG,cACN,MAAA,KAAW,WAAW,YAAA,KAA8C,GAAG,CACxE;AAGH,QAAA,YAAkB,SAAiB;AACjC,OAAI,mBAAmB,CACrB,QAAO,KAAK,KAAK,QAAQ,KAAK;AAEhC,WAAQ,KAAK,KAAK;;;CAItB,OAAO;AACL,SAAO,MAAA,aAAmB,SAAS;;CAGrC,MAAM,MAAc;AAClB,QAAA,GAAS,OAAO,qBAAqB,KAAK;AAC1C,QAAA,aAAmB,KAAK,MAAA,GAAS;AAC5B,QAAA,GAAS,OAAO,CAAC,cAAc,MAAA,SAAe,KAAK,CAAC;;CAG3D,YAAY,QAAgC;AAC1C,MAAI,MAAA,IAAU,SAAS,GAAG;AAExB,SAAA,GAAS,OAAO,cAAc,SAAS;AACvC,SAAA,KAAW,EAAE;;AAEf,QAAA,GAAS,OAAO,qBAAqB,OAAO,GAAG;AAC/C,QAAA,aAAmB,KAAK,KAAK;AAC7B,MAAI,MAAA,WAAiB,KACnB,OAAA,KAAW,MAAA,YAAkB,OAAO;MAEpC,OAAA,KAAW,MAAA,KAAW,OAAO;;CAIjC,cAAc,MAAkB,MAAkB,MAAc;AAC9D,MAAI,SAAS,cACX,OAAA,WAAiB,IAAI,KAAK;AAE5B,QAAA,IAAU,IAAI,KAAK;EAEnB,IAAI,SAAS;AACb,OAAK,GAAG,UAAU,MAAM,WAAW;AACjC,YAAS;AACT,SAAA,OAAa,MAAM,QAAQ,MAAM,MAAM,MAAM,KAAK;IAClD;AAMF,OAAK,GAAG,UAAS,QACf,MAAA,GAAS,SAAS,UAAU,UAC1B,cAAc,KAAK,GAAG,KAAK,OAC3B,IACD,CACF;;CAGH,gCAAyB,IAAI,KAAqB;CAClD,UAAU;CAEV,UAAU,QAAgB,MAAkB,MAAsB;AAChE,OAAK,cAAc,QAAQ,MAAM,KAAK;EAEtC,MAAM,KAAK,EAAE,MAAA;AACb,QAAA,aAAmB,IAAI,IAAI,KAAK;EAChC,MAAM,EAAC,SAAS,YAAW,UAAU;AACrC,QAAA,MAAY,KAAK,QAAQ;AAEzB,SAAO,gBAAgB,eAAe;AACpC,SAAA,GAAS,QAAQ,GAAG,KAAK,UAAU,KAAK,KAAK,GAAG,MAAA,MAAY,MAAM;AAClE,SAAA,aAAmB,OAAO,GAAG;AAC7B,YAAS;IACT;AAEF,SAAO;;CAGT,eAAyB;AACvB,SAAO,CAAC,GAAG,MAAA,aAAmB,QAAQ,CAAC;;CAGzC,MAAM,kBAAkB;AACtB,QAAM,QAAQ,IAAI,MAAA,MAAY;;CAGhC,gBAAgB,KAAc,MAAc;AAE1C,QAAA,OAAa,IAAI,MAAM,KAAK,eAAe,MAAM,KAAA,EAAU;;CAG7D,QACE,MACA,KACA,KACA,MACA,MACA,QACA;AAEA,MAAI,QAAQ;AACV,SAAA,WAAiB,OAAO,OAAO;AAC/B,SAAA,IAAU,OAAO,OAAO;;EAG1B,MAAM,MAAM,QAAQ,OAAO,QAAQ;AAEnC,MAAI,SAAS,cAAc;AAMzB,OAAI,SAAS,MAAM,MAAA,eAAqB,KAAK,MAAA,WAAiB,OAAO,GACnE,QAAA;GAEF,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA;AACN,SAAA,GAAS,OAAO,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAAI,OAAO,GAAG;AACzE,UAAO,MAAA,KAAW,KAAK;;EAGzB,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA,MAAA,aAAmB,KAAK,SAAA,KACtB,SACA;AACR,QAAA,GAAS,OACP,MACI,GAAG,KAAK,IAAI,IAAI,iBAAiB,IAAI,KACrC,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAC/C,OAAO,GACR;AAGD,MAAI,MAAA,WAAiB,SAAS,GAAG;AAC/B,SAAA,GAAS,OACP,MAAA,aACI,oCACE,KAAK,KAAK,GAAG,MAAA,WACd,QACD,iCACL;AACD,UAAO,MAAA,KAAW,EAAE;;AAGtB,MAAI,MAAA,eAAqB,EAGvB,QAAO,MAAA,KAAW,QAAQ,GAAG;;CAMjC,MAAM,SAA+B,QAAwB;AAC3D,OAAK,MAAM,UAAU,QACnB,KAAI;AACF,UAAO,KAAK,OAAO;WACZ,GAAG;AACV,SAAA,GAAS,QAAQ,EAAE;;;;;;;;;;;AAc3B,eAAsB,eACpB,IACA,QACA,GAAG,UACY;AACf,KAAI,SAAS,WAAW,EACtB;AAEF,MAAK,MAAM,UAAU,CAAC,GAAG,mBAAmB,GAAG,kBAAkB,CAC/D,QAAO,KAAK,cAAc;EACxB,MAAM,mBAAmB;AAEzB,WAAS,QAAQ,OAAM,QAAO;AAC5B,OAAI,iBAAiB,SAAS,OAAO,IAAI,IAAI,OAAO;AAClD,OAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,GAAG;AACnE,UAAM,IAAI,OAAO;;AAEnB,MAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,GAAG;AACnE,SAAM,IAAI,MAAM;IAChB;GACF;AAGJ,KAAI;EAEF,MAAM,MAAM,MAAM,QAAQ,KACxB,SAAS,KAAI,QAAO,IAAI,KAAK,CAAC,WAAW,IAAI,CAAC,CAC/C;AACD,KAAG,OAAO,GAAG,IAAI,YAAY,KAAK,IAAI,IAAI,GAAG,WAAW;UACjD,GAAG;AACV,KAAG,QAAQ,oBAAoB,EAAE;AACjC,QAAM;;;AAIV,eAAsB,UAAU,KAA0B;AACxD,KAAI;AACF,QAAM,KAAK;AAEX,UAAQ,KAAK,OAAO,IAAI,mBAAmB;AAC3C,UAAQ,KAAK,EAAE;UACR,GAAG;AAEV,UAAQ,MAAM,OAAO,IAAI,sBAAsB,EAAE;AACjD,UAAQ,KAAK,GAAG;;;AAIpB,IAAM,2BAA2B;;;;;;;;;;;;AAajC,IAAa,mBAAb,MAA8B;CAC5B;CAEA;CACA;CACA;CACA,iBAAiB;CAEjB,YAAY,IAAgB,eAAe,0BAA0B;AACnE,QAAA,KAAW;AACX,QAAA,eAAqB;;CAGvB,YAAY,YAAiC;AAC3C,QAAA,gBAAsB,KAAK,KAAK;AAChC,MAAI,MAAA,uBAA6B,KAAA,GAAW;AAC1C,SAAA,GAAS,OACP,iCACE,MAAA,eAAqB,IACtB,mBACD,WACD;AAGD,SAAA,qBAA2B,YACzB,MAAA,mBACA,MAAA,eAAqB,EACtB;;;CAIL,2BAA2B;AAsBzB,QAAA,sBAA4B,mBAAmB;AAC7C,SAAA,sBAA4B,KAAA;GAC5B,MAAM,yBAAyB,KAAK,KAAK,GAAG,MAAA;AAC5C,OAAI,0BAA0B,MAAA,cAAoB;AAChD,UAAA,GAAS,OACP,2BACE,yBAAyB,IAC1B,yBACF;AACD,YAAQ,KAAK,QAAQ,KAAK,kBAAkB,GAAG;;IAEjD;;CAGJ,OAAO;AACL,eAAa,MAAA,mBAAyB;AACtC,MAAI,MAAA,oBACF,gBAAe,MAAA,oBAA0B"}

package/out/zero-cache/src/services/litestream/commands.d.ts

@@ -1,10 +1,21 @@
-import type { LogContext } from '@rocicorp/logger';
 import type { ChildProcess } from 'node:child_process';
+import type { LogContext } from '@rocicorp/logger';
 import type { ZeroConfig } from '../../config/zero-config.ts';
+type ReplicaConstraints = {
+    replicaVersion: string;
+    minWatermark: string;
+};
+export declare class BackupNotFoundException extends Error {
+    static readonly name = "BackupNotFoundException";
+    constructor(backupURL: string | undefined);
+}
 /**
- * @returns The time at which the last restore started
- *          (i.e. not counting failed attempts).
+ * @param replicaConstraints The constraints of the restored backup when
+ *        restoring for the change-streamer (replication-manager). For the
+ *        view-syncer, this should be unspecified so that the constraints are
+ *        retrieved from the replication-manager via the snapshot protocol.
  */
-export declare function restoreReplica(lc: LogContext, config: ZeroConfig): Promise<Date>;
+export declare function restoreReplica(lc: LogContext, config: ZeroConfig, replicaConstraints: ReplicaConstraints | null): Promise<void>;
 export declare function startReplicaBackupProcess(lc: LogContext, config: ZeroConfig): ChildProcess;
+export {};
 //# sourceMappingURL=commands.d.ts.map

package/out/zero-cache/src/services/litestream/commands.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"commands.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAW,MAAM,kBAAkB,CAAC;AAE3D,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAOrD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAiB5D;;;GAGG;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,GACjB,OAAO,CAAC,IAAI,CAAC,CAwBf;AA2KD,wBAAgB,yBAAyB,CACvC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,GACjB,YAAY,CAQd"}
+{"version":3,"file":"commands.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAGrD,OAAO,KAAK,EAAC,UAAU,EAAW,MAAM,kBAAkB,CAAC;AAM3D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAiB5D,KAAK,kBAAkB,GAAG;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,qBAAa,uBAAwB,SAAQ,KAAK;IAChD,MAAM,CAAC,QAAQ,CAAC,IAAI,6BAA6B;gBAErC,SAAS,EAAE,MAAM,GAAG,SAAS;CAG1C;AAED;;;;;GAKG;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,EAClB,kBAAkB,EAAE,kBAAkB,GAAG,IAAI,iBAoB9C;AAgKD,wBAAgB,yBAAyB,CACvC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,GACjB,YAAY,CAQd"}

package/out/zero-cache/src/services/litestream/commands.js

@@ -1,35 +1,38 @@
 import { sleep } from "../../../../shared/src/sleep.js";
 import { must } from "../../../../shared/src/must.js";
 import { Database } from "../../../../zqlite/src/db.js";
-import { assertNormalized } from "../../config/normalize.js";
 import { getShardConfig } from "../../types/shards.js";
-import { StatementRunner } from "../../db/statements.js";
+import { assertNormalized } from "../../config/normalize.js";
 import { getSubscriptionState } from "../replicator/schema/replication-state.js";
+import { StatementRunner } from "../../db/statements.js";
 import { deleteLiteDB } from "../../db/delete-lite-db.js";
 import { ChangeStreamerHttpClient } from "../change-streamer/change-streamer-http.js";
 import { resolver } from "@rocicorp/resolver";
-import { spawn } from "node:child_process";
 import { existsSync } from "node:fs";
+import { spawn } from "node:child_process";
 //#region ../zero-cache/src/services/litestream/commands.ts
 var MAX_RETRIES = 60;
 var RETRY_INTERVAL_MS = 3e3;
+var BackupNotFoundException = class extends Error {
+	static name = "BackupNotFoundException";
+	constructor(backupURL) {
+		super(`backup not found at ${backupURL}`);
+	}
+};
 /**
-* @returns The time at which the last restore started
-*          (i.e. not counting failed attempts).
+* @param replicaConstraints The constraints of the restored backup when
+*        restoring for the change-streamer (replication-manager). For the
+*        view-syncer, this should be unspecified so that the constraints are
+*        retrieved from the replication-manager via the snapshot protocol.
 */
-async function restoreReplica(lc, config) {
-	const { changeStreamer } = config;
+async function restoreReplica(lc, config, replicaConstraints) {
 	for (let i = 0; i < MAX_RETRIES; i++) {
 		if (i > 0) {
 			lc.info?.(`replica not found. retrying in ${RETRY_INTERVAL_MS / 1e3} seconds`);
 			await sleep(RETRY_INTERVAL_MS);
 		}
-		const start = /* @__PURE__ */ new Date();
-		if (await tryRestore(lc, config)) return start;
-		if (changeStreamer.mode === "dedicated" && changeStreamer.uri === void 0) {
-			lc.info?.("no litestream backup found");
-			return start;
-		}
+		if (await tryRestore(lc, config, replicaConstraints)) return;
+		if (replicaConstraints) throw new BackupNotFoundException(config.litestream.backupURL);
 	}
 	throw new Error(`max attempts exceeded restoring replica`);
 }
@@ -56,15 +59,13 @@ function getLitestream(mode, config, logLevelOverride, backupURLOverride) {
 		}
 	};
 }
-async function tryRestore(lc, config) {
-	const { changeStreamer } = config;
-	const isViewSyncer = changeStreamer.mode === "discover" || changeStreamer.uri !== void 0;
-	const firstMessage = reserveAndGetSnapshotStatus(lc, config, isViewSyncer);
+async function tryRestore(lc, config, replicaConstraints) {
 	let snapshotStatus;
-	if (isViewSyncer) {
-		snapshotStatus = await firstMessage;
+	if (!replicaConstraints) {
+		snapshotStatus = await reserveAndGetSnapshotStatus(lc, config);
 		lc.info?.(`restoring backup from ${snapshotStatus.backupURL}`);
-	} else firstMessage.catch((e) => lc.debug?.(e));
+		replicaConstraints = snapshotStatus;
+	}
 	const { litestream, env } = getLitestream("restore", config, "debug", snapshotStatus?.backupURL);
 	const { restoreParallelism: parallelism } = config.litestream;
 	const proc = spawn(litestream, [
@@ -88,26 +89,26 @@ async function tryRestore(lc, config) {
 	});
 	await promise;
 	if (!existsSync(config.replica.file)) return false;
-	if (snapshotStatus && !replicaIsValid(lc, config.replica.file, snapshotStatus)) {
+	if (!replicaIsValid(lc, config.replica.file, replicaConstraints)) {
 		lc.info?.(`Deleting local replica and retrying restore`);
 		deleteLiteDB(config.replica.file);
 		return false;
 	}
 	return true;
 }
-function replicaIsValid(lc, replica, snapshot) {
+function replicaIsValid(lc, replica, constraints) {
 	const db = new Database(lc, replica);
 	try {
 		const { replicaVersion, watermark } = getSubscriptionState(new StatementRunner(db));
-		if (replicaVersion !== snapshot.replicaVersion) {
-			lc.warn?.(`Local replica version ${replicaVersion} does not match change-streamer replicaVersion ${snapshot.replicaVersion}`, snapshot);
+		if (replicaVersion !== constraints.replicaVersion) {
+			lc.warn?.(`Local replica version ${replicaVersion} does not match expected replicaVersion ${constraints.replicaVersion}`, constraints);
 			return false;
 		}
-		if (watermark < snapshot.minWatermark) {
-			lc.warn?.(`Local replica watermark ${watermark} is earlier than change-streamer minWatermark ${snapshot.minWatermark}`);
+		if (watermark < constraints.minWatermark) {
+			lc.warn?.(`Local replica watermark ${watermark} is earlier than minWatermark ${constraints.minWatermark}`);
 			return false;
 		}
-		lc.info?.(`Local replica at version ${replicaVersion} and watermark ${watermark} is compatible with change-streamer`, snapshot);
+		lc.info?.(`Local replica at version ${replicaVersion} and watermark ${watermark} is compatible`, constraints);
 		return true;
 	} catch (e) {
 		lc.error?.("Error while validating restored replica", e);
@@ -125,14 +126,14 @@ function startReplicaBackupProcess(lc, config) {
 		windowsHide: true
 	});
 }
-function reserveAndGetSnapshotStatus(lc, config, isViewSyncer) {
+function reserveAndGetSnapshotStatus(lc, config) {
 	const { promise: status, resolve, reject } = resolver();
 	(async function() {
 		const abort = new AbortController();
 		process.on("SIGINT", () => abort.abort());
 		process.on("SIGTERM", () => abort.abort());
 		for (let i = 0;; i++) {
-			let err = "";
+			let err;
 			try {
 				let resolved = false;
 				const stream = await reserveSnapshot(lc, config);
@@ -144,7 +145,6 @@ function reserveAndGetSnapshotStatus(lc, config, isViewSyncer) {
 			} catch (e) {
 				err = e;
 			}
-			if (!isViewSyncer) return reject(err);
 			lc.warn?.(`Unable to reserve snapshot (attempt ${i + 1}). Retrying in 5 seconds.`, String(err));
 			try {
 				await sleep(5e3, abort.signal);
@@ -161,6 +161,6 @@ function reserveSnapshot(lc, config) {
 	return new ChangeStreamerHttpClient(lc, getShardConfig(config), change.db, changeStreamer.uri).reserveSnapshot(taskID);
 }
 //#endregion
-export { restoreReplica, startReplicaBackupProcess };
+export { BackupNotFoundException, restoreReplica, startReplicaBackupProcess };
 
 //# sourceMappingURL=commands.js.map

package/out/zero-cache/src/services/litestream/commands.js.map

@@ -1 +1 @@
-{"version":3,"file":"commands.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync} from 'node:fs';\nimport {must} from '../../../../shared/src/must.ts';\nimport {sleep} from '../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {assertNormalized} from '../../config/normalize.ts';\nimport type {ZeroConfig} from '../../config/zero-config.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getShardConfig} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {ChangeStreamerHttpClient} from '../change-streamer/change-streamer-http.ts';\nimport type {\n  SnapshotMessage,\n  SnapshotStatus,\n} from '../change-streamer/snapshot.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\n\n// Retry for up to 3 minutes (60 times with 3 second delay).\n// Beyond that, let the container runner restart the task.\nconst MAX_RETRIES = 60;\nconst RETRY_INTERVAL_MS = 3000;\n\n/**\n * @returns The time at which the last restore started\n *          (i.e. not counting failed attempts).\n */\nexport async function restoreReplica(\n  lc: LogContext,\n  config: ZeroConfig,\n): Promise<Date> {\n  const {changeStreamer} = config;\n\n  for (let i = 0; i < MAX_RETRIES; i++) {\n    if (i > 0) {\n      lc.info?.(\n        `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n      );\n      await sleep(RETRY_INTERVAL_MS);\n    }\n    const start = new Date();\n    const restored = await tryRestore(lc, config);\n    if (restored) {\n      return start;\n    }\n    if (\n      changeStreamer.mode === 'dedicated' &&\n      changeStreamer.uri === undefined\n    ) {\n      lc.info?.('no litestream backup found');\n      return start;\n    }\n  }\n  throw new Error(`max attempts exceeded restoring replica`);\n}\n\nfunction getLitestream(\n  mode: 'restore' | 'replicate',\n  config: ZeroConfig,\n  logLevelOverride?: LogLevel,\n  backupURLOverride?: string,\n): {\n  litestream: string;\n  env: NodeJS.ProcessEnv;\n} {\n  const {\n    executable,\n    executableV5,\n    restoreUsingV5,\n    backupURL,\n    logLevel,\n    configPath,\n    endpoint,\n    port = config.port + 2,\n    checkpointThresholdMB,\n    minCheckpointPageCount = checkpointThresholdMB * 250, // SQLite page size is 4KB\n    maxCheckpointPageCount = minCheckpointPageCount * 10,\n    incrementalBackupIntervalMinutes,\n    snapshotBackupIntervalHours,\n    multipartConcurrency,\n    multipartSize,\n  } = config.litestream;\n\n  // Set the snapshot interval to something smaller than x hours so that\n  // the hourly check triggers on the hour, rather than the hour after.\n  const snapshotBackupIntervalMinutes = snapshotBackupIntervalHours * 60 - 5;\n\n  const litestream =\n    // The v0.5.8+ litestream executable can restore from either the new LTX\n    // format or the legacy WAL format, allowing forwards-compatibility /\n    // rollback safety with zero-cache versions that backup to LTX.\n    (mode === 'restore' && restoreUsingV5 ? executableV5 : executable) ??\n    must(executable, `Missing --litestream-executable`);\n  return {\n    litestream,\n    env: {\n      ...process.env,\n      ['ZERO_REPLICA_FILE']: config.replica.file,\n      ['ZERO_LITESTREAM_BACKUP_URL']: must(backupURLOverride ?? backupURL),\n      ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n        minCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n        maxCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n        incrementalBackupIntervalMinutes,\n      ),\n      ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n      ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_MINUTES']: String(\n        snapshotBackupIntervalMinutes,\n      ),\n      ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n      ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n      ['ZERO_LOG_FORMAT']: config.log.format,\n      ['LITESTREAM_CONFIG']: configPath,\n      ['LITESTREAM_PORT']: String(port),\n      ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n    },\n  };\n}\n\nasync function tryRestore(lc: LogContext, config: ZeroConfig) {\n  const {changeStreamer} = config;\n\n  const isViewSyncer =\n    changeStreamer.mode === 'discover' || changeStreamer.uri !== undefined;\n\n  // Fire off a snapshot reservation to the current replication-manager\n  // (if there is one).\n  const firstMessage = reserveAndGetSnapshotStatus(lc, config, isViewSyncer);\n  let snapshotStatus: SnapshotStatus | undefined;\n  if (isViewSyncer) {\n    // The return value is required by view-syncers ...\n    snapshotStatus = await firstMessage;\n    lc.info?.(`restoring backup from ${snapshotStatus.backupURL}`);\n  } else {\n    // but it is also useful to pause change-log cleanup when a new\n    // replication-manager is starting up. In this case, the request is\n    // best-effort. In particular, there may not be a previous\n    // replication-manager running at all.\n    void firstMessage.catch(e => lc.debug?.(e));\n  }\n\n  const {litestream, env} = getLitestream(\n    'restore',\n    config,\n    'debug', // Include all output from `litestream restore`, as it's minimal.\n    snapshotStatus?.backupURL,\n  );\n  const {restoreParallelism: parallelism} = config.litestream;\n  const proc = spawn(\n    litestream,\n    [\n      'restore',\n      '-if-db-not-exists',\n      '-if-replica-exists',\n      '-parallelism',\n      String(parallelism),\n      config.replica.file,\n    ],\n    {env, stdio: 'inherit', windowsHide: true},\n  );\n  const {promise, resolve, reject} = resolver();\n  proc.on('error', reject);\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      reject(`litestream killed with ${signal}`);\n    } else if (code !== 0) {\n      reject(`litestream exited with code ${code}`);\n    } else {\n      resolve();\n    }\n  });\n  await promise;\n  if (!existsSync(config.replica.file)) {\n    return false;\n  }\n  if (\n    snapshotStatus &&\n    !replicaIsValid(lc, config.replica.file, snapshotStatus)\n  ) {\n    lc.info?.(`Deleting local replica and retrying restore`);\n    deleteLiteDB(config.replica.file);\n    return false;\n  }\n  return true;\n}\n\nfunction replicaIsValid(\n  lc: LogContext,\n  replica: string,\n  snapshot: SnapshotStatus,\n) {\n  const db = new Database(lc, replica);\n  try {\n    const {replicaVersion, watermark} = getSubscriptionState(\n      new StatementRunner(db),\n    );\n    if (replicaVersion !== snapshot.replicaVersion) {\n      lc.warn?.(\n        `Local replica version ${replicaVersion} does not match change-streamer replicaVersion ${snapshot.replicaVersion}`,\n        snapshot,\n      );\n      return false;\n    }\n    if (watermark < snapshot.minWatermark) {\n      lc.warn?.(\n        `Local replica watermark ${watermark} is earlier than change-streamer minWatermark ${snapshot.minWatermark}`,\n      );\n      return false;\n    }\n    lc.info?.(\n      `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible with change-streamer`,\n      snapshot,\n    );\n    return true;\n  } catch (e) {\n    lc.error?.('Error while validating restored replica', e);\n    return false;\n  } finally {\n    db.close();\n  }\n}\n\nexport function startReplicaBackupProcess(\n  lc: LogContext,\n  config: ZeroConfig,\n): ChildProcess {\n  const {litestream, env} = getLitestream('replicate', config);\n  lc.info?.(`starting litestream backup to ${config.litestream.backupURL}`);\n  return spawn(litestream, ['replicate'], {\n    env,\n    stdio: 'inherit',\n    windowsHide: true,\n  });\n}\n\nfunction reserveAndGetSnapshotStatus(\n  lc: LogContext,\n  config: ZeroConfig,\n  isViewSyncer: boolean,\n): Promise<SnapshotStatus> {\n  const {promise: status, resolve, reject} = resolver<SnapshotStatus>();\n\n  void (async function () {\n    const abort = new AbortController();\n    process.on('SIGINT', () => abort.abort());\n    process.on('SIGTERM', () => abort.abort());\n\n    for (let i = 0; ; i++) {\n      let err: unknown | string = '';\n      try {\n        let resolved = false;\n        const stream = await reserveSnapshot(lc, config);\n        for await (const msg of stream) {\n          // Capture the value of the status message that the change-streamer\n          // (i.e. BackupMonitor) returns, and hold the connection open to\n          // \"reserve\" the snapshot and prevent change log cleanup.\n          resolve(msg[1]);\n          resolved = true;\n        }\n        // The change-streamer itself closes the connection when the\n        // subscription is started (or the reservation retried).\n        if (resolved) {\n          break;\n        }\n      } catch (e) {\n        err = e;\n      }\n      if (!isViewSyncer) {\n        return reject(err);\n      }\n      // Retry in the view-syncer since it cannot proceed until it connects\n      // to a (compatible) replication-manager. In particular, a\n      // replication-manager that does not support the view-syncer's\n      // change-streamer protocol will close the stream with an error; this\n      // retry logic essentially delays the startup of a view-syncer until\n      // a compatible replication-manager has been rolled out, allowing\n      // replication-manager and view-syncer services to be updated in\n      // parallel.\n      lc.warn?.(\n        `Unable to reserve snapshot (attempt ${i + 1}). Retrying in 5 seconds.`,\n        String(err),\n      );\n      try {\n        await sleep(5000, abort.signal);\n      } catch (e) {\n        return reject(e);\n      }\n    }\n  })();\n\n  return status;\n}\n\nfunction reserveSnapshot(\n  lc: LogContext,\n  config: ZeroConfig,\n): Promise<Source<SnapshotMessage>> {\n  assertNormalized(config);\n  const {taskID, change, changeStreamer} = config;\n  const shardID = getShardConfig(config);\n\n  const changeStreamerClient = new ChangeStreamerHttpClient(\n    lc,\n    shardID,\n    change.db,\n    changeStreamer.uri,\n  );\n\n  return changeStreamerClient.reserveSnapshot(taskID);\n}\n"],"mappings":";;;;;;;;;;;;;AAuBA,IAAM,cAAc;AACpB,IAAM,oBAAoB;;;;;AAM1B,eAAsB,eACpB,IACA,QACe;CACf,MAAM,EAAC,mBAAkB;AAEzB,MAAK,IAAI,IAAI,GAAG,IAAI,aAAa,KAAK;AACpC,MAAI,IAAI,GAAG;AACT,MAAG,OACD,kCAAkC,oBAAoB,IAAK,UAC5D;AACD,SAAM,MAAM,kBAAkB;;EAEhC,MAAM,wBAAQ,IAAI,MAAM;AAExB,MADiB,MAAM,WAAW,IAAI,OAAO,CAE3C,QAAO;AAET,MACE,eAAe,SAAS,eACxB,eAAe,QAAQ,KAAA,GACvB;AACA,MAAG,OAAO,6BAA6B;AACvC,UAAO;;;AAGX,OAAM,IAAI,MAAM,0CAA0C;;AAG5D,SAAS,cACP,MACA,QACA,kBACA,mBAIA;CACA,MAAM,EACJ,YACA,cACA,gBACA,WACA,UACA,YACA,UACA,OAAO,OAAO,OAAO,GACrB,uBACA,yBAAyB,wBAAwB,KACjD,yBAAyB,yBAAyB,IAClD,kCACA,6BACA,sBACA,kBACE,OAAO;CAIX,MAAM,gCAAgC,8BAA8B,KAAK;AAQzE,QAAO;EACL,aAHC,SAAS,aAAa,iBAAiB,eAAe,eACvD,KAAK,YAAY,kCAAkC;EAGnD,KAAK;GACH,GAAG,QAAQ;IACV,sBAAsB,OAAO,QAAQ;IACrC,+BAA+B,KAAK,qBAAqB,UAAU;IACnE,8CAA8C,OAC7C,uBACD;IACA,8CAA8C,OAC7C,uBACD;IACA,wDAAwD,OACvD,iCACD;IACA,8BAA8B,oBAAoB;IAClD,qDAAqD,OACpD,8BACD;IACA,0CAA0C,OAAO,qBAAqB;IACtE,mCAAmC,OAAO,cAAc;IACxD,oBAAoB,OAAO,IAAI;IAC/B,sBAAsB;IACtB,oBAAoB,OAAO,KAAK;GACjC,GAAI,WAAW,GAAE,6BAA6B,UAAS,GAAG,EAAE;GAC7D;EACF;;AAGH,eAAe,WAAW,IAAgB,QAAoB;CAC5D,MAAM,EAAC,mBAAkB;CAEzB,MAAM,eACJ,eAAe,SAAS,cAAc,eAAe,QAAQ,KAAA;CAI/D,MAAM,eAAe,4BAA4B,IAAI,QAAQ,aAAa;CAC1E,IAAI;AACJ,KAAI,cAAc;AAEhB,mBAAiB,MAAM;AACvB,KAAG,OAAO,yBAAyB,eAAe,YAAY;OAMzD,cAAa,OAAM,MAAK,GAAG,QAAQ,EAAE,CAAC;CAG7C,MAAM,EAAC,YAAY,QAAO,cACxB,WACA,QACA,SACA,gBAAgB,UACjB;CACD,MAAM,EAAC,oBAAoB,gBAAe,OAAO;CACjD,MAAM,OAAO,MACX,YACA;EACE;EACA;EACA;EACA;EACA,OAAO,YAAY;EACnB,OAAO,QAAQ;EAChB,EACD;EAAC;EAAK,OAAO;EAAW,aAAa;EAAK,CAC3C;CACD,MAAM,EAAC,SAAS,SAAS,WAAU,UAAU;AAC7C,MAAK,GAAG,SAAS,OAAO;AACxB,MAAK,GAAG,UAAU,MAAM,WAAW;AACjC,MAAI,OACF,QAAO,0BAA0B,SAAS;WACjC,SAAS,EAClB,QAAO,+BAA+B,OAAO;MAE7C,UAAS;GAEX;AACF,OAAM;AACN,KAAI,CAAC,WAAW,OAAO,QAAQ,KAAK,CAClC,QAAO;AAET,KACE,kBACA,CAAC,eAAe,IAAI,OAAO,QAAQ,MAAM,eAAe,EACxD;AACA,KAAG,OAAO,8CAA8C;AACxD,eAAa,OAAO,QAAQ,KAAK;AACjC,SAAO;;AAET,QAAO;;AAGT,SAAS,eACP,IACA,SACA,UACA;CACA,MAAM,KAAK,IAAI,SAAS,IAAI,QAAQ;AACpC,KAAI;EACF,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,GAAG,CACxB;AACD,MAAI,mBAAmB,SAAS,gBAAgB;AAC9C,MAAG,OACD,yBAAyB,eAAe,iDAAiD,SAAS,kBAClG,SACD;AACD,UAAO;;AAET,MAAI,YAAY,SAAS,cAAc;AACrC,MAAG,OACD,2BAA2B,UAAU,gDAAgD,SAAS,eAC/F;AACD,UAAO;;AAET,KAAG,OACD,4BAA4B,eAAe,iBAAiB,UAAU,sCACtE,SACD;AACD,SAAO;UACA,GAAG;AACV,KAAG,QAAQ,2CAA2C,EAAE;AACxD,SAAO;WACC;AACR,KAAG,OAAO;;;AAId,SAAgB,0BACd,IACA,QACc;CACd,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,OAAO;AAC5D,IAAG,OAAO,iCAAiC,OAAO,WAAW,YAAY;AACzE,QAAO,MAAM,YAAY,CAAC,YAAY,EAAE;EACtC;EACA,OAAO;EACP,aAAa;EACd,CAAC;;AAGJ,SAAS,4BACP,IACA,QACA,cACyB;CACzB,MAAM,EAAC,SAAS,QAAQ,SAAS,WAAU,UAA0B;AAErE,EAAM,iBAAkB;EACtB,MAAM,QAAQ,IAAI,iBAAiB;AACnC,UAAQ,GAAG,gBAAgB,MAAM,OAAO,CAAC;AACzC,UAAQ,GAAG,iBAAiB,MAAM,OAAO,CAAC;AAE1C,OAAK,IAAI,IAAI,IAAK,KAAK;GACrB,IAAI,MAAwB;AAC5B,OAAI;IACF,IAAI,WAAW;IACf,MAAM,SAAS,MAAM,gBAAgB,IAAI,OAAO;AAChD,eAAW,MAAM,OAAO,QAAQ;AAI9B,aAAQ,IAAI,GAAG;AACf,gBAAW;;AAIb,QAAI,SACF;YAEK,GAAG;AACV,UAAM;;AAER,OAAI,CAAC,aACH,QAAO,OAAO,IAAI;AAUpB,MAAG,OACD,uCAAuC,IAAI,EAAE,4BAC7C,OAAO,IAAI,CACZ;AACD,OAAI;AACF,UAAM,MAAM,KAAM,MAAM,OAAO;YACxB,GAAG;AACV,WAAO,OAAO,EAAE;;;KAGlB;AAEJ,QAAO;;AAGT,SAAS,gBACP,IACA,QACkC;AAClC,kBAAiB,OAAO;CACxB,MAAM,EAAC,QAAQ,QAAQ,mBAAkB;AAUzC,QAP6B,IAAI,yBAC/B,IAHc,eAAe,OAAO,EAKpC,OAAO,IACP,eAAe,IAChB,CAE2B,gBAAgB,OAAO"}
+{"version":3,"file":"commands.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync} from 'node:fs';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../../shared/src/must.ts';\nimport {sleep} from '../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {assertNormalized} from '../../config/normalize.ts';\nimport type {ZeroConfig} from '../../config/zero-config.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getShardConfig} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {ChangeStreamerHttpClient} from '../change-streamer/change-streamer-http.ts';\nimport type {\n  SnapshotMessage,\n  SnapshotStatus,\n} from '../change-streamer/snapshot.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\n\n// Retry for up to 3 minutes (60 times with 3 second delay).\n// Beyond that, let the container runner restart the task.\nconst MAX_RETRIES = 60;\nconst RETRY_INTERVAL_MS = 3000;\n\ntype ReplicaConstraints = {\n  replicaVersion: string;\n  minWatermark: string;\n};\n\nexport class BackupNotFoundException extends Error {\n  static readonly name = 'BackupNotFoundException';\n\n  constructor(backupURL: string | undefined) {\n    super(`backup not found at ${backupURL}`);\n  }\n}\n\n/**\n * @param replicaConstraints The constraints of the restored backup when\n *        restoring for the change-streamer (replication-manager). For the\n *        view-syncer, this should be unspecified so that the constraints are\n *        retrieved from the replication-manager via the snapshot protocol.\n */\nexport async function restoreReplica(\n  lc: LogContext,\n  config: ZeroConfig,\n  replicaConstraints: ReplicaConstraints | null,\n) {\n  for (let i = 0; i < MAX_RETRIES; i++) {\n    if (i > 0) {\n      lc.info?.(\n        `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n      );\n      await sleep(RETRY_INTERVAL_MS);\n    }\n    const restored = await tryRestore(lc, config, replicaConstraints);\n    if (restored) {\n      return;\n    }\n    if (replicaConstraints) {\n      // This can happen if the litestream URL is purposefully changed to\n      // force a resync.\n      throw new BackupNotFoundException(config.litestream.backupURL);\n    }\n  }\n  throw new Error(`max attempts exceeded restoring replica`);\n}\n\nfunction getLitestream(\n  mode: 'restore' | 'replicate',\n  config: ZeroConfig,\n  logLevelOverride?: LogLevel,\n  backupURLOverride?: string,\n): {\n  litestream: string;\n  env: NodeJS.ProcessEnv;\n} {\n  const {\n    executable,\n    executableV5,\n    restoreUsingV5,\n    backupURL,\n    logLevel,\n    configPath,\n    endpoint,\n    port = config.port + 2,\n    checkpointThresholdMB,\n    minCheckpointPageCount = checkpointThresholdMB * 250, // SQLite page size is 4KB\n    maxCheckpointPageCount = minCheckpointPageCount * 10,\n    incrementalBackupIntervalMinutes,\n    snapshotBackupIntervalHours,\n    multipartConcurrency,\n    multipartSize,\n  } = config.litestream;\n\n  // Set the snapshot interval to something smaller than x hours so that\n  // the hourly check triggers on the hour, rather than the hour after.\n  const snapshotBackupIntervalMinutes = snapshotBackupIntervalHours * 60 - 5;\n\n  const litestream =\n    // The v0.5.8+ litestream executable can restore from either the new LTX\n    // format or the legacy WAL format, allowing forwards-compatibility /\n    // rollback safety with zero-cache versions that backup to LTX.\n    (mode === 'restore' && restoreUsingV5 ? executableV5 : executable) ??\n    must(executable, `Missing --litestream-executable`);\n  return {\n    litestream,\n    env: {\n      ...process.env,\n      ['ZERO_REPLICA_FILE']: config.replica.file,\n      ['ZERO_LITESTREAM_BACKUP_URL']: must(backupURLOverride ?? backupURL),\n      ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n        minCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n        maxCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n        incrementalBackupIntervalMinutes,\n      ),\n      ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n      ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_MINUTES']: String(\n        snapshotBackupIntervalMinutes,\n      ),\n      ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n      ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n      ['ZERO_LOG_FORMAT']: config.log.format,\n      ['LITESTREAM_CONFIG']: configPath,\n      ['LITESTREAM_PORT']: String(port),\n      ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n    },\n  };\n}\n\nasync function tryRestore(\n  lc: LogContext,\n  config: ZeroConfig,\n  replicaConstraints: ReplicaConstraints | null,\n) {\n  let snapshotStatus: SnapshotStatus | undefined;\n  if (!replicaConstraints) {\n    // view-syncers fetch replica constraints from the replication-manager\n    // via the snapshot protocol.\n    snapshotStatus = await reserveAndGetSnapshotStatus(lc, config);\n    lc.info?.(`restoring backup from ${snapshotStatus.backupURL}`);\n    replicaConstraints = snapshotStatus;\n  }\n\n  const {litestream, env} = getLitestream(\n    'restore',\n    config,\n    'debug', // Include all output from `litestream restore`, as it's minimal.\n    snapshotStatus?.backupURL,\n  );\n  const {restoreParallelism: parallelism} = config.litestream;\n  const proc = spawn(\n    litestream,\n    [\n      'restore',\n      '-if-db-not-exists',\n      '-if-replica-exists',\n      '-parallelism',\n      String(parallelism),\n      config.replica.file,\n    ],\n    {env, stdio: 'inherit', windowsHide: true},\n  );\n  const {promise, resolve, reject} = resolver();\n  proc.on('error', reject);\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      reject(`litestream killed with ${signal}`);\n    } else if (code !== 0) {\n      reject(`litestream exited with code ${code}`);\n    } else {\n      resolve();\n    }\n  });\n  await promise;\n  if (!existsSync(config.replica.file)) {\n    return false;\n  }\n  if (!replicaIsValid(lc, config.replica.file, replicaConstraints)) {\n    lc.info?.(`Deleting local replica and retrying restore`);\n    deleteLiteDB(config.replica.file);\n    return false;\n  }\n  return true;\n}\n\nfunction replicaIsValid(\n  lc: LogContext,\n  replica: string,\n  constraints: ReplicaConstraints,\n) {\n  const db = new Database(lc, replica);\n  try {\n    const {replicaVersion, watermark} = getSubscriptionState(\n      new StatementRunner(db),\n    );\n    if (replicaVersion !== constraints.replicaVersion) {\n      lc.warn?.(\n        `Local replica version ${replicaVersion} does not match expected replicaVersion ${constraints.replicaVersion}`,\n        constraints,\n      );\n      return false;\n    }\n    if (watermark < constraints.minWatermark) {\n      lc.warn?.(\n        `Local replica watermark ${watermark} is earlier than minWatermark ${constraints.minWatermark}`,\n      );\n      return false;\n    }\n    lc.info?.(\n      `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible`,\n      constraints,\n    );\n    return true;\n  } catch (e) {\n    lc.error?.('Error while validating restored replica', e);\n    return false;\n  } finally {\n    db.close();\n  }\n}\n\nexport function startReplicaBackupProcess(\n  lc: LogContext,\n  config: ZeroConfig,\n): ChildProcess {\n  const {litestream, env} = getLitestream('replicate', config);\n  lc.info?.(`starting litestream backup to ${config.litestream.backupURL}`);\n  return spawn(litestream, ['replicate'], {\n    env,\n    stdio: 'inherit',\n    windowsHide: true,\n  });\n}\n\nfunction reserveAndGetSnapshotStatus(\n  lc: LogContext,\n  config: ZeroConfig,\n): Promise<SnapshotStatus> {\n  const {promise: status, resolve, reject} = resolver<SnapshotStatus>();\n\n  void (async function () {\n    const abort = new AbortController();\n    process.on('SIGINT', () => abort.abort());\n    process.on('SIGTERM', () => abort.abort());\n\n    for (let i = 0; ; i++) {\n      let err: unknown;\n      try {\n        let resolved = false;\n        const stream = await reserveSnapshot(lc, config);\n        for await (const msg of stream) {\n          // Capture the value of the status message that the change-streamer\n          // (i.e. BackupMonitor) returns, and hold the connection open to\n          // \"reserve\" the snapshot and prevent change log cleanup.\n          resolve(msg[1]);\n          resolved = true;\n        }\n        // The change-streamer itself closes the connection when the\n        // subscription is started (or the reservation retried).\n        if (resolved) {\n          break;\n        }\n      } catch (e) {\n        err = e;\n      }\n      // Retry in the view-syncer since it cannot proceed until it connects\n      // to a (compatible) replication-manager. In particular, a\n      // replication-manager that does not support the view-syncer's\n      // change-streamer protocol will close the stream with an error; this\n      // retry logic essentially delays the startup of a view-syncer until\n      // a compatible replication-manager has been rolled out, allowing\n      // replication-manager and view-syncer services to be updated in\n      // parallel.\n      lc.warn?.(\n        `Unable to reserve snapshot (attempt ${i + 1}). Retrying in 5 seconds.`,\n        String(err),\n      );\n      try {\n        await sleep(5000, abort.signal);\n      } catch (e) {\n        return reject(e);\n      }\n    }\n  })();\n\n  return status;\n}\n\nfunction reserveSnapshot(\n  lc: LogContext,\n  config: ZeroConfig,\n): Promise<Source<SnapshotMessage>> {\n  assertNormalized(config);\n  const {taskID, change, changeStreamer} = config;\n  const shardID = getShardConfig(config);\n\n  const changeStreamerClient = new ChangeStreamerHttpClient(\n    lc,\n    shardID,\n    change.db,\n    changeStreamer.uri,\n  );\n\n  return changeStreamerClient.reserveSnapshot(taskID);\n}\n"],"mappings":";;;;;;;;;;;;;AAuBA,IAAM,cAAc;AACpB,IAAM,oBAAoB;AAO1B,IAAa,0BAAb,cAA6C,MAAM;CACjD,OAAgB,OAAO;CAEvB,YAAY,WAA+B;AACzC,QAAM,uBAAuB,YAAY;;;;;;;;;AAU7C,eAAsB,eACpB,IACA,QACA,oBACA;AACA,MAAK,IAAI,IAAI,GAAG,IAAI,aAAa,KAAK;AACpC,MAAI,IAAI,GAAG;AACT,MAAG,OACD,kCAAkC,oBAAoB,IAAK,UAC5D;AACD,SAAM,MAAM,kBAAkB;;AAGhC,MADiB,MAAM,WAAW,IAAI,QAAQ,mBAAmB,CAE/D;AAEF,MAAI,mBAGF,OAAM,IAAI,wBAAwB,OAAO,WAAW,UAAU;;AAGlE,OAAM,IAAI,MAAM,0CAA0C;;AAG5D,SAAS,cACP,MACA,QACA,kBACA,mBAIA;CACA,MAAM,EACJ,YACA,cACA,gBACA,WACA,UACA,YACA,UACA,OAAO,OAAO,OAAO,GACrB,uBACA,yBAAyB,wBAAwB,KACjD,yBAAyB,yBAAyB,IAClD,kCACA,6BACA,sBACA,kBACE,OAAO;CAIX,MAAM,gCAAgC,8BAA8B,KAAK;AAQzE,QAAO;EACL,aAHC,SAAS,aAAa,iBAAiB,eAAe,eACvD,KAAK,YAAY,kCAAkC;EAGnD,KAAK;GACH,GAAG,QAAQ;IACV,sBAAsB,OAAO,QAAQ;IACrC,+BAA+B,KAAK,qBAAqB,UAAU;IACnE,8CAA8C,OAC7C,uBACD;IACA,8CAA8C,OAC7C,uBACD;IACA,wDAAwD,OACvD,iCACD;IACA,8BAA8B,oBAAoB;IAClD,qDAAqD,OACpD,8BACD;IACA,0CAA0C,OAAO,qBAAqB;IACtE,mCAAmC,OAAO,cAAc;IACxD,oBAAoB,OAAO,IAAI;IAC/B,sBAAsB;IACtB,oBAAoB,OAAO,KAAK;GACjC,GAAI,WAAW,GAAE,6BAA6B,UAAS,GAAG,EAAE;GAC7D;EACF;;AAGH,eAAe,WACb,IACA,QACA,oBACA;CACA,IAAI;AACJ,KAAI,CAAC,oBAAoB;AAGvB,mBAAiB,MAAM,4BAA4B,IAAI,OAAO;AAC9D,KAAG,OAAO,yBAAyB,eAAe,YAAY;AAC9D,uBAAqB;;CAGvB,MAAM,EAAC,YAAY,QAAO,cACxB,WACA,QACA,SACA,gBAAgB,UACjB;CACD,MAAM,EAAC,oBAAoB,gBAAe,OAAO;CACjD,MAAM,OAAO,MACX,YACA;EACE;EACA;EACA;EACA;EACA,OAAO,YAAY;EACnB,OAAO,QAAQ;EAChB,EACD;EAAC;EAAK,OAAO;EAAW,aAAa;EAAK,CAC3C;CACD,MAAM,EAAC,SAAS,SAAS,WAAU,UAAU;AAC7C,MAAK,GAAG,SAAS,OAAO;AACxB,MAAK,GAAG,UAAU,MAAM,WAAW;AACjC,MAAI,OACF,QAAO,0BAA0B,SAAS;WACjC,SAAS,EAClB,QAAO,+BAA+B,OAAO;MAE7C,UAAS;GAEX;AACF,OAAM;AACN,KAAI,CAAC,WAAW,OAAO,QAAQ,KAAK,CAClC,QAAO;AAET,KAAI,CAAC,eAAe,IAAI,OAAO,QAAQ,MAAM,mBAAmB,EAAE;AAChE,KAAG,OAAO,8CAA8C;AACxD,eAAa,OAAO,QAAQ,KAAK;AACjC,SAAO;;AAET,QAAO;;AAGT,SAAS,eACP,IACA,SACA,aACA;CACA,MAAM,KAAK,IAAI,SAAS,IAAI,QAAQ;AACpC,KAAI;EACF,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,GAAG,CACxB;AACD,MAAI,mBAAmB,YAAY,gBAAgB;AACjD,MAAG,OACD,yBAAyB,eAAe,0CAA0C,YAAY,kBAC9F,YACD;AACD,UAAO;;AAET,MAAI,YAAY,YAAY,cAAc;AACxC,MAAG,OACD,2BAA2B,UAAU,gCAAgC,YAAY,eAClF;AACD,UAAO;;AAET,KAAG,OACD,4BAA4B,eAAe,iBAAiB,UAAU,iBACtE,YACD;AACD,SAAO;UACA,GAAG;AACV,KAAG,QAAQ,2CAA2C,EAAE;AACxD,SAAO;WACC;AACR,KAAG,OAAO;;;AAId,SAAgB,0BACd,IACA,QACc;CACd,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,OAAO;AAC5D,IAAG,OAAO,iCAAiC,OAAO,WAAW,YAAY;AACzE,QAAO,MAAM,YAAY,CAAC,YAAY,EAAE;EACtC;EACA,OAAO;EACP,aAAa;EACd,CAAC;;AAGJ,SAAS,4BACP,IACA,QACyB;CACzB,MAAM,EAAC,SAAS,QAAQ,SAAS,WAAU,UAA0B;AAErE,EAAM,iBAAkB;EACtB,MAAM,QAAQ,IAAI,iBAAiB;AACnC,UAAQ,GAAG,gBAAgB,MAAM,OAAO,CAAC;AACzC,UAAQ,GAAG,iBAAiB,MAAM,OAAO,CAAC;AAE1C,OAAK,IAAI,IAAI,IAAK,KAAK;GACrB,IAAI;AACJ,OAAI;IACF,IAAI,WAAW;IACf,MAAM,SAAS,MAAM,gBAAgB,IAAI,OAAO;AAChD,eAAW,MAAM,OAAO,QAAQ;AAI9B,aAAQ,IAAI,GAAG;AACf,gBAAW;;AAIb,QAAI,SACF;YAEK,GAAG;AACV,UAAM;;AAUR,MAAG,OACD,uCAAuC,IAAI,EAAE,4BAC7C,OAAO,IAAI,CACZ;AACD,OAAI;AACF,UAAM,MAAM,KAAM,MAAM,OAAO;YACxB,GAAG;AACV,WAAO,OAAO,EAAE;;;KAGlB;AAEJ,QAAO;;AAGT,SAAS,gBACP,IACA,QACkC;AAClC,kBAAiB,OAAO;CACxB,MAAM,EAAC,QAAQ,QAAQ,mBAAkB;AAUzC,QAP6B,IAAI,yBAC/B,IAHc,eAAe,OAAO,EAKpC,OAAO,IACP,eAAe,IAChB,CAE2B,gBAAgB,OAAO"}

package/out/zero-cache/src/services/mutagen/mutagen.js

@@ -10,9 +10,9 @@ import { MutationAlreadyProcessedError } from "./error.js";
 import { Database } from "../../../../zqlite/src/db.js";
 import { upstreamSchema } from "../../types/shards.js";
 import "../../config/zero-config.js";
+import { getOrCreateCounter } from "../../observability/metrics.js";
 import { SERIALIZABLE } from "../../db/mode-enum.js";
 import { runTx } from "../../db/run-transaction.js";
-import { getOrCreateCounter } from "../../observability/metrics.js";
 import { recordMutation } from "../../server/anonymous-otel-start.js";
 import { WriteAuthorizerImpl } from "../../auth/write-authorizer.js";
 import { SlidingWindowLimiter } from "../limiter/sliding-window-limiter.js";

package/out/zero-cache/src/services/mutagen/pusher.d.ts

@@ -37,19 +37,7 @@ export declare class PusherService implements Service, Pusher {
     }] | ["connected", {
         timestamp?: number | undefined;
         wsid: string;
-    }] | import("../../../../zero-protocol/src/error.ts").ErrorMessage | ["transformError", ({
-        message?: string | undefined;
-        details?: import("../../../../shared/src/json.ts").ReadonlyJSONValue | undefined;
-        error: "app";
-        id: string;
-        name: string;
-    } | {
-        details?: import("../../../../shared/src/json.ts").ReadonlyJSONValue | undefined;
-        error: "parse";
-        id: string;
-        name: string;
-        message: string;
-    })[]] | ["pushResponse", {
+    }] | import("../../../../zero-protocol/src/error.ts").ErrorMessage | ["pushResponse", {
         mutationIDs?: {
             id: number;
             clientID: string;
@@ -93,7 +81,19 @@ export declare class PusherService implements Service, Pusher {
                 data?: import("../../../../shared/src/json.ts").ReadonlyJSONValue | undefined;
             };
         }[];
-    }] | ["inspect", {
+    }] | ["transformError", ({
+        message?: string | undefined;
+        details?: import("../../../../shared/src/json.ts").ReadonlyJSONValue | undefined;
+        error: "app";
+        id: string;
+        name: string;
+    } | {
+        details?: import("../../../../shared/src/json.ts").ReadonlyJSONValue | undefined;
+        error: "parse";
+        id: string;
+        name: string;
+        message: string;
+    })[]] | ["inspect", {
         id: string;
         op: "queries";
         value: {
@@ -336,19 +336,7 @@ export declare class PusherService implements Service, Pusher {
     }] | ["connected", {
         timestamp?: number | undefined;
         wsid: string;
-    }] | import("../../../../zero-protocol/src/error.ts").ErrorMessage | ["transformError", ({
-        message?: string | undefined;
-        details?: import("../../../../shared/src/json.ts").ReadonlyJSONValue | undefined;
-        error: "app";
-        id: string;
-        name: string;
-    } | {
-        details?: import("../../../../shared/src/json.ts").ReadonlyJSONValue | undefined;
-        error: "parse";
-        id: string;
-        name: string;
-        message: string;
-    })[]] | ["pushResponse", {
+    }] | import("../../../../zero-protocol/src/error.ts").ErrorMessage | ["pushResponse", {
         mutationIDs?: {
             id: number;
             clientID: string;
@@ -392,7 +380,19 @@ export declare class PusherService implements Service, Pusher {
                 data?: import("../../../../shared/src/json.ts").ReadonlyJSONValue | undefined;
             };
         }[];
-    }] | ["inspect", {
+    }] | ["transformError", ({
+        message?: string | undefined;
+        details?: import("../../../../shared/src/json.ts").ReadonlyJSONValue | undefined;
+        error: "app";
+        id: string;
+        name: string;
+    } | {
+        details?: import("../../../../shared/src/json.ts").ReadonlyJSONValue | undefined;
+        error: "parse";
+        id: string;
+        name: string;
+        message: string;
+    })[]] | ["inspect", {
         id: string;
         op: "queries";
         value: {

package/out/zero-cache/src/services/replicator/change-processor.js

@@ -2,15 +2,15 @@ import { assert, unreachable } from "../../../../shared/src/asserts.js";
 import { AbortError } from "../../../../shared/src/abort-error.js";
 import { must } from "../../../../shared/src/must.js";
 import { stringify } from "../../../../shared/src/bigint-json.js";
-import { ChangeLog } from "./schema/change-log.js";
 import { ZERO_VERSION_COLUMN_NAME } from "./schema/constants.js";
 import { liteRow } from "../../types/lite.js";
 import { liteTableName } from "../../types/names.js";
 import { mapPostgresToLite, mapPostgresToLiteColumn, mapPostgresToLiteIndex } from "../../db/pg-to-lite.js";
 import { ColumnMetadataStore } from "./schema/column-metadata.js";
 import { TableMetadataTracker } from "./schema/table-metadata.js";
-import { updateReplicationWatermark } from "./schema/replication-state.js";
 import { computeZqlSpecs, listIndexes, listTables } from "../../db/lite-tables.js";
+import { ChangeLog } from "./schema/change-log.js";
+import { updateReplicationWatermark } from "./schema/replication-state.js";
 import { id } from "../../types/sql.js";
 import { createLiteIndexStatement, createLiteTableStatement, liteColumnDef } from "../../db/create.js";
 import { SqliteError } from "@rocicorp/zero-sqlite3";

package/out/zero-cache/src/services/replicator/incremental-sync.js

@@ -1,6 +1,6 @@
 import { AbortError } from "../../../../shared/src/abort-error.js";
-import { RunningState } from "../running-state.js";
 import { getOrCreateCounter } from "../../observability/metrics.js";
+import { RunningState } from "../running-state.js";
 import { errorTypeToReadableName } from "../change-streamer/change-streamer.js";
 import { Notifier } from "./notifier.js";
 import { ReplicationReportRecorder } from "./reporter/recorder.js";

package/out/zero-cache/src/services/replicator/schema/replication-state.js

@@ -1,9 +1,9 @@
 import { parse, valita_exports } from "../../../../../shared/src/valita.js";
 import { jsonObjectSchema, stringify } from "../../../../../shared/src/bigint-json.js";
-import { CREATE_CHANGELOG_SCHEMA } from "./change-log.js";
 import "./constants.js";
 import { CREATE_COLUMN_METADATA_TABLE } from "./column-metadata.js";
 import { CREATE_TABLE_METADATA_TABLE } from "./table-metadata.js";
+import { CREATE_CHANGELOG_SCHEMA } from "./change-log.js";
 //#region ../zero-cache/src/services/replicator/schema/replication-state.ts
 /**
 * Replication metadata, used for incremental view maintenance and catchup.

package/out/zero-cache/src/services/replicator/write-worker-client.js.map

@@ -1 +1 @@
-{"version":3,"file":"write-worker-client.js","names":["#worker","#rejectAll","#errorHandler","#pending","#terminated","#call"],"sources":["../../../../../../zero-cache/src/services/replicator/write-worker-client.ts"],"sourcesContent":["import {resolver, type Resolver} from '@rocicorp/resolver';\nimport {Worker} from 'node:worker_threads';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport type {LogConfig} from '../../../../shared/src/logging.ts';\nimport type {Database} from '../../../../zqlite/src/db.ts';\nimport {WRITE_WORKER_URL} from '../../server/worker-urls.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport type {ChangeProcessorMode, CommitResult} from './change-processor.ts';\nimport type {SubscriptionState} from './schema/replication-state.ts';\n\nexport type PragmaConfig = {\n  busyTimeout: number;\n  analysisLimit: number;\n  walAutocheckpoint?: number | undefined;\n};\n\ntype ErrorHandler = (err: Error) => void;\n\n/**\n * Interface for a write worker that processes replication messages.\n */\nexport interface WriteWorkerClient {\n  getSubscriptionState(): Promise<SubscriptionState>;\n  processMessage(downstream: ChangeStreamData): Promise<CommitResult | null>;\n  abort(): void;\n  stop(): Promise<void>;\n  onError(handler: ErrorHandler): void;\n}\n\n// Wire protocol types — errors are passed directly via structured clone\nexport type ArgsMap = {\n  init: [string, ChangeProcessorMode, PragmaConfig, LogConfig];\n  getSubscriptionState: [];\n  processMessage: [ChangeStreamData];\n  abort: [];\n  stop: [];\n};\n\nexport type Method = keyof ArgsMap;\n\nexport type Request<M extends Method = Method> = {method: M; args: ArgsMap[M]};\n\nexport type ResultMap = {\n  init: void;\n  getSubscriptionState: SubscriptionState;\n  processMessage: CommitResult | null;\n  abort: void;\n  stop: void;\n};\n\nexport type Response<M extends Method = Method> =\n  | {method: M; result: ResultMap[M]; error?: undefined}\n  | {method: M; error: unknown; result?: undefined};\n\nexport type WriteError = {writeError: Error};\n\nexport function applyPragmas(db: Database, pragmas: PragmaConfig) {\n  db.pragma(`busy_timeout = ${pragmas.busyTimeout}`);\n  db.pragma(`analysis_limit = ${pragmas.analysisLimit}`);\n  if (pragmas.walAutocheckpoint !== undefined) {\n    db.pragma(`wal_autocheckpoint = ${pragmas.walAutocheckpoint}`);\n  }\n}\n\n/**\n * Delegates SQLite writes to a worker_thread,\n * keeping the main event loop free for WebSocket heartbeats and IPC.\n */\nexport class ThreadWriteWorkerClient implements WriteWorkerClient {\n  readonly #worker: Worker;\n  #pending: Resolver<unknown, Error> | null = null;\n  #errorHandler: ErrorHandler = () => {};\n  #terminated = false;\n\n  constructor() {\n    this.#worker = new Worker(WRITE_WORKER_URL);\n\n    this.#worker.on('message', (msg: Response | WriteError) => {\n      if ('writeError' in msg) {\n        const error =\n          msg.writeError instanceof Error\n            ? msg.writeError\n            : new Error(String(msg.writeError));\n        this.#rejectAll(error);\n        this.#errorHandler(error);\n        return;\n      }\n      const r = this.#pending;\n      if (!r) return; // stale abort response\n      this.#pending = null;\n      if (msg.error !== undefined) {\n        r.reject(\n          msg.error instanceof Error ? msg.error : new Error(String(msg.error)),\n        );\n      } else {\n        r.resolve(msg.result);\n      }\n    });\n\n    this.#worker.on('error', (err: Error) => {\n      this.#rejectAll(err);\n      this.#errorHandler(err);\n    });\n\n    this.#worker.on('exit', (code: number) => {\n      this.#terminated = true;\n      if (code !== 0) {\n        const err = new Error(`Worker exited with code ${code}`);\n        this.#rejectAll(err);\n        this.#errorHandler(err);\n      }\n    });\n  }\n\n  #rejectAll(err: Error) {\n    const r = this.#pending;\n    if (r) {\n      this.#pending = null;\n      r.reject(err);\n    }\n  }\n\n  #call<M extends Method>(method: M, args: ArgsMap[M]): Promise<ResultMap[M]> {\n    assert(this.#pending === null, `concurrent call: ${method}`);\n    const r = resolver<ResultMap[M]>();\n    this.#pending = r as Resolver<unknown, Error>;\n    this.#worker.postMessage({method, args} satisfies Request);\n    return r.promise;\n  }\n\n  init(\n    dbPath: string,\n    mode: ChangeProcessorMode,\n    pragmas: PragmaConfig,\n    logConfig: LogConfig,\n  ): Promise<void> {\n    return this.#call('init', [dbPath, mode, pragmas, logConfig]);\n  }\n\n  getSubscriptionState(): Promise<SubscriptionState> {\n    return this.#call('getSubscriptionState', []);\n  }\n\n  processMessage(downstream: ChangeStreamData): Promise<CommitResult | null> {\n    return this.#call('processMessage', [downstream]);\n  }\n\n  abort(): void {\n    if (!this.#terminated) {\n      this.#worker.postMessage({method: 'abort', args: []} satisfies Request);\n    }\n  }\n\n  async stop(): Promise<void> {\n    await this.#call('stop', []);\n    if (!this.#terminated) {\n      await this.#worker.terminate();\n    }\n  }\n\n  onError(handler: ErrorHandler): void {\n    this.#errorHandler = handler;\n  }\n}\n"],"mappings":";;;;;AAwDA,SAAgB,aAAa,IAAc,SAAuB;AAChE,IAAG,OAAO,kBAAkB,QAAQ,cAAc;AAClD,IAAG,OAAO,oBAAoB,QAAQ,gBAAgB;AACtD,KAAI,QAAQ,sBAAsB,KAAA,EAChC,IAAG,OAAO,wBAAwB,QAAQ,oBAAoB;;;;;;AAQlE,IAAa,0BAAb,MAAkE;CAChE;CACA,WAA4C;CAC5C,sBAAoC;CACpC,cAAc;CAEd,cAAc;AACZ,QAAA,SAAe,IAAI,OAAO,iBAAiB;AAE3C,QAAA,OAAa,GAAG,YAAY,QAA+B;AACzD,OAAI,gBAAgB,KAAK;IACvB,MAAM,QACJ,IAAI,sBAAsB,QACtB,IAAI,aACJ,IAAI,MAAM,OAAO,IAAI,WAAW,CAAC;AACvC,UAAA,UAAgB,MAAM;AACtB,UAAA,aAAmB,MAAM;AACzB;;GAEF,MAAM,IAAI,MAAA;AACV,OAAI,CAAC,EAAG;AACR,SAAA,UAAgB;AAChB,OAAI,IAAI,UAAU,KAAA,EAChB,GAAE,OACA,IAAI,iBAAiB,QAAQ,IAAI,QAAQ,IAAI,MAAM,OAAO,IAAI,MAAM,CAAC,CACtE;OAED,GAAE,QAAQ,IAAI,OAAO;IAEvB;AAEF,QAAA,OAAa,GAAG,UAAU,QAAe;AACvC,SAAA,UAAgB,IAAI;AACpB,SAAA,aAAmB,IAAI;IACvB;AAEF,QAAA,OAAa,GAAG,SAAS,SAAiB;AACxC,SAAA,aAAmB;AACnB,OAAI,SAAS,GAAG;IACd,MAAM,sBAAM,IAAI,MAAM,2BAA2B,OAAO;AACxD,UAAA,UAAgB,IAAI;AACpB,UAAA,aAAmB,IAAI;;IAEzB;;CAGJ,WAAW,KAAY;EACrB,MAAM,IAAI,MAAA;AACV,MAAI,GAAG;AACL,SAAA,UAAgB;AAChB,KAAE,OAAO,IAAI;;;CAIjB,MAAwB,QAAW,MAAyC;AAC1E,SAAO,MAAA,YAAkB,MAAM,oBAAoB,SAAS;EAC5D,MAAM,IAAI,UAAwB;AAClC,QAAA,UAAgB;AAChB,QAAA,OAAa,YAAY;GAAC;GAAQ;GAAK,CAAmB;AAC1D,SAAO,EAAE;;CAGX,KACE,QACA,MACA,SACA,WACe;AACf,SAAO,MAAA,KAAW,QAAQ;GAAC;GAAQ;GAAM;GAAS;GAAU,CAAC;;CAG/D,uBAAmD;AACjD,SAAO,MAAA,KAAW,wBAAwB,EAAE,CAAC;;CAG/C,eAAe,YAA4D;AACzE,SAAO,MAAA,KAAW,kBAAkB,CAAC,WAAW,CAAC;;CAGnD,QAAc;AACZ,MAAI,CAAC,MAAA,WACH,OAAA,OAAa,YAAY;GAAC,QAAQ;GAAS,MAAM,EAAE;GAAC,CAAmB;;CAI3E,MAAM,OAAsB;AAC1B,QAAM,MAAA,KAAW,QAAQ,EAAE,CAAC;AAC5B,MAAI,CAAC,MAAA,WACH,OAAM,MAAA,OAAa,WAAW;;CAIlC,QAAQ,SAA6B;AACnC,QAAA,eAAqB"}
+{"version":3,"file":"write-worker-client.js","names":["#worker","#rejectAll","#errorHandler","#pending","#terminated","#call"],"sources":["../../../../../../zero-cache/src/services/replicator/write-worker-client.ts"],"sourcesContent":["import {Worker} from 'node:worker_threads';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport type {LogConfig} from '../../../../shared/src/logging.ts';\nimport type {Database} from '../../../../zqlite/src/db.ts';\nimport {WRITE_WORKER_URL} from '../../server/worker-urls.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport type {ChangeProcessorMode, CommitResult} from './change-processor.ts';\nimport type {SubscriptionState} from './schema/replication-state.ts';\n\nexport type PragmaConfig = {\n  busyTimeout: number;\n  analysisLimit: number;\n  walAutocheckpoint?: number | undefined;\n};\n\ntype ErrorHandler = (err: Error) => void;\n\n/**\n * Interface for a write worker that processes replication messages.\n */\nexport interface WriteWorkerClient {\n  getSubscriptionState(): Promise<SubscriptionState>;\n  processMessage(downstream: ChangeStreamData): Promise<CommitResult | null>;\n  abort(): void;\n  stop(): Promise<void>;\n  onError(handler: ErrorHandler): void;\n}\n\n// Wire protocol types — errors are passed directly via structured clone\nexport type ArgsMap = {\n  init: [string, ChangeProcessorMode, PragmaConfig, LogConfig];\n  getSubscriptionState: [];\n  processMessage: [ChangeStreamData];\n  abort: [];\n  stop: [];\n};\n\nexport type Method = keyof ArgsMap;\n\nexport type Request<M extends Method = Method> = {method: M; args: ArgsMap[M]};\n\nexport type ResultMap = {\n  init: void;\n  getSubscriptionState: SubscriptionState;\n  processMessage: CommitResult | null;\n  abort: void;\n  stop: void;\n};\n\nexport type Response<M extends Method = Method> =\n  | {method: M; result: ResultMap[M]; error?: undefined}\n  | {method: M; error: unknown; result?: undefined};\n\nexport type WriteError = {writeError: Error};\n\nexport function applyPragmas(db: Database, pragmas: PragmaConfig) {\n  db.pragma(`busy_timeout = ${pragmas.busyTimeout}`);\n  db.pragma(`analysis_limit = ${pragmas.analysisLimit}`);\n  if (pragmas.walAutocheckpoint !== undefined) {\n    db.pragma(`wal_autocheckpoint = ${pragmas.walAutocheckpoint}`);\n  }\n}\n\n/**\n * Delegates SQLite writes to a worker_thread,\n * keeping the main event loop free for WebSocket heartbeats and IPC.\n */\nexport class ThreadWriteWorkerClient implements WriteWorkerClient {\n  readonly #worker: Worker;\n  #pending: Resolver<unknown, Error> | null = null;\n  #errorHandler: ErrorHandler = () => {};\n  #terminated = false;\n\n  constructor() {\n    this.#worker = new Worker(WRITE_WORKER_URL);\n\n    this.#worker.on('message', (msg: Response | WriteError) => {\n      if ('writeError' in msg) {\n        const error =\n          msg.writeError instanceof Error\n            ? msg.writeError\n            : new Error(String(msg.writeError));\n        this.#rejectAll(error);\n        this.#errorHandler(error);\n        return;\n      }\n      const r = this.#pending;\n      if (!r) return; // stale abort response\n      this.#pending = null;\n      if (msg.error !== undefined) {\n        r.reject(\n          msg.error instanceof Error ? msg.error : new Error(String(msg.error)),\n        );\n      } else {\n        r.resolve(msg.result);\n      }\n    });\n\n    this.#worker.on('error', (err: Error) => {\n      this.#rejectAll(err);\n      this.#errorHandler(err);\n    });\n\n    this.#worker.on('exit', (code: number) => {\n      this.#terminated = true;\n      if (code !== 0) {\n        const err = new Error(`Worker exited with code ${code}`);\n        this.#rejectAll(err);\n        this.#errorHandler(err);\n      }\n    });\n  }\n\n  #rejectAll(err: Error) {\n    const r = this.#pending;\n    if (r) {\n      this.#pending = null;\n      r.reject(err);\n    }\n  }\n\n  #call<M extends Method>(method: M, args: ArgsMap[M]): Promise<ResultMap[M]> {\n    assert(this.#pending === null, `concurrent call: ${method}`);\n    const r = resolver<ResultMap[M]>();\n    this.#pending = r as Resolver<unknown, Error>;\n    this.#worker.postMessage({method, args} satisfies Request);\n    return r.promise;\n  }\n\n  init(\n    dbPath: string,\n    mode: ChangeProcessorMode,\n    pragmas: PragmaConfig,\n    logConfig: LogConfig,\n  ): Promise<void> {\n    return this.#call('init', [dbPath, mode, pragmas, logConfig]);\n  }\n\n  getSubscriptionState(): Promise<SubscriptionState> {\n    return this.#call('getSubscriptionState', []);\n  }\n\n  processMessage(downstream: ChangeStreamData): Promise<CommitResult | null> {\n    return this.#call('processMessage', [downstream]);\n  }\n\n  abort(): void {\n    if (!this.#terminated) {\n      this.#worker.postMessage({method: 'abort', args: []} satisfies Request);\n    }\n  }\n\n  async stop(): Promise<void> {\n    await this.#call('stop', []);\n    if (!this.#terminated) {\n      await this.#worker.terminate();\n    }\n  }\n\n  onError(handler: ErrorHandler): void {\n    this.#errorHandler = handler;\n  }\n}\n"],"mappings":";;;;;AAwDA,SAAgB,aAAa,IAAc,SAAuB;AAChE,IAAG,OAAO,kBAAkB,QAAQ,cAAc;AAClD,IAAG,OAAO,oBAAoB,QAAQ,gBAAgB;AACtD,KAAI,QAAQ,sBAAsB,KAAA,EAChC,IAAG,OAAO,wBAAwB,QAAQ,oBAAoB;;;;;;AAQlE,IAAa,0BAAb,MAAkE;CAChE;CACA,WAA4C;CAC5C,sBAAoC;CACpC,cAAc;CAEd,cAAc;AACZ,QAAA,SAAe,IAAI,OAAO,iBAAiB;AAE3C,QAAA,OAAa,GAAG,YAAY,QAA+B;AACzD,OAAI,gBAAgB,KAAK;IACvB,MAAM,QACJ,IAAI,sBAAsB,QACtB,IAAI,aACJ,IAAI,MAAM,OAAO,IAAI,WAAW,CAAC;AACvC,UAAA,UAAgB,MAAM;AACtB,UAAA,aAAmB,MAAM;AACzB;;GAEF,MAAM,IAAI,MAAA;AACV,OAAI,CAAC,EAAG;AACR,SAAA,UAAgB;AAChB,OAAI,IAAI,UAAU,KAAA,EAChB,GAAE,OACA,IAAI,iBAAiB,QAAQ,IAAI,QAAQ,IAAI,MAAM,OAAO,IAAI,MAAM,CAAC,CACtE;OAED,GAAE,QAAQ,IAAI,OAAO;IAEvB;AAEF,QAAA,OAAa,GAAG,UAAU,QAAe;AACvC,SAAA,UAAgB,IAAI;AACpB,SAAA,aAAmB,IAAI;IACvB;AAEF,QAAA,OAAa,GAAG,SAAS,SAAiB;AACxC,SAAA,aAAmB;AACnB,OAAI,SAAS,GAAG;IACd,MAAM,sBAAM,IAAI,MAAM,2BAA2B,OAAO;AACxD,UAAA,UAAgB,IAAI;AACpB,UAAA,aAAmB,IAAI;;IAEzB;;CAGJ,WAAW,KAAY;EACrB,MAAM,IAAI,MAAA;AACV,MAAI,GAAG;AACL,SAAA,UAAgB;AAChB,KAAE,OAAO,IAAI;;;CAIjB,MAAwB,QAAW,MAAyC;AAC1E,SAAO,MAAA,YAAkB,MAAM,oBAAoB,SAAS;EAC5D,MAAM,IAAI,UAAwB;AAClC,QAAA,UAAgB;AAChB,QAAA,OAAa,YAAY;GAAC;GAAQ;GAAK,CAAmB;AAC1D,SAAO,EAAE;;CAGX,KACE,QACA,MACA,SACA,WACe;AACf,SAAO,MAAA,KAAW,QAAQ;GAAC;GAAQ;GAAM;GAAS;GAAU,CAAC;;CAG/D,uBAAmD;AACjD,SAAO,MAAA,KAAW,wBAAwB,EAAE,CAAC;;CAG/C,eAAe,YAA4D;AACzE,SAAO,MAAA,KAAW,kBAAkB,CAAC,WAAW,CAAC;;CAGnD,QAAc;AACZ,MAAI,CAAC,MAAA,WACH,OAAA,OAAa,YAAY;GAAC,QAAQ;GAAS,MAAM,EAAE;GAAC,CAAmB;;CAI3E,MAAM,OAAsB;AAC1B,QAAM,MAAA,KAAW,QAAQ,EAAE,CAAC;AAC5B,MAAI,CAAC,MAAA,WACH,OAAM,MAAA,OAAa,WAAW;;CAIlC,QAAQ,SAA6B;AACnC,QAAA,eAAqB"}

package/out/zero-cache/src/services/replicator/write-worker.js

@@ -1,8 +1,8 @@
 import { must } from "../../../../shared/src/must.js";
 import { Database } from "../../../../zqlite/src/db.js";
-import { createLogContext } from "../../server/logging.js";
-import { StatementRunner } from "../../db/statements.js";
 import { getSubscriptionState } from "./schema/replication-state.js";
+import { StatementRunner } from "../../db/statements.js";
+import { createLogContext } from "../../server/logging.js";
 import { ChangeProcessor } from "./change-processor.js";
 import { applyPragmas } from "./write-worker-client.js";
 import { parentPort } from "node:worker_threads";

package/out/zero-cache/src/services/replicator/write-worker.js.map

@@ -1 +1 @@
-{"version":3,"file":"write-worker.js","names":[],"sources":["../../../../../../zero-cache/src/services/replicator/write-worker.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {parentPort} from 'node:worker_threads';\nimport type {LogConfig} from '../../../../shared/src/logging.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {createLogContext} from '../../server/logging.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport {ChangeProcessor, type ChangeProcessorMode} from './change-processor.ts';\nimport {getSubscriptionState} from './schema/replication-state.ts';\nimport {\n  applyPragmas,\n  type ArgsMap,\n  type Method,\n  type PragmaConfig,\n  type Request,\n  type Response,\n  type ResultMap,\n  type WriteError,\n} from './write-worker-client.ts';\n\nif (!parentPort) {\n  throw new Error('write-worker must be run as a worker thread');\n}\n\nconst port = parentPort;\n\ntype API = {[M in Method]: (...args: ArgsMap[M]) => ResultMap[M]};\n\nfunction createAPI(): API {\n  let db: Database | undefined;\n  let runner: StatementRunner | undefined;\n  let processor: ChangeProcessor | undefined;\n  let mode: ChangeProcessorMode | undefined;\n  let lc: LogContext | undefined;\n\n  function createProcessor() {\n    processor = new ChangeProcessor(must(runner), must(mode), (_lc, err) => {\n      port.postMessage({\n        writeError: err instanceof Error ? err : new Error(String(err)),\n      } satisfies WriteError);\n    });\n  }\n\n  return {\n    init(\n      dbPath: string,\n      cpMode: ChangeProcessorMode,\n      pragmas: PragmaConfig,\n      logConfig: LogConfig,\n    ) {\n      lc = createLogContext({log: logConfig}, {worker: 'write-worker'});\n      db = new Database(lc, dbPath);\n      applyPragmas(db, pragmas);\n      runner = new StatementRunner(db);\n      mode = cpMode;\n      createProcessor();\n    },\n\n    getSubscriptionState() {\n      return getSubscriptionState(must(runner));\n    },\n\n    processMessage(downstream: ChangeStreamData) {\n      return must(processor).processMessage(must(lc), downstream);\n    },\n\n    abort() {\n      must(processor).abort(must(lc));\n      createProcessor();\n    },\n\n    stop() {\n      db?.close();\n      db = undefined;\n      runner = undefined;\n      processor = undefined;\n    },\n  };\n}\n\nconst api = createAPI();\n\nport.on('message', (msg: Request) => {\n  try {\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any -- TS can't narrow msg.method + msg.args together\n    const result = (api[msg.method] as (...args: any[]) => unknown)(\n      ...msg.args,\n    );\n    // abort is fire-and-forget — no pending slot on the client side.\n    if (msg.method !== 'abort') {\n      port.postMessage({method: msg.method, result} as Response);\n    }\n  } catch (e) {\n    if (msg.method !== 'abort') {\n      port.postMessage({method: msg.method, error: e} as Response);\n    }\n  }\n});\n"],"mappings":";;;;;;;;;AAqBA,IAAI,CAAC,WACH,OAAM,IAAI,MAAM,8CAA8C;AAGhE,IAAM,OAAO;AAIb,SAAS,YAAiB;CACxB,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CAEJ,SAAS,kBAAkB;AACzB,cAAY,IAAI,gBAAgB,KAAK,OAAO,EAAE,KAAK,KAAK,GAAG,KAAK,QAAQ;AACtE,QAAK,YAAY,EACf,YAAY,eAAe,QAAQ,MAAM,IAAI,MAAM,OAAO,IAAI,CAAC,EAChE,CAAsB;IACvB;;AAGJ,QAAO;EACL,KACE,QACA,QACA,SACA,WACA;AACA,QAAK,iBAAiB,EAAC,KAAK,WAAU,EAAE,EAAC,QAAQ,gBAAe,CAAC;AACjE,QAAK,IAAI,SAAS,IAAI,OAAO;AAC7B,gBAAa,IAAI,QAAQ;AACzB,YAAS,IAAI,gBAAgB,GAAG;AAChC,UAAO;AACP,oBAAiB;;EAGnB,uBAAuB;AACrB,UAAO,qBAAqB,KAAK,OAAO,CAAC;;EAG3C,eAAe,YAA8B;AAC3C,UAAO,KAAK,UAAU,CAAC,eAAe,KAAK,GAAG,EAAE,WAAW;;EAG7D,QAAQ;AACN,QAAK,UAAU,CAAC,MAAM,KAAK,GAAG,CAAC;AAC/B,oBAAiB;;EAGnB,OAAO;AACL,OAAI,OAAO;AACX,QAAK,KAAA;AACL,YAAS,KAAA;AACT,eAAY,KAAA;;EAEf;;AAGH,IAAM,MAAM,WAAW;AAEvB,KAAK,GAAG,YAAY,QAAiB;AACnC,KAAI;EAEF,MAAM,SAAU,IAAI,IAAI,QACtB,GAAG,IAAI,KACR;AAED,MAAI,IAAI,WAAW,QACjB,MAAK,YAAY;GAAC,QAAQ,IAAI;GAAQ;GAAO,CAAa;UAErD,GAAG;AACV,MAAI,IAAI,WAAW,QACjB,MAAK,YAAY;GAAC,QAAQ,IAAI;GAAQ,OAAO;GAAE,CAAa;;EAGhE"}
+{"version":3,"file":"write-worker.js","names":[],"sources":["../../../../../../zero-cache/src/services/replicator/write-worker.ts"],"sourcesContent":["import {parentPort} from 'node:worker_threads';\nimport type {LogContext} from '@rocicorp/logger';\nimport type {LogConfig} from '../../../../shared/src/logging.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {createLogContext} from '../../server/logging.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport {ChangeProcessor, type ChangeProcessorMode} from './change-processor.ts';\nimport {getSubscriptionState} from './schema/replication-state.ts';\nimport {\n  applyPragmas,\n  type ArgsMap,\n  type Method,\n  type PragmaConfig,\n  type Request,\n  type Response,\n  type ResultMap,\n  type WriteError,\n} from './write-worker-client.ts';\n\nif (!parentPort) {\n  throw new Error('write-worker must be run as a worker thread');\n}\n\nconst port = parentPort;\n\ntype API = {[M in Method]: (...args: ArgsMap[M]) => ResultMap[M]};\n\nfunction createAPI(): API {\n  let db: Database | undefined;\n  let runner: StatementRunner | undefined;\n  let processor: ChangeProcessor | undefined;\n  let mode: ChangeProcessorMode | undefined;\n  let lc: LogContext | undefined;\n\n  function createProcessor() {\n    processor = new ChangeProcessor(must(runner), must(mode), (_lc, err) => {\n      port.postMessage({\n        writeError: err instanceof Error ? err : new Error(String(err)),\n      } satisfies WriteError);\n    });\n  }\n\n  return {\n    init(\n      dbPath: string,\n      cpMode: ChangeProcessorMode,\n      pragmas: PragmaConfig,\n      logConfig: LogConfig,\n    ) {\n      lc = createLogContext({log: logConfig}, {worker: 'write-worker'});\n      db = new Database(lc, dbPath);\n      applyPragmas(db, pragmas);\n      runner = new StatementRunner(db);\n      mode = cpMode;\n      createProcessor();\n    },\n\n    getSubscriptionState() {\n      return getSubscriptionState(must(runner));\n    },\n\n    processMessage(downstream: ChangeStreamData) {\n      return must(processor).processMessage(must(lc), downstream);\n    },\n\n    abort() {\n      must(processor).abort(must(lc));\n      createProcessor();\n    },\n\n    stop() {\n      db?.close();\n      db = undefined;\n      runner = undefined;\n      processor = undefined;\n    },\n  };\n}\n\nconst api = createAPI();\n\nport.on('message', (msg: Request) => {\n  try {\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any -- TS can't narrow msg.method + msg.args together\n    const result = (api[msg.method] as (...args: any[]) => unknown)(\n      ...msg.args,\n    );\n    // abort is fire-and-forget — no pending slot on the client side.\n    if (msg.method !== 'abort') {\n      port.postMessage({method: msg.method, result} as Response);\n    }\n  } catch (e) {\n    if (msg.method !== 'abort') {\n      port.postMessage({method: msg.method, error: e} as Response);\n    }\n  }\n});\n"],"mappings":";;;;;;;;;AAqBA,IAAI,CAAC,WACH,OAAM,IAAI,MAAM,8CAA8C;AAGhE,IAAM,OAAO;AAIb,SAAS,YAAiB;CACxB,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CAEJ,SAAS,kBAAkB;AACzB,cAAY,IAAI,gBAAgB,KAAK,OAAO,EAAE,KAAK,KAAK,GAAG,KAAK,QAAQ;AACtE,QAAK,YAAY,EACf,YAAY,eAAe,QAAQ,MAAM,IAAI,MAAM,OAAO,IAAI,CAAC,EAChE,CAAsB;IACvB;;AAGJ,QAAO;EACL,KACE,QACA,QACA,SACA,WACA;AACA,QAAK,iBAAiB,EAAC,KAAK,WAAU,EAAE,EAAC,QAAQ,gBAAe,CAAC;AACjE,QAAK,IAAI,SAAS,IAAI,OAAO;AAC7B,gBAAa,IAAI,QAAQ;AACzB,YAAS,IAAI,gBAAgB,GAAG;AAChC,UAAO;AACP,oBAAiB;;EAGnB,uBAAuB;AACrB,UAAO,qBAAqB,KAAK,OAAO,CAAC;;EAG3C,eAAe,YAA8B;AAC3C,UAAO,KAAK,UAAU,CAAC,eAAe,KAAK,GAAG,EAAE,WAAW;;EAG7D,QAAQ;AACN,QAAK,UAAU,CAAC,MAAM,KAAK,GAAG,CAAC;AAC/B,oBAAiB;;EAGnB,OAAO;AACL,OAAI,OAAO;AACX,QAAK,KAAA;AACL,YAAS,KAAA;AACT,eAAY,KAAA;;EAEf;;AAGH,IAAM,MAAM,WAAW;AAEvB,KAAK,GAAG,YAAY,QAAiB;AACnC,KAAI;EAEF,MAAM,SAAU,IAAI,IAAI,QACtB,GAAG,IAAI,KACR;AAED,MAAI,IAAI,WAAW,QACjB,MAAK,YAAY;GAAC,QAAQ,IAAI;GAAQ;GAAO,CAAa;UAErD,GAAG;AACV,MAAI,IAAI,WAAW,QACjB,MAAK,YAAY;GAAC,QAAQ,IAAI;GAAQ,OAAO;GAAE,CAAa;;EAGhE"}

package/out/zero-cache/src/services/run-ast.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run-ast.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/run-ast.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,oDAAoD,CAAC;AAC3F,OAAO,KAAK,EAAC,GAAG,EAAe,MAAM,mCAAmC,CAAC;AAEzE,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6CAA6C,CAAC;AAG9E,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,kDAAkD,CAAC;AACxF,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,yCAAyC,CAAC;AACxE,OAAO,EAEL,KAAK,eAAe,EACrB,MAAM,qCAAqC,CAAC;AAG7C,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,gDAAgD,CAAC;AACxF,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,2CAA2C,CAAC;AAC5E,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,2BAA2B,CAAC;AAExD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,iBAAiB,CAAC;AAG7C,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,gBAAgB,CAAC;AAGnD,MAAM,MAAM,aAAa,GAAG;IAC1B,gBAAgB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACvC,IAAI,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC3B,oBAAoB,CAAC,EAAE,UAAU,GAAG,SAAS,CAAC;IAC9C,SAAS,CAAC,EAAE,mBAAmB,GAAG,SAAS,CAAC;IAC5C,EAAE,EAAE,QAAQ,CAAC;IACb,IAAI,EAAE,eAAe,CAAC;IACtB,WAAW,CAAC,EAAE,iBAAiB,GAAG,SAAS,CAAC;IAC5C,YAAY,CAAC,EAAE,YAAY,GAAG,SAAS,CAAC;IACxC,UAAU,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACjC,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IACxC,UAAU,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CAClC,CAAC;AAEF,wBAAsB,MAAM,CAC1B,EAAE,EAAE,UAAU,EACd,YAAY,EAAE,YAAY,EAC1B,GAAG,EAAE,GAAG,EACR,aAAa,EAAE,OAAO,EACtB,OAAO,EAAE,aAAa,EACtB,YAAY,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAChC,OAAO,CAAC,kBAAkB,CAAC,CAyI7B"}
+{"version":3,"file":"run-ast.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/run-ast.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,oDAAoD,CAAC;AAC3F,OAAO,KAAK,EAAC,GAAG,EAAe,MAAM,mCAAmC,CAAC;AAEzE,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6CAA6C,CAAC;AAG9E,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,kDAAkD,CAAC;AACxF,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,yCAAyC,CAAC;AACxE,OAAO,EAEL,KAAK,eAAe,EACrB,MAAM,qCAAqC,CAAC;AAI7C,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,gDAAgD,CAAC;AACxF,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,2CAA2C,CAAC;AAC5E,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,2BAA2B,CAAC;AAExD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,iBAAiB,CAAC;AAG7C,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,gBAAgB,CAAC;AAGnD,MAAM,MAAM,aAAa,GAAG;IAC1B,gBAAgB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACvC,IAAI,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC3B,oBAAoB,CAAC,EAAE,UAAU,GAAG,SAAS,CAAC;IAC9C,SAAS,CAAC,EAAE,mBAAmB,GAAG,SAAS,CAAC;IAC5C,EAAE,EAAE,QAAQ,CAAC;IACb,IAAI,EAAE,eAAe,CAAC;IACtB,WAAW,CAAC,EAAE,iBAAiB,GAAG,SAAS,CAAC;IAC5C,YAAY,CAAC,EAAE,YAAY,GAAG,SAAS,CAAC;IACxC,UAAU,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACjC,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IACxC,UAAU,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CAClC,CAAC;AAEF,wBAAsB,MAAM,CAC1B,EAAE,EAAE,UAAU,EACd,YAAY,EAAE,YAAY,EAC1B,GAAG,EAAE,GAAG,EACR,aAAa,EAAE,OAAO,EACtB,OAAO,EAAE,aAAa,EACtB,YAAY,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAChC,OAAO,CAAC,kBAAkB,CAAC,CA4I7B"}

package/out/zero-cache/src/services/run-ast.js

@@ -5,9 +5,9 @@ import { mapAST } from "../../../zero-protocol/src/ast.js";
 import { hashOfAST } from "../../../zero-protocol/src/query-hash.js";
 import { skipYields } from "../../../zql/src/ivm/operator.js";
 import { buildPipeline } from "../../../zql/src/builder/builder.js";
-import { computeZqlSpecs } from "../db/lite-tables.js";
 import { astToZQL } from "../../../ast-to-zql/src/ast-to-zql.js";
 import { formatOutput } from "../../../ast-to-zql/src/format.js";
+import { computeZqlSpecs } from "../db/lite-tables.js";
 import { resolveSimpleScalarSubqueries } from "../../../zqlite/src/resolve-scalar-subqueries.js";
 import { transformAndHashQuery } from "../auth/read-authorizer.js";
 import { hydrate } from "./view-syncer/pipeline-driver.js";
@@ -51,7 +51,7 @@ async function runAst(lc, clientSchema, ast, isTransformed, options, yieldProces
 			await yieldProcess();
 			continue;
 		}
-		assert(rowChange.type === "add", "Hydration only handles add row changes");
+		assert(rowChange.type === 0, "Hydration only handles add row changes");
 		if (syncedRowCount % 10 === 0) await Promise.resolve();
 		if (syncedRowCount % 100 === 0) await sleep(1);
 		let rows = rowsByTable[rowChange.table];