@rocicorp/zero 1.4.0-canary.5 → 1.5.0-canary.0

package/out/zero-cache/src/services/life-cycle.js.map

@@ -1 +1 @@
-{"version":3,"file":"life-cycle.js","names":["#lc","#userFacing","#all","#exitImpl","#start","#ready","#startDrain","#kill","#exit","#runningState","#drainStart","#onExit","#initializing","#nextID","#stopInterval","#lastHeartbeat","#checkIntervalTimer","#checkStopInterval","#checkImmediateTimer"],"sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"sourcesContent":["import type {IncomingHttpHeaders} from 'node:http';\nimport {pid} from 'node:process';\nimport type {EventEmitter} from 'stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {\n  singleProcessMode,\n  type Subprocess,\n  type Worker,\n} from '../types/processes.ts';\nimport {RunningState} from './running-state.ts';\nimport type {SingletonService} from './service.ts';\n\n/**\n * * `user-facing` workers serve external requests and are the first to\n *   receive a `SIGTERM` or `SIGINT` signal for graceful shutdown.\n *\n * * `supporting` workers support `user-facing` workers and are sent\n *   the `SIGTERM` signal only after all `user-facing` workers have\n *   exited.\n *\n * For other kill signals, such as `SIGQUIT` and `SIGABRT`, all workers\n * are stopped without draining. `SIGQUIT` is used to represent an\n * intentional shutdown (for which draining is not beneficial), whereas\n * `SIGABRT` is used for unexpected process exits.\n */\nexport type WorkerType = 'user-facing' | 'supporting';\n\nexport const GRACEFUL_SHUTDOWN = ['SIGTERM', 'SIGINT'] as const;\nexport const FORCEFUL_SHUTDOWN = ['SIGQUIT', 'SIGABRT'] as const;\n\ntype GracefulShutdownSignal = (typeof GRACEFUL_SHUTDOWN)[number];\n\n// An internal error code used to indicate that a message has already been\n// logged at level ERRROR. When a process exits with this error code, the\n// parent process logs the exit at level WARN instead of ERROR.\nexport const UNHANDLED_EXCEPTION_ERROR_CODE = 13;\n\n// An internal error code used to indicate that the server should exit\n// without draining (e.g. due to a supporting worker get a signal to shut\n// down), but the exit is otherwise intentional.\nexport const INTENTIONAL_SHUTDOWN_ERROR_CODE = 14;\n\n/**\n * Handles readiness, termination signals, and coordination of graceful\n * shutdown.\n */\nexport class ProcessManager {\n  readonly #lc: LogContext;\n  readonly #userFacing = new Set<Subprocess>();\n  readonly #all = new Set<Subprocess>();\n  readonly #exitImpl: (code: number) => never;\n  readonly #start = Date.now();\n  readonly #ready: Promise<void>[] = [];\n\n  #runningState = new RunningState('process-manager');\n  #drainStart = 0;\n\n  constructor(lc: LogContext, proc: EventEmitter) {\n    this.#lc = lc.withContext('component', 'process-manager');\n\n    // Propagate `SIGTERM` and `SIGINT` to all user-facing workers,\n    // initiating a graceful shutdown. The parent process will\n    // exit once all user-facing workers have exited ...\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.on(signal, () => this.#startDrain(signal));\n    }\n\n    // ... which will result in sending `SIGTERM` to the remaining workers.\n    proc.on('exit', code =>\n      this.#kill(\n        this.#all,\n        code === 0\n          ? 'SIGTERM' // graceful, drained shutdown\n          : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n            ? 'SIGQUIT' // intentional abort without drain\n            : 'SIGABRT', // unintentional shutdown, alertable error\n      ),\n    );\n\n    // For other (catchable) kill signals, exit with a non-zero error code\n    // to send a `SIGQUIT` (intentional shutdown) or `SIGABRT` (unexpected\n    // shutdown) to all workers. For these signals, workers are stopped\n    // immediately without draining, since there is no merit to slowly draining\n    // when supporting workers have stopped.\n    //\n    // The logic for handling these signals is in `runUntilKilled()`.\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      proc.on(signal, () =>\n        this.#exit(signal === 'SIGQUIT' ? INTENTIONAL_SHUTDOWN_ERROR_CODE : -1),\n      );\n    }\n\n    this.#exitImpl = (code: number) => {\n      if (singleProcessMode()) {\n        return proc.emit('exit', code) as never; // For unit / integration tests.\n      }\n      process.exit(code);\n    };\n  }\n\n  done() {\n    return this.#runningState.stopped();\n  }\n\n  #exit(code: number) {\n    this.#lc.info?.('exiting with code', code);\n    this.#runningState.stop(this.#lc);\n    void this.#lc.flush().finally(() => this.#exitImpl(code));\n  }\n\n  #startDrain(signal: GracefulShutdownSignal) {\n    if (this.#all.size === 0) {\n      // Shutdown if a signal is received before any subprocesses are added.\n      this.#lc.info?.(`exiting on ${signal}`);\n      this.#exit(0);\n    }\n    this.#lc.info?.(`initiating drain (${signal})`);\n    this.#drainStart = Date.now();\n    if (this.#userFacing.size) {\n      this.#kill(this.#userFacing, signal);\n    } else {\n      this.#kill(this.#all, signal);\n    }\n  }\n\n  addSubprocess(proc: Subprocess, type: WorkerType, name: string) {\n    if (type === 'user-facing') {\n      this.#userFacing.add(proc);\n    }\n    this.#all.add(proc);\n\n    let isOpen = true;\n    proc.on('close', (code, signal) => {\n      isOpen = false;\n      this.#onExit(code, signal, null, type, name, proc);\n    });\n\n    // As per https://nodejs.org/api/child_process.html#event-error\n    // 'error' events can happen when sending a message to a child process\n    // fails. This is not really an error when the server is shutting down,\n    // so log any post-close errors at 'warn'.\n    proc.on('error', err =>\n      this.#lc[!isOpen || this.#drainStart > 0 ? 'warn' : 'error']?.(\n        `error from ${name} ${proc.pid}`,\n        err,\n      ),\n    );\n  }\n\n  readonly #initializing = new Map<number, string>();\n  #nextID = 0;\n\n  addWorker(worker: Worker, type: WorkerType, name: string): Worker {\n    this.addSubprocess(worker, type, name);\n\n    const id = ++this.#nextID;\n    this.#initializing.set(id, name);\n    const {promise, resolve} = resolver();\n    this.#ready.push(promise);\n\n    worker.onceMessageType('ready', () => {\n      this.#lc.debug?.(`${name} ready (${Date.now() - this.#start} ms)`);\n      this.#initializing.delete(id);\n      resolve();\n    });\n\n    return worker;\n  }\n\n  initializing(): string[] {\n    return [...this.#initializing.values()];\n  }\n\n  async allWorkersReady() {\n    await Promise.all(this.#ready);\n  }\n\n  logErrorAndExit(err: unknown, name: string) {\n    // only accessible by the main (i.e. user-facing) process.\n    this.#onExit(-1, null, err, 'user-facing', name, undefined);\n  }\n\n  #onExit(\n    code: number,\n    sig: NodeJS.Signals | null,\n    err: unknown,\n    type: WorkerType,\n    name: string,\n    worker: Subprocess | undefined,\n  ) {\n    // Remove the worker from maps to avoid attempting to send more signals to it.\n    if (worker) {\n      this.#userFacing.delete(worker);\n      this.#all.delete(worker);\n    }\n\n    const pid = worker?.pid ?? process.pid;\n\n    if (type === 'supporting') {\n      // Supporting workers like the replication-manager shut down without a\n      // drain signal when receiving protocol-specific instructions (like auto\n      // reset). In this case, a special error code is used to signal that the\n      // server should be shut down without draining, but it is otherwise not\n      // considered an unexpected/alertable error.\n      if (code === 0 && (this.#drainStart === 0 || this.#userFacing.size > 0)) {\n        code = INTENTIONAL_SHUTDOWN_ERROR_CODE;\n      }\n      const log =\n        code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n          ? 'info'\n          : 'warn';\n      this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n      return this.#exit(code);\n    }\n\n    const log =\n      code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n        ? 'info'\n        : this.#drainStart > 0 || code === UNHANDLED_EXCEPTION_ERROR_CODE\n          ? 'warn'\n          : 'error';\n    this.#lc[log]?.(\n      sig\n        ? `${name} (${pid}) killed with (${sig})`\n        : `${name} (${pid}) exited with code (${code})`,\n      err ?? '',\n    );\n\n    // user-facing workers exited or finished draining.\n    if (this.#userFacing.size === 0) {\n      this.#lc.info?.(\n        this.#drainStart\n          ? `all user-facing workers drained (${\n              Date.now() - this.#drainStart\n            } ms)`\n          : `all user-facing workers exited`,\n      );\n      return this.#exit(0);\n    }\n\n    if (this.#drainStart === 0) {\n      // If a user-facing worker exits without receiving a drain signal,\n      // shutdown the server.\n      return this.#exit(code || -1);\n    }\n\n    return undefined;\n  }\n\n  #kill(workers: Iterable<Subprocess>, signal: NodeJS.Signals) {\n    for (const worker of workers) {\n      try {\n        worker.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\n/**\n * Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with\n * an optional {@link SingletonService.drain drain()}, or stopping them\n * without draining for `SIGQUIT`.\n *\n * @returns a Promise that resolves/rejects when any of the services stops/throws.\n */\n\nexport async function runUntilKilled(\n  lc: LogContext,\n  parent: EventEmitter,\n  ...services: SingletonService[]\n): Promise<void> {\n  if (services.length === 0) {\n    return;\n  }\n  for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) {\n    parent.once(signal, () => {\n      const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN as readonly NodeJS.Signals[];\n\n      services.forEach(async svc => {\n        if (GRACEFUL_SIGNALS.includes(signal) && svc.drain) {\n          lc.info?.(`draining ${svc.constructor.name} ${svc.id} (${signal})`);\n          await svc.drain();\n        }\n        lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${signal})`);\n        await svc.stop();\n      });\n    });\n  }\n\n  try {\n    // Run all services and resolve when any of them stops.\n    const svc = await Promise.race(\n      services.map(svc => svc.run().then(() => svc)),\n    );\n    lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);\n  } catch (e) {\n    lc.error?.(`exiting on error`, e);\n    throw e;\n  }\n}\n\nexport async function exitAfter(run: () => Promise<void>) {\n  try {\n    await run();\n    // oxlint-disable-next-line no-console\n    console.info(`pid ${pid} exiting normally`);\n    process.exit(0);\n  } catch (e) {\n    // oxlint-disable-next-line no-console\n    console.error(`pid ${pid} exiting with error`, e);\n    process.exit(-1);\n  }\n}\n\nconst DEFAULT_STOP_INTERVAL_MS = 20_000;\n\n/**\n * The HeartbeatMonitor monitors the cadence heartbeats (e.g. \"/keepalive\"\n * health checks made to HttpServices) that signal that the server\n * should continue processing requests. When a configurable `stopInterval`\n * elapses without receiving these heartbeats, the monitor initiates a\n * graceful shutdown of the server. This works with common load balancing\n * frameworks such as AWS Elastic Load Balancing.\n *\n * The HeartbeatMonitor is **opt-in** in that it only kicks in after it\n * starts receiving keepalives.\n */\nexport class HeartbeatMonitor {\n  readonly #stopInterval: number;\n\n  #lc: LogContext;\n  #checkIntervalTimer: NodeJS.Timeout | undefined;\n  #checkImmediateTimer: NodeJS.Immediate | undefined;\n  #lastHeartbeat = 0;\n\n  constructor(lc: LogContext, stopInterval = DEFAULT_STOP_INTERVAL_MS) {\n    this.#lc = lc;\n    this.#stopInterval = stopInterval;\n  }\n\n  onHeartbeat(reqHeaders: IncomingHttpHeaders) {\n    this.#lastHeartbeat = Date.now();\n    if (this.#checkIntervalTimer === undefined) {\n      this.#lc.info?.(\n        `starting heartbeat monitor at ${\n          this.#stopInterval / 1000\n        } second interval`,\n        reqHeaders,\n      );\n      // e.g. check every 5 seconds to see if it's been over 20 seconds\n      //      since the last heartbeat.\n      this.#checkIntervalTimer = setInterval(\n        this.#checkStopInterval,\n        this.#stopInterval / 4,\n      );\n    }\n  }\n\n  #checkStopInterval = () => {\n    // In the Node.js event loop, timers like setInterval and setTimeout\n    // run *before* I/O events coming from network sockets or file reads/writes.\n    // When this process gets starved of CPU resources for long periods of time,\n    // for example when other processes are monopolizing all available cores,\n    // pathological behavior can emerge:\n    // - keepalive network request comes in, but is queued in Node internals waiting\n    //   for time on the event loop\n    // - CPU is starved/monopolized by other processes for longer than the time\n    //   configured via this.#stopInterval\n    // - When CPU becomes available and the event loop wakes up, this stop interval\n    //   check is run *before* the keepalive request is processed. The value of\n    //   this.#lastHeartbeat is now very stale, and erroneously triggers a shutdown\n    //   even though keepalive requests were about to be processed and update\n    //   this.#lastHeartbeat. Downtime ensues.\n    //\n    // To avoid this, we push the check out to a phase of the event loop *after*\n    // I/O events are processed, using setImmediate():\n    // https://nodejs.org/en/learn/asynchronous-work/event-loop-timers-and-nexttick#setimmediate-vs-settimeout\n    //\n    // This ensures we see a value for this.#lastHeartbeat that reflects\n    // any keepalive requests that came in during the current event loop turn.\n    this.#checkImmediateTimer = setImmediate(() => {\n      this.#checkImmediateTimer = undefined;\n      const timeSinceLastHeartbeat = Date.now() - this.#lastHeartbeat;\n      if (timeSinceLastHeartbeat >= this.#stopInterval) {\n        this.#lc.info?.(\n          `last heartbeat received ${\n            timeSinceLastHeartbeat / 1000\n          } seconds ago. draining.`,\n        );\n        process.kill(process.pid, GRACEFUL_SHUTDOWN[0]);\n      }\n    });\n  };\n\n  stop() {\n    clearTimeout(this.#checkIntervalTimer);\n    if (this.#checkImmediateTimer) {\n      clearImmediate(this.#checkImmediateTimer);\n    }\n  }\n}\n"],"mappings":";;;;;AA4BA,IAAa,oBAAoB,CAAC,WAAW,SAAS;AACtD,IAAa,oBAAoB,CAAC,WAAW,UAAU;;;;;AAkBvD,IAAa,iBAAb,MAA4B;CAC1B;CACA,8BAAuB,IAAI,KAAiB;CAC5C,uBAAgB,IAAI,KAAiB;CACrC;CACA,SAAkB,KAAK,KAAK;CAC5B,SAAmC,EAAE;CAErC,gBAAgB,IAAI,aAAa,kBAAkB;CACnD,cAAc;CAEd,YAAY,IAAgB,MAAoB;AAC9C,QAAA,KAAW,GAAG,YAAY,aAAa,kBAAkB;AAKzD,OAAK,MAAM,UAAU,kBACnB,MAAK,GAAG,cAAc,MAAA,WAAiB,OAAO,CAAC;AAIjD,OAAK,GAAG,SAAQ,SACd,MAAA,KACE,MAAA,KACA,SAAS,IACL,YACA,SAAA,KACE,YACA,UACP,CACF;AASD,OAAK,MAAM,UAAU,kBACnB,MAAK,GAAG,cACN,MAAA,KAAW,WAAW,YAAA,KAA8C,GAAG,CACxE;AAGH,QAAA,YAAkB,SAAiB;AACjC,OAAI,mBAAmB,CACrB,QAAO,KAAK,KAAK,QAAQ,KAAK;AAEhC,WAAQ,KAAK,KAAK;;;CAItB,OAAO;AACL,SAAO,MAAA,aAAmB,SAAS;;CAGrC,MAAM,MAAc;AAClB,QAAA,GAAS,OAAO,qBAAqB,KAAK;AAC1C,QAAA,aAAmB,KAAK,MAAA,GAAS;AAC5B,QAAA,GAAS,OAAO,CAAC,cAAc,MAAA,SAAe,KAAK,CAAC;;CAG3D,YAAY,QAAgC;AAC1C,MAAI,MAAA,IAAU,SAAS,GAAG;AAExB,SAAA,GAAS,OAAO,cAAc,SAAS;AACvC,SAAA,KAAW,EAAE;;AAEf,QAAA,GAAS,OAAO,qBAAqB,OAAO,GAAG;AAC/C,QAAA,aAAmB,KAAK,KAAK;AAC7B,MAAI,MAAA,WAAiB,KACnB,OAAA,KAAW,MAAA,YAAkB,OAAO;MAEpC,OAAA,KAAW,MAAA,KAAW,OAAO;;CAIjC,cAAc,MAAkB,MAAkB,MAAc;AAC9D,MAAI,SAAS,cACX,OAAA,WAAiB,IAAI,KAAK;AAE5B,QAAA,IAAU,IAAI,KAAK;EAEnB,IAAI,SAAS;AACb,OAAK,GAAG,UAAU,MAAM,WAAW;AACjC,YAAS;AACT,SAAA,OAAa,MAAM,QAAQ,MAAM,MAAM,MAAM,KAAK;IAClD;AAMF,OAAK,GAAG,UAAS,QACf,MAAA,GAAS,CAAC,UAAU,MAAA,aAAmB,IAAI,SAAS,WAClD,cAAc,KAAK,GAAG,KAAK,OAC3B,IACD,CACF;;CAGH,gCAAyB,IAAI,KAAqB;CAClD,UAAU;CAEV,UAAU,QAAgB,MAAkB,MAAsB;AAChE,OAAK,cAAc,QAAQ,MAAM,KAAK;EAEtC,MAAM,KAAK,EAAE,MAAA;AACb,QAAA,aAAmB,IAAI,IAAI,KAAK;EAChC,MAAM,EAAC,SAAS,YAAW,UAAU;AACrC,QAAA,MAAY,KAAK,QAAQ;AAEzB,SAAO,gBAAgB,eAAe;AACpC,SAAA,GAAS,QAAQ,GAAG,KAAK,UAAU,KAAK,KAAK,GAAG,MAAA,MAAY,MAAM;AAClE,SAAA,aAAmB,OAAO,GAAG;AAC7B,YAAS;IACT;AAEF,SAAO;;CAGT,eAAyB;AACvB,SAAO,CAAC,GAAG,MAAA,aAAmB,QAAQ,CAAC;;CAGzC,MAAM,kBAAkB;AACtB,QAAM,QAAQ,IAAI,MAAA,MAAY;;CAGhC,gBAAgB,KAAc,MAAc;AAE1C,QAAA,OAAa,IAAI,MAAM,KAAK,eAAe,MAAM,KAAA,EAAU;;CAG7D,QACE,MACA,KACA,KACA,MACA,MACA,QACA;AAEA,MAAI,QAAQ;AACV,SAAA,WAAiB,OAAO,OAAO;AAC/B,SAAA,IAAU,OAAO,OAAO;;EAG1B,MAAM,MAAM,QAAQ,OAAO,QAAQ;AAEnC,MAAI,SAAS,cAAc;AAMzB,OAAI,SAAS,MAAM,MAAA,eAAqB,KAAK,MAAA,WAAiB,OAAO,GACnE,QAAA;GAEF,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA;AACN,SAAA,GAAS,OAAO,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAAI,OAAO,GAAG;AACzE,UAAO,MAAA,KAAW,KAAK;;EAGzB,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA,MAAA,aAAmB,KAAK,SAAA,KACtB,SACA;AACR,QAAA,GAAS,OACP,MACI,GAAG,KAAK,IAAI,IAAI,iBAAiB,IAAI,KACrC,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAC/C,OAAO,GACR;AAGD,MAAI,MAAA,WAAiB,SAAS,GAAG;AAC/B,SAAA,GAAS,OACP,MAAA,aACI,oCACE,KAAK,KAAK,GAAG,MAAA,WACd,QACD,iCACL;AACD,UAAO,MAAA,KAAW,EAAE;;AAGtB,MAAI,MAAA,eAAqB,EAGvB,QAAO,MAAA,KAAW,QAAQ,GAAG;;CAMjC,MAAM,SAA+B,QAAwB;AAC3D,OAAK,MAAM,UAAU,QACnB,KAAI;AACF,UAAO,KAAK,OAAO;WACZ,GAAG;AACV,SAAA,GAAS,QAAQ,EAAE;;;;;;;;;;;AAc3B,eAAsB,eACpB,IACA,QACA,GAAG,UACY;AACf,KAAI,SAAS,WAAW,EACtB;AAEF,MAAK,MAAM,UAAU,CAAC,GAAG,mBAAmB,GAAG,kBAAkB,CAC/D,QAAO,KAAK,cAAc;EACxB,MAAM,mBAAmB;AAEzB,WAAS,QAAQ,OAAM,QAAO;AAC5B,OAAI,iBAAiB,SAAS,OAAO,IAAI,IAAI,OAAO;AAClD,OAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,GAAG;AACnE,UAAM,IAAI,OAAO;;AAEnB,MAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,GAAG;AACnE,SAAM,IAAI,MAAM;IAChB;GACF;AAGJ,KAAI;EAEF,MAAM,MAAM,MAAM,QAAQ,KACxB,SAAS,KAAI,QAAO,IAAI,KAAK,CAAC,WAAW,IAAI,CAAC,CAC/C;AACD,KAAG,OAAO,GAAG,IAAI,YAAY,KAAK,IAAI,IAAI,GAAG,WAAW;UACjD,GAAG;AACV,KAAG,QAAQ,oBAAoB,EAAE;AACjC,QAAM;;;AAIV,eAAsB,UAAU,KAA0B;AACxD,KAAI;AACF,QAAM,KAAK;AAEX,UAAQ,KAAK,OAAO,IAAI,mBAAmB;AAC3C,UAAQ,KAAK,EAAE;UACR,GAAG;AAEV,UAAQ,MAAM,OAAO,IAAI,sBAAsB,EAAE;AACjD,UAAQ,KAAK,GAAG;;;AAIpB,IAAM,2BAA2B;;;;;;;;;;;;AAajC,IAAa,mBAAb,MAA8B;CAC5B;CAEA;CACA;CACA;CACA,iBAAiB;CAEjB,YAAY,IAAgB,eAAe,0BAA0B;AACnE,QAAA,KAAW;AACX,QAAA,eAAqB;;CAGvB,YAAY,YAAiC;AAC3C,QAAA,gBAAsB,KAAK,KAAK;AAChC,MAAI,MAAA,uBAA6B,KAAA,GAAW;AAC1C,SAAA,GAAS,OACP,iCACE,MAAA,eAAqB,IACtB,mBACD,WACD;AAGD,SAAA,qBAA2B,YACzB,MAAA,mBACA,MAAA,eAAqB,EACtB;;;CAIL,2BAA2B;AAsBzB,QAAA,sBAA4B,mBAAmB;AAC7C,SAAA,sBAA4B,KAAA;GAC5B,MAAM,yBAAyB,KAAK,KAAK,GAAG,MAAA;AAC5C,OAAI,0BAA0B,MAAA,cAAoB;AAChD,UAAA,GAAS,OACP,2BACE,yBAAyB,IAC1B,yBACF;AACD,YAAQ,KAAK,QAAQ,KAAK,kBAAkB,GAAG;;IAEjD;;CAGJ,OAAO;AACL,eAAa,MAAA,mBAAyB;AACtC,MAAI,MAAA,oBACF,gBAAe,MAAA,oBAA0B"}
+{"version":3,"file":"life-cycle.js","names":["#lc","#userFacing","#all","#exitImpl","#start","#ready","#startDrain","#kill","#exit","#runningState","#drainStart","#onExit","#initializing","#nextID","#stopInterval","#lastHeartbeat","#checkIntervalTimer","#checkStopInterval","#checkImmediateTimer"],"sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"sourcesContent":["import type {IncomingHttpHeaders} from 'node:http';\nimport {pid} from 'node:process';\nimport type {EventEmitter} from 'stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {\n  singleProcessMode,\n  type Subprocess,\n  type Worker,\n} from '../types/processes.ts';\nimport {RunningState} from './running-state.ts';\nimport type {SingletonService} from './service.ts';\n\n/**\n * * `user-facing` workers serve external requests and are the first to\n *   receive a `SIGTERM` or `SIGINT` signal for graceful shutdown.\n *\n * * `supporting` workers support `user-facing` workers and are sent\n *   the `SIGTERM` signal only after all `user-facing` workers have\n *   exited.\n *\n * For other kill signals, such as `SIGQUIT` and `SIGABRT`, all workers\n * are stopped without draining. `SIGQUIT` is used to represent an\n * intentional shutdown (for which draining is not beneficial), whereas\n * `SIGABRT` is used for unexpected process exits.\n */\nexport type WorkerType = 'user-facing' | 'supporting';\n\nexport const GRACEFUL_SHUTDOWN = ['SIGTERM', 'SIGINT'] as const;\nexport const FORCEFUL_SHUTDOWN = ['SIGQUIT', 'SIGABRT'] as const;\n\ntype GracefulShutdownSignal = (typeof GRACEFUL_SHUTDOWN)[number];\n\n// An internal error code used to indicate that a message has already been\n// logged at level ERRROR. When a process exits with this error code, the\n// parent process logs the exit at level WARN instead of ERROR.\nexport const UNHANDLED_EXCEPTION_ERROR_CODE = 13;\n\n// An internal error code used to indicate that the server should exit\n// without draining (e.g. due to a supporting worker get a signal to shut\n// down), but the exit is otherwise intentional.\nexport const INTENTIONAL_SHUTDOWN_ERROR_CODE = 14;\n\n/**\n * Handles readiness, termination signals, and coordination of graceful\n * shutdown.\n */\nexport class ProcessManager {\n  readonly #lc: LogContext;\n  readonly #userFacing = new Set<Subprocess>();\n  readonly #all = new Set<Subprocess>();\n  readonly #exitImpl: (code: number) => never;\n  readonly #start = Date.now();\n  readonly #ready: Promise<void>[] = [];\n\n  #runningState = new RunningState('process-manager');\n  #drainStart = 0;\n\n  constructor(lc: LogContext, proc: EventEmitter) {\n    this.#lc = lc.withContext('component', 'process-manager');\n\n    // Propagate `SIGTERM` and `SIGINT` to all user-facing workers,\n    // initiating a graceful shutdown. The parent process will\n    // exit once all user-facing workers have exited ...\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.on(signal, () => this.#startDrain(signal));\n    }\n\n    // ... which will result in sending `SIGTERM` to the remaining workers.\n    proc.on('exit', code =>\n      this.#kill(\n        this.#all,\n        code === 0\n          ? 'SIGTERM' // graceful, drained shutdown\n          : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n            ? 'SIGQUIT' // intentional abort without drain\n            : 'SIGABRT', // unintentional shutdown, alertable error\n      ),\n    );\n\n    // For other (catchable) kill signals, exit with a non-zero error code\n    // to send a `SIGQUIT` (intentional shutdown) or `SIGABRT` (unexpected\n    // shutdown) to all workers. For these signals, workers are stopped\n    // immediately without draining, since there is no merit to slowly draining\n    // when supporting workers have stopped.\n    //\n    // The logic for handling these signals is in `runUntilKilled()`.\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      proc.on(signal, () =>\n        this.#exit(signal === 'SIGQUIT' ? INTENTIONAL_SHUTDOWN_ERROR_CODE : -1),\n      );\n    }\n\n    this.#exitImpl = (code: number) => {\n      if (singleProcessMode()) {\n        return proc.emit('exit', code) as never; // For unit / integration tests.\n      }\n      process.exit(code);\n    };\n  }\n\n  done() {\n    return this.#runningState.stopped();\n  }\n\n  #exit(code: number) {\n    this.#lc.info?.('exiting with code', code);\n    this.#runningState.stop(this.#lc);\n    void this.#lc.flush().finally(() => this.#exitImpl(code));\n  }\n\n  #startDrain(signal: GracefulShutdownSignal) {\n    if (this.#all.size === 0) {\n      // Shutdown if a signal is received before any subprocesses are added.\n      this.#lc.info?.(`exiting on ${signal}`);\n      this.#exit(0);\n    }\n    this.#lc.info?.(`initiating drain (${signal})`);\n    this.#drainStart = Date.now();\n    if (this.#userFacing.size) {\n      this.#kill(this.#userFacing, signal);\n    } else {\n      this.#kill(this.#all, signal);\n    }\n  }\n\n  addSubprocess(proc: Subprocess, type: WorkerType, name: string) {\n    if (type === 'user-facing') {\n      this.#userFacing.add(proc);\n    }\n    this.#all.add(proc);\n\n    let isOpen = true;\n    proc.on('close', (code, signal) => {\n      isOpen = false;\n      this.#onExit(code, signal, null, type, name, proc);\n    });\n\n    // As per https://nodejs.org/api/child_process.html#event-error\n    // 'error' events can happen when sending a message to a child process\n    // fails. This is not really an error when the server is shutting down,\n    // so log any post-close errors at 'warn'.\n    proc.on('error', err =>\n      this.#lc[!isOpen || this.#drainStart > 0 ? 'warn' : 'error']?.(\n        `error from ${name} ${proc.pid}`,\n        err,\n      ),\n    );\n  }\n\n  readonly #initializing = new Map<number, string>();\n  #nextID = 0;\n\n  addWorker(worker: Worker, type: WorkerType, name: string): Worker {\n    this.addSubprocess(worker, type, name);\n\n    const id = ++this.#nextID;\n    this.#initializing.set(id, name);\n    const {promise, resolve} = resolver();\n    this.#ready.push(promise);\n\n    worker.onceMessageType('ready', () => {\n      this.#lc.debug?.(`${name} ready (${Date.now() - this.#start} ms)`);\n      this.#initializing.delete(id);\n      resolve();\n    });\n\n    return worker;\n  }\n\n  initializing(): string[] {\n    return [...this.#initializing.values()];\n  }\n\n  async allWorkersReady() {\n    await Promise.all(this.#ready);\n  }\n\n  logErrorAndExit(err: unknown, name: string) {\n    // only accessible by the main (i.e. user-facing) process.\n    this.#onExit(-1, null, err, 'user-facing', name, undefined);\n  }\n\n  #onExit(\n    code: number,\n    sig: NodeJS.Signals | null,\n    err: unknown,\n    type: WorkerType,\n    name: string,\n    worker: Subprocess | undefined,\n  ) {\n    // Remove the worker from maps to avoid attempting to send more signals to it.\n    if (worker) {\n      this.#userFacing.delete(worker);\n      this.#all.delete(worker);\n    }\n\n    const pid = worker?.pid ?? process.pid;\n\n    if (type === 'supporting') {\n      // Supporting workers like the replication-manager shut down without a\n      // drain signal when receiving protocol-specific instructions (like auto\n      // reset). In this case, a special error code is used to signal that the\n      // server should be shut down without draining, but it is otherwise not\n      // considered an unexpected/alertable error.\n      if (code === 0 && (this.#drainStart === 0 || this.#userFacing.size > 0)) {\n        code = INTENTIONAL_SHUTDOWN_ERROR_CODE;\n      }\n      const log =\n        code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n          ? 'info'\n          : 'warn';\n      this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n      return this.#exit(code);\n    }\n\n    const log =\n      code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n        ? 'info'\n        : this.#drainStart > 0 || code === UNHANDLED_EXCEPTION_ERROR_CODE\n          ? 'warn'\n          : 'error';\n    this.#lc[log]?.(\n      sig\n        ? `${name} (${pid}) killed with (${sig})`\n        : `${name} (${pid}) exited with code (${code})`,\n      err ?? '',\n    );\n\n    // user-facing workers exited or finished draining.\n    if (this.#userFacing.size === 0) {\n      this.#lc.info?.(\n        this.#drainStart\n          ? `all user-facing workers drained (${\n              Date.now() - this.#drainStart\n            } ms)`\n          : `all user-facing workers exited`,\n      );\n      return this.#exit(0);\n    }\n\n    if (this.#drainStart === 0) {\n      // If a user-facing worker exits without receiving a drain signal,\n      // shutdown the server.\n      return this.#exit(code || -1);\n    }\n\n    return undefined;\n  }\n\n  #kill(workers: Iterable<Subprocess>, signal: NodeJS.Signals) {\n    for (const worker of workers) {\n      try {\n        worker.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\n/**\n * Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with\n * an optional {@link SingletonService.drain drain()}, or stopping them\n * without draining for `SIGQUIT`.\n *\n * @returns a Promise that resolves/rejects when any of the services stops/throws.\n */\n\nexport async function runUntilKilled(\n  lc: LogContext,\n  parent: EventEmitter,\n  ...services: SingletonService[]\n): Promise<void> {\n  if (services.length === 0) {\n    return;\n  }\n  for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) {\n    parent.once(signal, () => {\n      const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN as readonly NodeJS.Signals[];\n\n      services.forEach(async svc => {\n        if (GRACEFUL_SIGNALS.includes(signal) && svc.drain) {\n          lc.info?.(`draining ${svc.constructor.name} ${svc.id} (${signal})`);\n          await svc.drain();\n        }\n        lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${signal})`);\n        await svc.stop();\n      });\n    });\n  }\n\n  try {\n    // Run all services and resolve when any of them stops.\n    const svc = await Promise.race(\n      services.map(svc => svc.run().then(() => svc)),\n    );\n    lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);\n  } catch (e) {\n    lc.error?.(`exiting on error`, e);\n    throw e;\n  }\n}\n\nexport async function exitAfter(run: () => Promise<void>) {\n  try {\n    await run();\n    // oxlint-disable-next-line no-console\n    console.info(`pid ${pid} exiting normally`);\n    process.exit(0);\n  } catch (e) {\n    // oxlint-disable-next-line no-console\n    console.error(`pid ${pid} exiting with error`, e);\n    process.exit(-1);\n  }\n}\n\n/**\n * The HeartbeatMonitor monitors the cadence heartbeats (e.g. \"/keepalive\"\n * health checks made to HttpServices) that signal that the server\n * should continue processing requests. When a configurable `stopInterval`\n * elapses without receiving these heartbeats, the monitor initiates a\n * graceful shutdown of the server. This works with common load balancing\n * frameworks such as AWS Elastic Load Balancing.\n *\n * The HeartbeatMonitor is **opt-in** in that it only kicks in after it\n * starts receiving keepalives.\n */\nexport class HeartbeatMonitor {\n  readonly #stopInterval: number;\n\n  #lc: LogContext;\n  #checkIntervalTimer: NodeJS.Timeout | undefined;\n  #checkImmediateTimer: NodeJS.Immediate | undefined;\n  #lastHeartbeat = 0;\n\n  constructor(lc: LogContext, stopInterval: number) {\n    this.#lc = lc;\n    this.#stopInterval = stopInterval;\n  }\n\n  onHeartbeat(reqHeaders: IncomingHttpHeaders) {\n    this.#lastHeartbeat = Date.now();\n    if (this.#checkIntervalTimer === undefined) {\n      this.#lc.info?.(\n        `starting heartbeat monitor at ${\n          this.#stopInterval / 1000\n        } second interval`,\n        reqHeaders,\n      );\n      // e.g. check every 5 seconds to see if it's been over 20 seconds\n      //      since the last heartbeat.\n      this.#checkIntervalTimer = setInterval(\n        this.#checkStopInterval,\n        this.#stopInterval / 4,\n      );\n    }\n  }\n\n  #checkStopInterval = () => {\n    // In the Node.js event loop, timers like setInterval and setTimeout\n    // run *before* I/O events coming from network sockets or file reads/writes.\n    // When this process gets starved of CPU resources for long periods of time,\n    // for example when other processes are monopolizing all available cores,\n    // pathological behavior can emerge:\n    // - keepalive network request comes in, but is queued in Node internals waiting\n    //   for time on the event loop\n    // - CPU is starved/monopolized by other processes for longer than the time\n    //   configured via this.#stopInterval\n    // - When CPU becomes available and the event loop wakes up, this stop interval\n    //   check is run *before* the keepalive request is processed. The value of\n    //   this.#lastHeartbeat is now very stale, and erroneously triggers a shutdown\n    //   even though keepalive requests were about to be processed and update\n    //   this.#lastHeartbeat. Downtime ensues.\n    //\n    // To avoid this, we push the check out to a phase of the event loop *after*\n    // I/O events are processed, using setImmediate():\n    // https://nodejs.org/en/learn/asynchronous-work/event-loop-timers-and-nexttick#setimmediate-vs-settimeout\n    //\n    // This ensures we see a value for this.#lastHeartbeat that reflects\n    // any keepalive requests that came in during the current event loop turn.\n    this.#checkImmediateTimer = setImmediate(() => {\n      this.#checkImmediateTimer = undefined;\n      const timeSinceLastHeartbeat = Date.now() - this.#lastHeartbeat;\n      if (timeSinceLastHeartbeat >= this.#stopInterval) {\n        this.#lc.info?.(\n          `last heartbeat received ${\n            timeSinceLastHeartbeat / 1000\n          } seconds ago. draining.`,\n        );\n        process.kill(process.pid, GRACEFUL_SHUTDOWN[0]);\n      }\n    });\n  };\n\n  stop() {\n    clearTimeout(this.#checkIntervalTimer);\n    if (this.#checkImmediateTimer) {\n      clearImmediate(this.#checkImmediateTimer);\n    }\n  }\n}\n"],"mappings":";;;;;AA4BA,IAAa,oBAAoB,CAAC,WAAW,SAAS;AACtD,IAAa,oBAAoB,CAAC,WAAW,UAAU;;;;;AAkBvD,IAAa,iBAAb,MAA4B;CAC1B;CACA,8BAAuB,IAAI,KAAiB;CAC5C,uBAAgB,IAAI,KAAiB;CACrC;CACA,SAAkB,KAAK,KAAK;CAC5B,SAAmC,EAAE;CAErC,gBAAgB,IAAI,aAAa,kBAAkB;CACnD,cAAc;CAEd,YAAY,IAAgB,MAAoB;AAC9C,QAAA,KAAW,GAAG,YAAY,aAAa,kBAAkB;AAKzD,OAAK,MAAM,UAAU,kBACnB,MAAK,GAAG,cAAc,MAAA,WAAiB,OAAO,CAAC;AAIjD,OAAK,GAAG,SAAQ,SACd,MAAA,KACE,MAAA,KACA,SAAS,IACL,YACA,SAAA,KACE,YACA,UACP,CACF;AASD,OAAK,MAAM,UAAU,kBACnB,MAAK,GAAG,cACN,MAAA,KAAW,WAAW,YAAA,KAA8C,GAAG,CACxE;AAGH,QAAA,YAAkB,SAAiB;AACjC,OAAI,mBAAmB,CACrB,QAAO,KAAK,KAAK,QAAQ,KAAK;AAEhC,WAAQ,KAAK,KAAK;;;CAItB,OAAO;AACL,SAAO,MAAA,aAAmB,SAAS;;CAGrC,MAAM,MAAc;AAClB,QAAA,GAAS,OAAO,qBAAqB,KAAK;AAC1C,QAAA,aAAmB,KAAK,MAAA,GAAS;AAC5B,QAAA,GAAS,OAAO,CAAC,cAAc,MAAA,SAAe,KAAK,CAAC;;CAG3D,YAAY,QAAgC;AAC1C,MAAI,MAAA,IAAU,SAAS,GAAG;AAExB,SAAA,GAAS,OAAO,cAAc,SAAS;AACvC,SAAA,KAAW,EAAE;;AAEf,QAAA,GAAS,OAAO,qBAAqB,OAAO,GAAG;AAC/C,QAAA,aAAmB,KAAK,KAAK;AAC7B,MAAI,MAAA,WAAiB,KACnB,OAAA,KAAW,MAAA,YAAkB,OAAO;MAEpC,OAAA,KAAW,MAAA,KAAW,OAAO;;CAIjC,cAAc,MAAkB,MAAkB,MAAc;AAC9D,MAAI,SAAS,cACX,OAAA,WAAiB,IAAI,KAAK;AAE5B,QAAA,IAAU,IAAI,KAAK;EAEnB,IAAI,SAAS;AACb,OAAK,GAAG,UAAU,MAAM,WAAW;AACjC,YAAS;AACT,SAAA,OAAa,MAAM,QAAQ,MAAM,MAAM,MAAM,KAAK;IAClD;AAMF,OAAK,GAAG,UAAS,QACf,MAAA,GAAS,CAAC,UAAU,MAAA,aAAmB,IAAI,SAAS,WAClD,cAAc,KAAK,GAAG,KAAK,OAC3B,IACD,CACF;;CAGH,gCAAyB,IAAI,KAAqB;CAClD,UAAU;CAEV,UAAU,QAAgB,MAAkB,MAAsB;AAChE,OAAK,cAAc,QAAQ,MAAM,KAAK;EAEtC,MAAM,KAAK,EAAE,MAAA;AACb,QAAA,aAAmB,IAAI,IAAI,KAAK;EAChC,MAAM,EAAC,SAAS,YAAW,UAAU;AACrC,QAAA,MAAY,KAAK,QAAQ;AAEzB,SAAO,gBAAgB,eAAe;AACpC,SAAA,GAAS,QAAQ,GAAG,KAAK,UAAU,KAAK,KAAK,GAAG,MAAA,MAAY,MAAM;AAClE,SAAA,aAAmB,OAAO,GAAG;AAC7B,YAAS;IACT;AAEF,SAAO;;CAGT,eAAyB;AACvB,SAAO,CAAC,GAAG,MAAA,aAAmB,QAAQ,CAAC;;CAGzC,MAAM,kBAAkB;AACtB,QAAM,QAAQ,IAAI,MAAA,MAAY;;CAGhC,gBAAgB,KAAc,MAAc;AAE1C,QAAA,OAAa,IAAI,MAAM,KAAK,eAAe,MAAM,KAAA,EAAU;;CAG7D,QACE,MACA,KACA,KACA,MACA,MACA,QACA;AAEA,MAAI,QAAQ;AACV,SAAA,WAAiB,OAAO,OAAO;AAC/B,SAAA,IAAU,OAAO,OAAO;;EAG1B,MAAM,MAAM,QAAQ,OAAO,QAAQ;AAEnC,MAAI,SAAS,cAAc;AAMzB,OAAI,SAAS,MAAM,MAAA,eAAqB,KAAK,MAAA,WAAiB,OAAO,GACnE,QAAA;GAEF,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA;AACN,SAAA,GAAS,OAAO,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAAI,OAAO,GAAG;AACzE,UAAO,MAAA,KAAW,KAAK;;EAGzB,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA,MAAA,aAAmB,KAAK,SAAA,KACtB,SACA;AACR,QAAA,GAAS,OACP,MACI,GAAG,KAAK,IAAI,IAAI,iBAAiB,IAAI,KACrC,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAC/C,OAAO,GACR;AAGD,MAAI,MAAA,WAAiB,SAAS,GAAG;AAC/B,SAAA,GAAS,OACP,MAAA,aACI,oCACE,KAAK,KAAK,GAAG,MAAA,WACd,QACD,iCACL;AACD,UAAO,MAAA,KAAW,EAAE;;AAGtB,MAAI,MAAA,eAAqB,EAGvB,QAAO,MAAA,KAAW,QAAQ,GAAG;;CAMjC,MAAM,SAA+B,QAAwB;AAC3D,OAAK,MAAM,UAAU,QACnB,KAAI;AACF,UAAO,KAAK,OAAO;WACZ,GAAG;AACV,SAAA,GAAS,QAAQ,EAAE;;;;;;;;;;;AAc3B,eAAsB,eACpB,IACA,QACA,GAAG,UACY;AACf,KAAI,SAAS,WAAW,EACtB;AAEF,MAAK,MAAM,UAAU,CAAC,GAAG,mBAAmB,GAAG,kBAAkB,CAC/D,QAAO,KAAK,cAAc;EACxB,MAAM,mBAAmB;AAEzB,WAAS,QAAQ,OAAM,QAAO;AAC5B,OAAI,iBAAiB,SAAS,OAAO,IAAI,IAAI,OAAO;AAClD,OAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,GAAG;AACnE,UAAM,IAAI,OAAO;;AAEnB,MAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,GAAG;AACnE,SAAM,IAAI,MAAM;IAChB;GACF;AAGJ,KAAI;EAEF,MAAM,MAAM,MAAM,QAAQ,KACxB,SAAS,KAAI,QAAO,IAAI,KAAK,CAAC,WAAW,IAAI,CAAC,CAC/C;AACD,KAAG,OAAO,GAAG,IAAI,YAAY,KAAK,IAAI,IAAI,GAAG,WAAW;UACjD,GAAG;AACV,KAAG,QAAQ,oBAAoB,EAAE;AACjC,QAAM;;;AAIV,eAAsB,UAAU,KAA0B;AACxD,KAAI;AACF,QAAM,KAAK;AAEX,UAAQ,KAAK,OAAO,IAAI,mBAAmB;AAC3C,UAAQ,KAAK,EAAE;UACR,GAAG;AAEV,UAAQ,MAAM,OAAO,IAAI,sBAAsB,EAAE;AACjD,UAAQ,KAAK,GAAG;;;;;;;;;;;;;;AAepB,IAAa,mBAAb,MAA8B;CAC5B;CAEA;CACA;CACA;CACA,iBAAiB;CAEjB,YAAY,IAAgB,cAAsB;AAChD,QAAA,KAAW;AACX,QAAA,eAAqB;;CAGvB,YAAY,YAAiC;AAC3C,QAAA,gBAAsB,KAAK,KAAK;AAChC,MAAI,MAAA,uBAA6B,KAAA,GAAW;AAC1C,SAAA,GAAS,OACP,iCACE,MAAA,eAAqB,IACtB,mBACD,WACD;AAGD,SAAA,qBAA2B,YACzB,MAAA,mBACA,MAAA,eAAqB,EACtB;;;CAIL,2BAA2B;AAsBzB,QAAA,sBAA4B,mBAAmB;AAC7C,SAAA,sBAA4B,KAAA;GAC5B,MAAM,yBAAyB,KAAK,KAAK,GAAG,MAAA;AAC5C,OAAI,0BAA0B,MAAA,cAAoB;AAChD,UAAA,GAAS,OACP,2BACE,yBAAyB,IAC1B,yBACF;AACD,YAAQ,KAAK,QAAQ,KAAK,kBAAkB,GAAG;;IAEjD;;CAGJ,OAAO;AACL,eAAa,MAAA,mBAAyB;AACtC,MAAI,MAAA,oBACF,gBAAe,MAAA,oBAA0B"}

package/out/zero-cache/src/services/mutagen/mutagen.d.ts

@@ -2,7 +2,7 @@ import type { LogContext } from '@rocicorp/logger';
 import type { JWTPayload } from 'jose';
 import postgres from 'postgres';
 import { ErrorKind } from '../../../../zero-protocol/src/error-kind.ts';
-import { type CRUDMutation, type InsertOp, type Mutation, type UpsertOp } from '../../../../zero-protocol/src/push.ts';
+import { type CRUDMutation, type InsertOp, type Mutation, type UpsertOp } from '../../../../zero-protocol/src/mutation.ts';
 import type { DatabaseStorage } from '../../../../zqlite/src/database-storage.ts';
 import { type WriteAuthorizer } from '../../auth/write-authorizer.ts';
 import { type ZeroConfig } from '../../config/zero-config.ts';

package/out/zero-cache/src/services/mutagen/mutagen.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mutagen.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/mutagen/mutagen.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,MAAM,CAAC;AACrC,OAAO,QAAQ,MAAM,UAAU,CAAC;AAGhC,OAAO,EAAC,SAAS,EAAC,MAAM,6CAA6C,CAAC;AAWtE,OAAO,EACL,KAAK,YAAY,EAEjB,KAAK,QAAQ,EACb,KAAK,QAAQ,EAEb,KAAK,QAAQ,EACd,MAAM,uCAAuC,CAAC;AAC/C,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,4CAA4C,CAAC;AAEhF,OAAO,EAEL,KAAK,eAAe,EACrB,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAK5D,OAAO,KAAK,EAAC,UAAU,EAAE,mBAAmB,EAAC,MAAM,mBAAmB,CAAC;AACvE,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAEnE,OAAO,KAAK,EAAC,iBAAiB,EAAE,OAAO,EAAC,MAAM,eAAe,CAAC;AAK9D,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,SAAS,CAAC,cAAc,GAAG,SAAS,CAAC,mBAAmB;IAC9D,IAAI,EAAE,MAAM;CACb,CAAC;AAEF,MAAM,WAAW,OAAQ,SAAQ,iBAAiB;IAChD,eAAe,CACb,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,qBAAqB,EAAE,OAAO,GAC7B,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAAC;CACvC;AAED,qBAAa,cAAe,YAAW,OAAO,EAAE,OAAO;;IACrD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;gBAkBlB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,UAAU,EACpB,MAAM,EAAE,UAAU,EAClB,iBAAiB,EAAE,eAAe;IA0BpC,GAAG;IAKH,KAAK;IAQL,OAAO,IAAI,OAAO;IAIlB,eAAe,CACb,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,qBAAqB,UAAQ,GAC5B,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC;IAuBrC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAIpB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAStB;AAID,wBAAsB,eAAe,CACnC,EAAE,EAAE,UAAU,EACd,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,QAAQ,EAClB,eAAe,EAAE,eAAe,EAChC,SAAS,CAAC,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAAE,cAAc;AACtD,qBAAqB,UAAQ,GAC5B,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAqIpC;AAED,wBAAsB,qBAAqB,CACzC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,mBAAmB,EACvB,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,YAAY,EACtB,SAAS,EAAE,OAAO,EAClB,UAAU,EAAE,eAAe,iBA8D5B;AAED,wBAAgB,YAAY,CAC1B,EAAE,EAAE,QAAQ,CAAC,cAAc,EAC3B,MAAM,EAAE,QAAQ,GACf,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAEvC;AAED,wBAAgB,YAAY,CAC1B,EAAE,EAAE,QAAQ,CAAC,cAAc,EAC3B,GAAG,EAAE,QAAQ,GACZ,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAOvC"}
+{"version":3,"file":"mutagen.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/mutagen/mutagen.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,MAAM,CAAC;AACrC,OAAO,QAAQ,MAAM,UAAU,CAAC;AAGhC,OAAO,EAAC,SAAS,EAAC,MAAM,6CAA6C,CAAC;AAOtE,OAAO,EACL,KAAK,YAAY,EAEjB,KAAK,QAAQ,EACb,KAAK,QAAQ,EAEb,KAAK,QAAQ,EACd,MAAM,2CAA2C,CAAC;AAKnD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,4CAA4C,CAAC;AAEhF,OAAO,EAEL,KAAK,eAAe,EACrB,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAK5D,OAAO,KAAK,EAAC,UAAU,EAAE,mBAAmB,EAAC,MAAM,mBAAmB,CAAC;AACvE,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAEnE,OAAO,KAAK,EAAC,iBAAiB,EAAE,OAAO,EAAC,MAAM,eAAe,CAAC;AAK9D,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,SAAS,CAAC,cAAc,GAAG,SAAS,CAAC,mBAAmB;IAC9D,IAAI,EAAE,MAAM;CACb,CAAC;AAEF,MAAM,WAAW,OAAQ,SAAQ,iBAAiB;IAChD,eAAe,CACb,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,qBAAqB,EAAE,OAAO,GAC7B,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAAC;CACvC;AAED,qBAAa,cAAe,YAAW,OAAO,EAAE,OAAO;;IACrD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;gBAkBlB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,UAAU,EACpB,MAAM,EAAE,UAAU,EAClB,iBAAiB,EAAE,eAAe;IA0BpC,GAAG;IAKH,KAAK;IAQL,OAAO,IAAI,OAAO;IAIlB,eAAe,CACb,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,qBAAqB,UAAQ,GAC5B,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC;IAuBrC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAIpB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAStB;AAID,wBAAsB,eAAe,CACnC,EAAE,EAAE,UAAU,EACd,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,QAAQ,EAClB,eAAe,EAAE,eAAe,EAChC,SAAS,CAAC,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAAE,cAAc;AACtD,qBAAqB,UAAQ,GAC5B,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAqIpC;AAED,wBAAsB,qBAAqB,CACzC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,mBAAmB,EACvB,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,YAAY,EACtB,SAAS,EAAE,OAAO,EAClB,UAAU,EAAE,eAAe,iBA8D5B;AAED,wBAAgB,YAAY,CAC1B,EAAE,EAAE,QAAQ,CAAC,cAAc,EAC3B,MAAM,EAAE,QAAQ,GACf,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAEvC;AAED,wBAAgB,YAAY,CAC1B,EAAE,EAAE,QAAQ,CAAC,cAAc,EAC3B,GAAG,EAAE,QAAQ,GACZ,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAOvC"}

package/out/zero-cache/src/services/mutagen/mutagen.js

@@ -5,7 +5,7 @@ import { ZeroCache } from "../../../../zero-protocol/src/error-origin-enum.js";
 import { ProtocolError, isProtocolError } from "../../../../zero-protocol/src/error.js";
 import { CRUD } from "../../../../zero-protocol/src/mutation-type-enum.js";
 import { primaryKeyValueSchema } from "../../../../zero-protocol/src/primary-key.js";
-import "../../../../zero-protocol/src/push.js";
+import "../../../../zero-protocol/src/mutation.js";
 import { upstreamSchema } from "../../types/shards.js";
 import "../../config/zero-config.js";
 import { MutationAlreadyProcessedError } from "./error.js";

package/out/zero-cache/src/services/mutagen/mutagen.js.map

@@ -1 +1 @@
-{"version":3,"file":"mutagen.js","names":["#lc","#upstream","#shard","#stopped","#replica","#writeAuthorizer","#limiter","#crudMutations","#isStopped","#refCount"],"sources":["../../../../../../zero-cache/src/services/mutagen/mutagen.ts"],"sourcesContent":["import {PG_SERIALIZATION_FAILURE} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {JWTPayload} from 'jose';\nimport postgres from 'postgres';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {ErrorKind} from '../../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../../zero-protocol/src/error-origin.ts';\nimport {\n  isProtocolError,\n  ProtocolError,\n} from '../../../../zero-protocol/src/error.ts';\nimport * as MutationType from '../../../../zero-protocol/src/mutation-type-enum.ts';\nimport {\n  primaryKeyValueSchema,\n  type PrimaryKeyValue,\n} from '../../../../zero-protocol/src/primary-key.ts';\nimport {\n  type CRUDMutation,\n  type DeleteOp,\n  type InsertOp,\n  type Mutation,\n  type UpdateOp,\n  type UpsertOp,\n} from '../../../../zero-protocol/src/push.ts';\nimport type {DatabaseStorage} from '../../../../zqlite/src/database-storage.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {\n  WriteAuthorizerImpl,\n  type WriteAuthorizer,\n} from '../../auth/write-authorizer.ts';\nimport {type ZeroConfig} from '../../config/zero-config.ts';\nimport * as Mode from '../../db/mode-enum.ts';\nimport {runTx} from '../../db/run-transaction.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {recordMutation} from '../../server/anonymous-otel-start.ts';\nimport type {PostgresDB, PostgresTransaction} from '../../types/pg.ts';\nimport {upstreamSchema, type ShardID} from '../../types/shards.ts';\nimport {SlidingWindowLimiter} from '../limiter/sliding-window-limiter.ts';\nimport type {RefCountedService, Service} from '../service.ts';\nimport {MutationAlreadyProcessedError} from './error.ts';\n\n// An error encountered processing a mutation.\n// Returned back to application for display to user.\nexport type MutationError = [\n  kind: ErrorKind.MutationFailed | ErrorKind.MutationRateLimited,\n  desc: string,\n];\n\nexport interface Mutagen extends RefCountedService {\n  processMutation(\n    mutation: Mutation,\n    authData: JWTPayload | undefined,\n    customMutatorsEnabled: boolean,\n  ): Promise<MutationError | undefined>;\n}\n\nexport class MutagenService implements Mutagen, Service {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #upstream: PostgresDB;\n  readonly #shard: ShardID;\n  readonly #stopped = resolver();\n  readonly #replica: Database;\n  readonly #writeAuthorizer: WriteAuthorizerImpl;\n  readonly #limiter: SlidingWindowLimiter | undefined;\n  #refCount = 0;\n  #isStopped = false;\n\n  readonly #crudMutations = getOrCreateCounter(\n    'mutation',\n    'crud',\n    'Number of CRUD mutations processed',\n  );\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    clientGroupID: string,\n    upstream: PostgresDB,\n    config: ZeroConfig,\n    writeAuthzStorage: DatabaseStorage,\n  ) {\n    this.id = clientGroupID;\n    this.#lc = lc;\n    this.#upstream = upstream;\n    this.#shard = shard;\n    this.#replica = new Database(this.#lc, config.replica.file, {\n      fileMustExist: true,\n    });\n    this.#writeAuthorizer = new WriteAuthorizerImpl(\n      this.#lc,\n      config,\n      this.#replica,\n      shard.appID,\n      clientGroupID,\n      writeAuthzStorage,\n    );\n\n    if (config.perUserMutationLimit.max !== undefined) {\n      this.#limiter = new SlidingWindowLimiter(\n        config.perUserMutationLimit.windowMs,\n        config.perUserMutationLimit.max,\n      );\n    }\n  }\n\n  ref() {\n    assert(!this.#isStopped, 'MutagenService is already stopped');\n    ++this.#refCount;\n  }\n\n  unref() {\n    assert(!this.#isStopped, 'MutagenService is already stopped');\n    --this.#refCount;\n    if (this.#refCount <= 0) {\n      void this.stop();\n    }\n  }\n\n  hasRefs(): boolean {\n    return this.#refCount > 0;\n  }\n\n  processMutation(\n    mutation: Mutation,\n    authData: JWTPayload | undefined,\n    customMutatorsEnabled = false,\n  ): Promise<MutationError | undefined> {\n    if (this.#limiter?.canDo() === false) {\n      return Promise.resolve([\n        ErrorKind.MutationRateLimited,\n        'Rate limit exceeded',\n      ]);\n    }\n    this.#crudMutations.add(1, {\n      clientGroupID: this.id,\n    });\n    return processMutation(\n      this.#lc,\n      authData,\n      this.#upstream,\n      this.#shard,\n      this.id,\n      mutation,\n      this.#writeAuthorizer,\n      undefined,\n      customMutatorsEnabled,\n    );\n  }\n\n  run(): Promise<void> {\n    return this.#stopped.promise;\n  }\n\n  stop(): Promise<void> {\n    if (this.#isStopped) {\n      return this.#stopped.promise;\n    }\n    this.#writeAuthorizer.destroy();\n    this.#isStopped = true;\n    this.#stopped.resolve();\n    return this.#stopped.promise;\n  }\n}\n\nconst MAX_SERIALIZATION_ATTEMPTS = 10;\n\nexport async function processMutation(\n  lc: LogContext,\n  authData: JWTPayload | undefined,\n  db: PostgresDB,\n  shard: ShardID,\n  clientGroupID: string,\n  mutation: Mutation,\n  writeAuthorizer: WriteAuthorizer,\n  onTxStart?: () => void | Promise<void>, // for testing\n  customMutatorsEnabled = false,\n): Promise<MutationError | undefined> {\n  assert(\n    mutation.type === MutationType.CRUD,\n    'Only CRUD mutations are supported',\n  );\n  lc = lc.withContext('mutationID', mutation.id);\n  lc = lc.withContext('processMutation');\n  lc.debug?.('Process mutation start', mutation);\n\n  // Record mutation processing attempt for telemetry (regardless of success/failure)\n  recordMutation('crud');\n\n  let result: MutationError | undefined;\n\n  const start = Date.now();\n  try {\n    // Mutations can fail for a variety of reasons:\n    //\n    // - application error\n    // - network/db error\n    // - zero bug\n    //\n    // For application errors what we want is to re-run the mutation in\n    // \"error mode\", which skips the actual mutation and just updates the\n    // lastMutationID. Then return the error to the app.\n    //\n    // However, it's hard to tell the difference between application errors\n    // and the other types.\n    //\n    // A reasonable policy ends up being to just retry every mutation once\n    // in error mode. If the error mode mutation succeeds then we assume it\n    // was an application error and return the error to the app. Otherwise,\n    // we know it was something internal and we log it.\n    //\n    // This is not 100% correct - there are theoretical cases where we\n    // return an internal error to the app that shouldn't have been. But it\n    // would have to be a crazy coincidence: we'd have to have a network\n    // error on the first attempt that resolves by the second attempt.\n    //\n    // One might ask why not try/catch just the calls to the mutators and\n    // consider those application errors. That is actually what we do in\n    // Replicache:\n    //\n    // https://github.com/rocicorp/todo-row-versioning/blob/9a0a79dc2d2de32c4fac61b5d1634bd9a9e66b7c/server/src/push.ts#L131\n    //\n    // We don't do it here because:\n    //\n    // 1. It's still not perfect. It's hard to isolate SQL errors in\n    //    mutators due to app developer mistakes from SQL errors due to\n    //    Zero mistakes.\n    // 2. It's not possible to do this with the pg library we're using in\n    //    Zero anyway: https://github.com/porsager/postgres/issues/455.\n    //\n    // Personally I think this simple retry policy is nice.\n    let errorMode = false;\n    for (let i = 0; i < MAX_SERIALIZATION_ATTEMPTS; i++) {\n      try {\n        await runTx(\n          db,\n          async tx => {\n            // Simulates a concurrent request for testing. In production this is a noop.\n            const done = onTxStart?.();\n            try {\n              return await processMutationWithTx(\n                lc,\n                tx,\n                authData,\n                shard,\n                clientGroupID,\n                mutation,\n                errorMode,\n                writeAuthorizer,\n              );\n            } finally {\n              await done;\n            }\n          },\n          {mode: Mode.SERIALIZABLE},\n        );\n        if (errorMode) {\n          lc.debug?.('Ran mutation successfully in error mode');\n        }\n        break;\n      } catch (e) {\n        if (e instanceof MutationAlreadyProcessedError) {\n          lc.debug?.(e.message);\n          // Don't double-count already processed mutations, but they were counted above\n          return undefined;\n        }\n        if (\n          isProtocolError(e) &&\n          !errorMode &&\n          e.kind === ErrorKind.InvalidPush &&\n          customMutatorsEnabled &&\n          i < 2\n        ) {\n          // We're temporarily supporting custom mutators AND CRUD mutators at the same time.\n          // This can create a lot of OOO mutation errors since we do not know when the API server\n          // has applied a custom mutation before moving on to process CRUD mutations.\n          // The temporary workaround (since CRUD is being deprecated) is to retry the mutation\n          // after a small delay. Users are not expected to be running both CRUD and Custom mutators.\n          // They should migrate completely to custom mutators.\n          lc.info?.(\n            'Both CRUD and Custom mutators are being used at once. This is supported for now but IS NOT RECOMMENDED. Migrate completely to custom mutators.',\n            e,\n          );\n          await new Promise(resolve => setTimeout(resolve, 100));\n          continue;\n        }\n        if (isProtocolError(e) || errorMode) {\n          lc.error?.('Process mutation error', e);\n          throw e;\n        }\n        if (\n          e instanceof postgres.PostgresError &&\n          e.code === PG_SERIALIZATION_FAILURE\n        ) {\n          lc.info?.(`attempt ${i + 1}: ${String(e)}`, e);\n          continue; // Retry up to MAX_SERIALIZATION_ATTEMPTS.\n        }\n        result = [ErrorKind.MutationFailed, String(e)];\n        if (errorMode) {\n          break;\n        }\n        lc.error?.('Got error running mutation, re-running in error mode', e);\n        errorMode = true;\n        i--;\n      }\n    }\n  } finally {\n    lc.debug?.('Process mutation complete in', Date.now() - start);\n  }\n  return result;\n}\n\nexport async function processMutationWithTx(\n  lc: LogContext,\n  tx: PostgresTransaction,\n  authData: JWTPayload | undefined,\n  shard: ShardID,\n  clientGroupID: string,\n  mutation: CRUDMutation,\n  errorMode: boolean,\n  authorizer: WriteAuthorizer,\n) {\n  const tasks: (() => Promise<unknown>)[] = [];\n\n  async function execute(stmt: postgres.PendingQuery<postgres.Row[]>) {\n    try {\n      return await stmt.execute();\n    } finally {\n      const q = stmt as unknown as Query;\n      lc.debug?.(`${q.string}: ${JSON.stringify(q.parameters)}`);\n    }\n  }\n\n  authorizer.reloadPermissions();\n\n  if (!errorMode) {\n    const {ops} = mutation.args[0];\n\n    // Validate all table names before processing\n    authorizer.validateTableNames(ops);\n\n    const normalizedOps = authorizer.normalizeOps(ops);\n    const [canPre, canPost] = await Promise.all([\n      authorizer.canPreMutation(authData, normalizedOps),\n      authorizer.canPostMutation(authData, normalizedOps),\n    ]);\n    if (canPre && canPost) {\n      for (const op of ops) {\n        switch (op.op) {\n          case 'insert':\n            tasks.push(() => execute(getInsertSQL(tx, op)));\n            break;\n          case 'upsert':\n            tasks.push(() => execute(getUpsertSQL(tx, op)));\n            break;\n          case 'update':\n            tasks.push(() => execute(getUpdateSQL(tx, op)));\n            break;\n          case 'delete':\n            tasks.push(() => execute(getDeleteSQL(tx, op)));\n            break;\n          default:\n            unreachable(op);\n        }\n      }\n    }\n  }\n\n  // Confirm the mutation even though it may have been blocked by the authorizer.\n  // Authorizer blocking a mutation is not an error but the correct result of the mutation.\n  tasks.unshift(() =>\n    checkSchemaVersionAndIncrementLastMutationID(\n      tx,\n      shard,\n      clientGroupID,\n      mutation.clientID,\n      mutation.id,\n    ),\n  );\n\n  // Note: An error thrown from any Promise aborts the entire transaction.\n  await Promise.all(tasks.map(task => task()));\n}\n\nexport function getInsertSQL(\n  tx: postgres.TransactionSql,\n  create: InsertOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  return tx`INSERT INTO ${tx(create.tableName)} ${tx(create.value)}`;\n}\n\nexport function getUpsertSQL(\n  tx: postgres.TransactionSql,\n  set: UpsertOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  const {tableName, primaryKey, value} = set;\n  return tx`\n    INSERT INTO ${tx(tableName)} ${tx(value)}\n    ON CONFLICT (${tx(primaryKey)})\n    DO UPDATE SET ${tx(value)}\n  `;\n}\n\nfunction getUpdateSQL(\n  tx: postgres.TransactionSql,\n  update: UpdateOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  const table = update.tableName;\n  const {primaryKey, value} = update;\n  const id: Record<string, PrimaryKeyValue> = {};\n  for (const key of primaryKey) {\n    id[key] = v.parse(value[key], primaryKeyValueSchema);\n  }\n  return tx`UPDATE ${tx(table)} SET ${tx(value)} WHERE ${Object.entries(\n    id,\n  ).flatMap(([key, value], i) =>\n    i ? [tx`AND`, tx`${tx(key)} = ${value}`] : tx`${tx(key)} = ${value}`,\n  )}`;\n}\n\nfunction getDeleteSQL(\n  tx: postgres.TransactionSql,\n  deleteOp: DeleteOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  const {tableName, primaryKey, value} = deleteOp;\n\n  const conditions = [];\n  for (const key of primaryKey) {\n    if (conditions.length > 0) {\n      conditions.push(tx`AND`);\n    }\n    conditions.push(tx`${tx(key)} = ${value[key]}`);\n  }\n\n  return tx`DELETE FROM ${tx(tableName)} WHERE ${conditions}`;\n}\n\nasync function checkSchemaVersionAndIncrementLastMutationID(\n  tx: PostgresTransaction,\n  shard: ShardID,\n  clientGroupID: string,\n  clientID: string,\n  receivedMutationID: number,\n) {\n  const [{lastMutationID}] = await tx<{lastMutationID: bigint}[]>`\n    INSERT INTO ${tx(upstreamSchema(shard))}.clients \n      as current (\"clientGroupID\", \"clientID\", \"lastMutationID\")\n          VALUES (${clientGroupID}, ${clientID}, ${1})\n      ON CONFLICT (\"clientGroupID\", \"clientID\")\n      DO UPDATE SET \"lastMutationID\" = current.\"lastMutationID\" + 1\n      RETURNING \"lastMutationID\"\n  `;\n\n  // ABORT if the resulting lastMutationID is not equal to the receivedMutationID.\n  if (receivedMutationID < lastMutationID) {\n    throw new MutationAlreadyProcessedError(\n      clientID,\n      receivedMutationID,\n      lastMutationID,\n    );\n  } else if (receivedMutationID > lastMutationID) {\n    throw new ProtocolError({\n      kind: ErrorKind.InvalidPush,\n      message: `Push contains unexpected mutation id ${receivedMutationID} for client ${clientID}. Expected mutation id ${lastMutationID.toString()}.`,\n      origin: ErrorOrigin.ZeroCache,\n    });\n  }\n}\n\n// The slice of information from the Query object in Postgres.js that gets logged for debugging.\n// https://github.com/porsager/postgres/blob/f58cd4f3affd3e8ce8f53e42799672d86cd2c70b/src/connection.js#L219\ntype Query = {string: string; parameters: object[]};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AA0DA,IAAa,iBAAb,MAAwD;CACtD;CACA;CACA;CACA;CACA,WAAoB,UAAU;CAC9B;CACA;CACA;CACA,YAAY;CACZ,aAAa;CAEb,iBAA0B,mBACxB,YACA,QACA,qCACD;CAED,YACE,IACA,OACA,eACA,UACA,QACA,mBACA;AACA,OAAK,KAAK;AACV,QAAA,KAAW;AACX,QAAA,WAAiB;AACjB,QAAA,QAAc;AACd,QAAA,UAAgB,IAAI,SAAS,MAAA,IAAU,OAAO,QAAQ,MAAM,EAC1D,eAAe,MAChB,CAAC;AACF,QAAA,kBAAwB,IAAI,oBAC1B,MAAA,IACA,QACA,MAAA,SACA,MAAM,OACN,eACA,kBACD;AAED,MAAI,OAAO,qBAAqB,QAAQ,KAAA,EACtC,OAAA,UAAgB,IAAI,qBAClB,OAAO,qBAAqB,UAC5B,OAAO,qBAAqB,IAC7B;;CAIL,MAAM;AACJ,SAAO,CAAC,MAAA,WAAiB,oCAAoC;AAC7D,IAAE,MAAA;;CAGJ,QAAQ;AACN,SAAO,CAAC,MAAA,WAAiB,oCAAoC;AAC7D,IAAE,MAAA;AACF,MAAI,MAAA,YAAkB,EACf,MAAK,MAAM;;CAIpB,UAAmB;AACjB,SAAO,MAAA,WAAiB;;CAG1B,gBACE,UACA,UACA,wBAAwB,OACY;AACpC,MAAI,MAAA,SAAe,OAAO,KAAK,MAC7B,QAAO,QAAQ,QAAQ,CACrB,qBACA,sBACD,CAAC;AAEJ,QAAA,cAAoB,IAAI,GAAG,EACzB,eAAe,KAAK,IACrB,CAAC;AACF,SAAO,gBACL,MAAA,IACA,UACA,MAAA,UACA,MAAA,OACA,KAAK,IACL,UACA,MAAA,iBACA,KAAA,GACA,sBACD;;CAGH,MAAqB;AACnB,SAAO,MAAA,QAAc;;CAGvB,OAAsB;AACpB,MAAI,MAAA,UACF,QAAO,MAAA,QAAc;AAEvB,QAAA,gBAAsB,SAAS;AAC/B,QAAA,YAAkB;AAClB,QAAA,QAAc,SAAS;AACvB,SAAO,MAAA,QAAc;;;AAIzB,IAAM,6BAA6B;AAEnC,eAAsB,gBACpB,IACA,UACA,IACA,OACA,eACA,UACA,iBACA,WACA,wBAAwB,OACY;AACpC,QACE,SAAS,SAAS,MAClB,oCACD;AACD,MAAK,GAAG,YAAY,cAAc,SAAS,GAAG;AAC9C,MAAK,GAAG,YAAY,kBAAkB;AACtC,IAAG,QAAQ,0BAA0B,SAAS;AAG9C,gBAAe,OAAO;CAEtB,IAAI;CAEJ,MAAM,QAAQ,KAAK,KAAK;AACxB,KAAI;EAuCF,IAAI,YAAY;AAChB,OAAK,IAAI,IAAI,GAAG,IAAI,4BAA4B,IAC9C,KAAI;AACF,SAAM,MACJ,IACA,OAAM,OAAM;IAEV,MAAM,OAAO,aAAa;AAC1B,QAAI;AACF,YAAO,MAAM,sBACX,IACA,IACA,UACA,OACA,eACA,UACA,WACA,gBACD;cACO;AACR,WAAM;;MAGV,EAAC,MAAM,cAAkB,CAC1B;AACD,OAAI,UACF,IAAG,QAAQ,0CAA0C;AAEvD;WACO,GAAG;AACV,OAAI,aAAa,+BAA+B;AAC9C,OAAG,QAAQ,EAAE,QAAQ;AAErB;;AAEF,OACE,gBAAgB,EAAE,IAClB,CAAC,aACD,EAAE,SAAS,iBACX,yBACA,IAAI,GACJ;AAOA,OAAG,OACD,kJACA,EACD;AACD,UAAM,IAAI,SAAQ,YAAW,WAAW,SAAS,IAAI,CAAC;AACtD;;AAEF,OAAI,gBAAgB,EAAE,IAAI,WAAW;AACnC,OAAG,QAAQ,0BAA0B,EAAE;AACvC,UAAM;;AAER,OACE,aAAa,SAAS,iBACtB,EAAE,SAAS,0BACX;AACA,OAAG,OAAO,WAAW,IAAI,EAAE,IAAI,OAAO,EAAE,IAAI,EAAE;AAC9C;;AAEF,YAAS,CAAC,gBAA0B,OAAO,EAAE,CAAC;AAC9C,OAAI,UACF;AAEF,MAAG,QAAQ,wDAAwD,EAAE;AACrE,eAAY;AACZ;;WAGI;AACR,KAAG,QAAQ,gCAAgC,KAAK,KAAK,GAAG,MAAM;;AAEhE,QAAO;;AAGT,eAAsB,sBACpB,IACA,IACA,UACA,OACA,eACA,UACA,WACA,YACA;CACA,MAAM,QAAoC,EAAE;CAE5C,eAAe,QAAQ,MAA6C;AAClE,MAAI;AACF,UAAO,MAAM,KAAK,SAAS;YACnB;GACR,MAAM,IAAI;AACV,MAAG,QAAQ,GAAG,EAAE,OAAO,IAAI,KAAK,UAAU,EAAE,WAAW,GAAG;;;AAI9D,YAAW,mBAAmB;AAE9B,KAAI,CAAC,WAAW;EACd,MAAM,EAAC,QAAO,SAAS,KAAK;AAG5B,aAAW,mBAAmB,IAAI;EAElC,MAAM,gBAAgB,WAAW,aAAa,IAAI;EAClD,MAAM,CAAC,QAAQ,WAAW,MAAM,QAAQ,IAAI,CAC1C,WAAW,eAAe,UAAU,cAAc,EAClD,WAAW,gBAAgB,UAAU,cAAc,CACpD,CAAC;AACF,MAAI,UAAU,QACZ,MAAK,MAAM,MAAM,IACf,SAAQ,GAAG,IAAX;GACE,KAAK;AACH,UAAM,WAAW,QAAQ,aAAa,IAAI,GAAG,CAAC,CAAC;AAC/C;GACF,KAAK;AACH,UAAM,WAAW,QAAQ,aAAa,IAAI,GAAG,CAAC,CAAC;AAC/C;GACF,KAAK;AACH,UAAM,WAAW,QAAQ,aAAa,IAAI,GAAG,CAAC,CAAC;AAC/C;GACF,KAAK;AACH,UAAM,WAAW,QAAQ,aAAa,IAAI,GAAG,CAAC,CAAC;AAC/C;GACF,QACE,aAAY,GAAG;;;AAQzB,OAAM,cACJ,6CACE,IACA,OACA,eACA,SAAS,UACT,SAAS,GACV,CACF;AAGD,OAAM,QAAQ,IAAI,MAAM,KAAI,SAAQ,MAAM,CAAC,CAAC;;AAG9C,SAAgB,aACd,IACA,QACuC;AACvC,QAAO,EAAE,eAAe,GAAG,OAAO,UAAU,CAAC,GAAG,GAAG,OAAO,MAAM;;AAGlE,SAAgB,aACd,IACA,KACuC;CACvC,MAAM,EAAC,WAAW,YAAY,UAAS;AACvC,QAAO,EAAE;kBACO,GAAG,UAAU,CAAC,GAAG,GAAG,MAAM,CAAC;mBAC1B,GAAG,WAAW,CAAC;oBACd,GAAG,MAAM,CAAC;;;AAI9B,SAAS,aACP,IACA,QACuC;CACvC,MAAM,QAAQ,OAAO;CACrB,MAAM,EAAC,YAAY,UAAS;CAC5B,MAAM,KAAsC,EAAE;AAC9C,MAAK,MAAM,OAAO,WAChB,IAAG,OAAO,MAAQ,MAAM,MAAM,sBAAsB;AAEtD,QAAO,EAAE,UAAU,GAAG,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC,SAAS,OAAO,QAC5D,GACD,CAAC,SAAS,CAAC,KAAK,QAAQ,MACvB,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,GAAG,IAAI,CAAC,KAAK,QAAQ,GAAG,EAAE,GAAG,GAAG,IAAI,CAAC,KAAK,QAC9D;;AAGH,SAAS,aACP,IACA,UACuC;CACvC,MAAM,EAAC,WAAW,YAAY,UAAS;CAEvC,MAAM,aAAa,EAAE;AACrB,MAAK,MAAM,OAAO,YAAY;AAC5B,MAAI,WAAW,SAAS,EACtB,YAAW,KAAK,EAAE,MAAM;AAE1B,aAAW,KAAK,EAAE,GAAG,GAAG,IAAI,CAAC,KAAK,MAAM,OAAO;;AAGjD,QAAO,EAAE,eAAe,GAAG,UAAU,CAAC,SAAS;;AAGjD,eAAe,6CACb,IACA,OACA,eACA,UACA,oBACA;CACA,MAAM,CAAC,EAAC,oBAAmB,MAAM,EAA8B;kBAC/C,GAAG,eAAe,MAAM,CAAC,CAAC;;oBAExB,cAAc,IAAI,SAAS,IAAI,EAAE;;;;;AAOnD,KAAI,qBAAqB,eACvB,OAAM,IAAI,8BACR,UACA,oBACA,eACD;UACQ,qBAAqB,eAC9B,OAAM,IAAI,cAAc;EACtB,MAAM;EACN,SAAS,wCAAwC,mBAAmB,cAAc,SAAS,yBAAyB,eAAe,UAAU,CAAC;EAC9I,QAAQ;EACT,CAAC"}
+{"version":3,"file":"mutagen.js","names":["#lc","#upstream","#shard","#stopped","#replica","#writeAuthorizer","#limiter","#crudMutations","#isStopped","#refCount"],"sources":["../../../../../../zero-cache/src/services/mutagen/mutagen.ts"],"sourcesContent":["import {PG_SERIALIZATION_FAILURE} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {JWTPayload} from 'jose';\nimport postgres from 'postgres';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {ErrorKind} from '../../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../../zero-protocol/src/error-origin.ts';\nimport {\n  isProtocolError,\n  ProtocolError,\n} from '../../../../zero-protocol/src/error.ts';\nimport * as MutationType from '../../../../zero-protocol/src/mutation-type-enum.ts';\nimport {\n  type CRUDMutation,\n  type DeleteOp,\n  type InsertOp,\n  type Mutation,\n  type UpdateOp,\n  type UpsertOp,\n} from '../../../../zero-protocol/src/mutation.ts';\nimport {\n  primaryKeyValueSchema,\n  type PrimaryKeyValue,\n} from '../../../../zero-protocol/src/primary-key.ts';\nimport type {DatabaseStorage} from '../../../../zqlite/src/database-storage.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {\n  WriteAuthorizerImpl,\n  type WriteAuthorizer,\n} from '../../auth/write-authorizer.ts';\nimport {type ZeroConfig} from '../../config/zero-config.ts';\nimport * as Mode from '../../db/mode-enum.ts';\nimport {runTx} from '../../db/run-transaction.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {recordMutation} from '../../server/anonymous-otel-start.ts';\nimport type {PostgresDB, PostgresTransaction} from '../../types/pg.ts';\nimport {upstreamSchema, type ShardID} from '../../types/shards.ts';\nimport {SlidingWindowLimiter} from '../limiter/sliding-window-limiter.ts';\nimport type {RefCountedService, Service} from '../service.ts';\nimport {MutationAlreadyProcessedError} from './error.ts';\n\n// An error encountered processing a mutation.\n// Returned back to application for display to user.\nexport type MutationError = [\n  kind: ErrorKind.MutationFailed | ErrorKind.MutationRateLimited,\n  desc: string,\n];\n\nexport interface Mutagen extends RefCountedService {\n  processMutation(\n    mutation: Mutation,\n    authData: JWTPayload | undefined,\n    customMutatorsEnabled: boolean,\n  ): Promise<MutationError | undefined>;\n}\n\nexport class MutagenService implements Mutagen, Service {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #upstream: PostgresDB;\n  readonly #shard: ShardID;\n  readonly #stopped = resolver();\n  readonly #replica: Database;\n  readonly #writeAuthorizer: WriteAuthorizerImpl;\n  readonly #limiter: SlidingWindowLimiter | undefined;\n  #refCount = 0;\n  #isStopped = false;\n\n  readonly #crudMutations = getOrCreateCounter(\n    'mutation',\n    'crud',\n    'Number of CRUD mutations processed',\n  );\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    clientGroupID: string,\n    upstream: PostgresDB,\n    config: ZeroConfig,\n    writeAuthzStorage: DatabaseStorage,\n  ) {\n    this.id = clientGroupID;\n    this.#lc = lc;\n    this.#upstream = upstream;\n    this.#shard = shard;\n    this.#replica = new Database(this.#lc, config.replica.file, {\n      fileMustExist: true,\n    });\n    this.#writeAuthorizer = new WriteAuthorizerImpl(\n      this.#lc,\n      config,\n      this.#replica,\n      shard.appID,\n      clientGroupID,\n      writeAuthzStorage,\n    );\n\n    if (config.perUserMutationLimit.max !== undefined) {\n      this.#limiter = new SlidingWindowLimiter(\n        config.perUserMutationLimit.windowMs,\n        config.perUserMutationLimit.max,\n      );\n    }\n  }\n\n  ref() {\n    assert(!this.#isStopped, 'MutagenService is already stopped');\n    ++this.#refCount;\n  }\n\n  unref() {\n    assert(!this.#isStopped, 'MutagenService is already stopped');\n    --this.#refCount;\n    if (this.#refCount <= 0) {\n      void this.stop();\n    }\n  }\n\n  hasRefs(): boolean {\n    return this.#refCount > 0;\n  }\n\n  processMutation(\n    mutation: Mutation,\n    authData: JWTPayload | undefined,\n    customMutatorsEnabled = false,\n  ): Promise<MutationError | undefined> {\n    if (this.#limiter?.canDo() === false) {\n      return Promise.resolve([\n        ErrorKind.MutationRateLimited,\n        'Rate limit exceeded',\n      ]);\n    }\n    this.#crudMutations.add(1, {\n      clientGroupID: this.id,\n    });\n    return processMutation(\n      this.#lc,\n      authData,\n      this.#upstream,\n      this.#shard,\n      this.id,\n      mutation,\n      this.#writeAuthorizer,\n      undefined,\n      customMutatorsEnabled,\n    );\n  }\n\n  run(): Promise<void> {\n    return this.#stopped.promise;\n  }\n\n  stop(): Promise<void> {\n    if (this.#isStopped) {\n      return this.#stopped.promise;\n    }\n    this.#writeAuthorizer.destroy();\n    this.#isStopped = true;\n    this.#stopped.resolve();\n    return this.#stopped.promise;\n  }\n}\n\nconst MAX_SERIALIZATION_ATTEMPTS = 10;\n\nexport async function processMutation(\n  lc: LogContext,\n  authData: JWTPayload | undefined,\n  db: PostgresDB,\n  shard: ShardID,\n  clientGroupID: string,\n  mutation: Mutation,\n  writeAuthorizer: WriteAuthorizer,\n  onTxStart?: () => void | Promise<void>, // for testing\n  customMutatorsEnabled = false,\n): Promise<MutationError | undefined> {\n  assert(\n    mutation.type === MutationType.CRUD,\n    'Only CRUD mutations are supported',\n  );\n  lc = lc.withContext('mutationID', mutation.id);\n  lc = lc.withContext('processMutation');\n  lc.debug?.('Process mutation start', mutation);\n\n  // Record mutation processing attempt for telemetry (regardless of success/failure)\n  recordMutation('crud');\n\n  let result: MutationError | undefined;\n\n  const start = Date.now();\n  try {\n    // Mutations can fail for a variety of reasons:\n    //\n    // - application error\n    // - network/db error\n    // - zero bug\n    //\n    // For application errors what we want is to re-run the mutation in\n    // \"error mode\", which skips the actual mutation and just updates the\n    // lastMutationID. Then return the error to the app.\n    //\n    // However, it's hard to tell the difference between application errors\n    // and the other types.\n    //\n    // A reasonable policy ends up being to just retry every mutation once\n    // in error mode. If the error mode mutation succeeds then we assume it\n    // was an application error and return the error to the app. Otherwise,\n    // we know it was something internal and we log it.\n    //\n    // This is not 100% correct - there are theoretical cases where we\n    // return an internal error to the app that shouldn't have been. But it\n    // would have to be a crazy coincidence: we'd have to have a network\n    // error on the first attempt that resolves by the second attempt.\n    //\n    // One might ask why not try/catch just the calls to the mutators and\n    // consider those application errors. That is actually what we do in\n    // Replicache:\n    //\n    // https://github.com/rocicorp/todo-row-versioning/blob/9a0a79dc2d2de32c4fac61b5d1634bd9a9e66b7c/server/src/push.ts#L131\n    //\n    // We don't do it here because:\n    //\n    // 1. It's still not perfect. It's hard to isolate SQL errors in\n    //    mutators due to app developer mistakes from SQL errors due to\n    //    Zero mistakes.\n    // 2. It's not possible to do this with the pg library we're using in\n    //    Zero anyway: https://github.com/porsager/postgres/issues/455.\n    //\n    // Personally I think this simple retry policy is nice.\n    let errorMode = false;\n    for (let i = 0; i < MAX_SERIALIZATION_ATTEMPTS; i++) {\n      try {\n        await runTx(\n          db,\n          async tx => {\n            // Simulates a concurrent request for testing. In production this is a noop.\n            const done = onTxStart?.();\n            try {\n              return await processMutationWithTx(\n                lc,\n                tx,\n                authData,\n                shard,\n                clientGroupID,\n                mutation,\n                errorMode,\n                writeAuthorizer,\n              );\n            } finally {\n              await done;\n            }\n          },\n          {mode: Mode.SERIALIZABLE},\n        );\n        if (errorMode) {\n          lc.debug?.('Ran mutation successfully in error mode');\n        }\n        break;\n      } catch (e) {\n        if (e instanceof MutationAlreadyProcessedError) {\n          lc.debug?.(e.message);\n          // Don't double-count already processed mutations, but they were counted above\n          return undefined;\n        }\n        if (\n          isProtocolError(e) &&\n          !errorMode &&\n          e.kind === ErrorKind.InvalidPush &&\n          customMutatorsEnabled &&\n          i < 2\n        ) {\n          // We're temporarily supporting custom mutators AND CRUD mutators at the same time.\n          // This can create a lot of OOO mutation errors since we do not know when the API server\n          // has applied a custom mutation before moving on to process CRUD mutations.\n          // The temporary workaround (since CRUD is being deprecated) is to retry the mutation\n          // after a small delay. Users are not expected to be running both CRUD and Custom mutators.\n          // They should migrate completely to custom mutators.\n          lc.info?.(\n            'Both CRUD and Custom mutators are being used at once. This is supported for now but IS NOT RECOMMENDED. Migrate completely to custom mutators.',\n            e,\n          );\n          await new Promise(resolve => setTimeout(resolve, 100));\n          continue;\n        }\n        if (isProtocolError(e) || errorMode) {\n          lc.error?.('Process mutation error', e);\n          throw e;\n        }\n        if (\n          e instanceof postgres.PostgresError &&\n          e.code === PG_SERIALIZATION_FAILURE\n        ) {\n          lc.info?.(`attempt ${i + 1}: ${String(e)}`, e);\n          continue; // Retry up to MAX_SERIALIZATION_ATTEMPTS.\n        }\n        result = [ErrorKind.MutationFailed, String(e)];\n        if (errorMode) {\n          break;\n        }\n        lc.error?.('Got error running mutation, re-running in error mode', e);\n        errorMode = true;\n        i--;\n      }\n    }\n  } finally {\n    lc.debug?.('Process mutation complete in', Date.now() - start);\n  }\n  return result;\n}\n\nexport async function processMutationWithTx(\n  lc: LogContext,\n  tx: PostgresTransaction,\n  authData: JWTPayload | undefined,\n  shard: ShardID,\n  clientGroupID: string,\n  mutation: CRUDMutation,\n  errorMode: boolean,\n  authorizer: WriteAuthorizer,\n) {\n  const tasks: (() => Promise<unknown>)[] = [];\n\n  async function execute(stmt: postgres.PendingQuery<postgres.Row[]>) {\n    try {\n      return await stmt.execute();\n    } finally {\n      const q = stmt as unknown as Query;\n      lc.debug?.(`${q.string}: ${JSON.stringify(q.parameters)}`);\n    }\n  }\n\n  authorizer.reloadPermissions();\n\n  if (!errorMode) {\n    const {ops} = mutation.args[0];\n\n    // Validate all table names before processing\n    authorizer.validateTableNames(ops);\n\n    const normalizedOps = authorizer.normalizeOps(ops);\n    const [canPre, canPost] = await Promise.all([\n      authorizer.canPreMutation(authData, normalizedOps),\n      authorizer.canPostMutation(authData, normalizedOps),\n    ]);\n    if (canPre && canPost) {\n      for (const op of ops) {\n        switch (op.op) {\n          case 'insert':\n            tasks.push(() => execute(getInsertSQL(tx, op)));\n            break;\n          case 'upsert':\n            tasks.push(() => execute(getUpsertSQL(tx, op)));\n            break;\n          case 'update':\n            tasks.push(() => execute(getUpdateSQL(tx, op)));\n            break;\n          case 'delete':\n            tasks.push(() => execute(getDeleteSQL(tx, op)));\n            break;\n          default:\n            unreachable(op);\n        }\n      }\n    }\n  }\n\n  // Confirm the mutation even though it may have been blocked by the authorizer.\n  // Authorizer blocking a mutation is not an error but the correct result of the mutation.\n  tasks.unshift(() =>\n    checkSchemaVersionAndIncrementLastMutationID(\n      tx,\n      shard,\n      clientGroupID,\n      mutation.clientID,\n      mutation.id,\n    ),\n  );\n\n  // Note: An error thrown from any Promise aborts the entire transaction.\n  await Promise.all(tasks.map(task => task()));\n}\n\nexport function getInsertSQL(\n  tx: postgres.TransactionSql,\n  create: InsertOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  return tx`INSERT INTO ${tx(create.tableName)} ${tx(create.value)}`;\n}\n\nexport function getUpsertSQL(\n  tx: postgres.TransactionSql,\n  set: UpsertOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  const {tableName, primaryKey, value} = set;\n  return tx`\n    INSERT INTO ${tx(tableName)} ${tx(value)}\n    ON CONFLICT (${tx(primaryKey)})\n    DO UPDATE SET ${tx(value)}\n  `;\n}\n\nfunction getUpdateSQL(\n  tx: postgres.TransactionSql,\n  update: UpdateOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  const table = update.tableName;\n  const {primaryKey, value} = update;\n  const id: Record<string, PrimaryKeyValue> = {};\n  for (const key of primaryKey) {\n    id[key] = v.parse(value[key], primaryKeyValueSchema);\n  }\n  return tx`UPDATE ${tx(table)} SET ${tx(value)} WHERE ${Object.entries(\n    id,\n  ).flatMap(([key, value], i) =>\n    i ? [tx`AND`, tx`${tx(key)} = ${value}`] : tx`${tx(key)} = ${value}`,\n  )}`;\n}\n\nfunction getDeleteSQL(\n  tx: postgres.TransactionSql,\n  deleteOp: DeleteOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  const {tableName, primaryKey, value} = deleteOp;\n\n  const conditions = [];\n  for (const key of primaryKey) {\n    if (conditions.length > 0) {\n      conditions.push(tx`AND`);\n    }\n    conditions.push(tx`${tx(key)} = ${value[key]}`);\n  }\n\n  return tx`DELETE FROM ${tx(tableName)} WHERE ${conditions}`;\n}\n\nasync function checkSchemaVersionAndIncrementLastMutationID(\n  tx: PostgresTransaction,\n  shard: ShardID,\n  clientGroupID: string,\n  clientID: string,\n  receivedMutationID: number,\n) {\n  const [{lastMutationID}] = await tx<{lastMutationID: bigint}[]>`\n    INSERT INTO ${tx(upstreamSchema(shard))}.clients \n      as current (\"clientGroupID\", \"clientID\", \"lastMutationID\")\n          VALUES (${clientGroupID}, ${clientID}, ${1})\n      ON CONFLICT (\"clientGroupID\", \"clientID\")\n      DO UPDATE SET \"lastMutationID\" = current.\"lastMutationID\" + 1\n      RETURNING \"lastMutationID\"\n  `;\n\n  // ABORT if the resulting lastMutationID is not equal to the receivedMutationID.\n  if (receivedMutationID < lastMutationID) {\n    throw new MutationAlreadyProcessedError(\n      clientID,\n      receivedMutationID,\n      lastMutationID,\n    );\n  } else if (receivedMutationID > lastMutationID) {\n    throw new ProtocolError({\n      kind: ErrorKind.InvalidPush,\n      message: `Push contains unexpected mutation id ${receivedMutationID} for client ${clientID}. Expected mutation id ${lastMutationID.toString()}.`,\n      origin: ErrorOrigin.ZeroCache,\n    });\n  }\n}\n\n// The slice of information from the Query object in Postgres.js that gets logged for debugging.\n// https://github.com/porsager/postgres/blob/f58cd4f3affd3e8ce8f53e42799672d86cd2c70b/src/connection.js#L219\ntype Query = {string: string; parameters: object[]};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AA0DA,IAAa,iBAAb,MAAwD;CACtD;CACA;CACA;CACA;CACA,WAAoB,UAAU;CAC9B;CACA;CACA;CACA,YAAY;CACZ,aAAa;CAEb,iBAA0B,mBACxB,YACA,QACA,qCACD;CAED,YACE,IACA,OACA,eACA,UACA,QACA,mBACA;AACA,OAAK,KAAK;AACV,QAAA,KAAW;AACX,QAAA,WAAiB;AACjB,QAAA,QAAc;AACd,QAAA,UAAgB,IAAI,SAAS,MAAA,IAAU,OAAO,QAAQ,MAAM,EAC1D,eAAe,MAChB,CAAC;AACF,QAAA,kBAAwB,IAAI,oBAC1B,MAAA,IACA,QACA,MAAA,SACA,MAAM,OACN,eACA,kBACD;AAED,MAAI,OAAO,qBAAqB,QAAQ,KAAA,EACtC,OAAA,UAAgB,IAAI,qBAClB,OAAO,qBAAqB,UAC5B,OAAO,qBAAqB,IAC7B;;CAIL,MAAM;AACJ,SAAO,CAAC,MAAA,WAAiB,oCAAoC;AAC7D,IAAE,MAAA;;CAGJ,QAAQ;AACN,SAAO,CAAC,MAAA,WAAiB,oCAAoC;AAC7D,IAAE,MAAA;AACF,MAAI,MAAA,YAAkB,EACf,MAAK,MAAM;;CAIpB,UAAmB;AACjB,SAAO,MAAA,WAAiB;;CAG1B,gBACE,UACA,UACA,wBAAwB,OACY;AACpC,MAAI,MAAA,SAAe,OAAO,KAAK,MAC7B,QAAO,QAAQ,QAAQ,CACrB,qBACA,sBACD,CAAC;AAEJ,QAAA,cAAoB,IAAI,GAAG,EACzB,eAAe,KAAK,IACrB,CAAC;AACF,SAAO,gBACL,MAAA,IACA,UACA,MAAA,UACA,MAAA,OACA,KAAK,IACL,UACA,MAAA,iBACA,KAAA,GACA,sBACD;;CAGH,MAAqB;AACnB,SAAO,MAAA,QAAc;;CAGvB,OAAsB;AACpB,MAAI,MAAA,UACF,QAAO,MAAA,QAAc;AAEvB,QAAA,gBAAsB,SAAS;AAC/B,QAAA,YAAkB;AAClB,QAAA,QAAc,SAAS;AACvB,SAAO,MAAA,QAAc;;;AAIzB,IAAM,6BAA6B;AAEnC,eAAsB,gBACpB,IACA,UACA,IACA,OACA,eACA,UACA,iBACA,WACA,wBAAwB,OACY;AACpC,QACE,SAAS,SAAS,MAClB,oCACD;AACD,MAAK,GAAG,YAAY,cAAc,SAAS,GAAG;AAC9C,MAAK,GAAG,YAAY,kBAAkB;AACtC,IAAG,QAAQ,0BAA0B,SAAS;AAG9C,gBAAe,OAAO;CAEtB,IAAI;CAEJ,MAAM,QAAQ,KAAK,KAAK;AACxB,KAAI;EAuCF,IAAI,YAAY;AAChB,OAAK,IAAI,IAAI,GAAG,IAAI,4BAA4B,IAC9C,KAAI;AACF,SAAM,MACJ,IACA,OAAM,OAAM;IAEV,MAAM,OAAO,aAAa;AAC1B,QAAI;AACF,YAAO,MAAM,sBACX,IACA,IACA,UACA,OACA,eACA,UACA,WACA,gBACD;cACO;AACR,WAAM;;MAGV,EAAC,MAAM,cAAkB,CAC1B;AACD,OAAI,UACF,IAAG,QAAQ,0CAA0C;AAEvD;WACO,GAAG;AACV,OAAI,aAAa,+BAA+B;AAC9C,OAAG,QAAQ,EAAE,QAAQ;AAErB;;AAEF,OACE,gBAAgB,EAAE,IAClB,CAAC,aACD,EAAE,SAAS,iBACX,yBACA,IAAI,GACJ;AAOA,OAAG,OACD,kJACA,EACD;AACD,UAAM,IAAI,SAAQ,YAAW,WAAW,SAAS,IAAI,CAAC;AACtD;;AAEF,OAAI,gBAAgB,EAAE,IAAI,WAAW;AACnC,OAAG,QAAQ,0BAA0B,EAAE;AACvC,UAAM;;AAER,OACE,aAAa,SAAS,iBACtB,EAAE,SAAS,0BACX;AACA,OAAG,OAAO,WAAW,IAAI,EAAE,IAAI,OAAO,EAAE,IAAI,EAAE;AAC9C;;AAEF,YAAS,CAAC,gBAA0B,OAAO,EAAE,CAAC;AAC9C,OAAI,UACF;AAEF,MAAG,QAAQ,wDAAwD,EAAE;AACrE,eAAY;AACZ;;WAGI;AACR,KAAG,QAAQ,gCAAgC,KAAK,KAAK,GAAG,MAAM;;AAEhE,QAAO;;AAGT,eAAsB,sBACpB,IACA,IACA,UACA,OACA,eACA,UACA,WACA,YACA;CACA,MAAM,QAAoC,EAAE;CAE5C,eAAe,QAAQ,MAA6C;AAClE,MAAI;AACF,UAAO,MAAM,KAAK,SAAS;YACnB;GACR,MAAM,IAAI;AACV,MAAG,QAAQ,GAAG,EAAE,OAAO,IAAI,KAAK,UAAU,EAAE,WAAW,GAAG;;;AAI9D,YAAW,mBAAmB;AAE9B,KAAI,CAAC,WAAW;EACd,MAAM,EAAC,QAAO,SAAS,KAAK;AAG5B,aAAW,mBAAmB,IAAI;EAElC,MAAM,gBAAgB,WAAW,aAAa,IAAI;EAClD,MAAM,CAAC,QAAQ,WAAW,MAAM,QAAQ,IAAI,CAC1C,WAAW,eAAe,UAAU,cAAc,EAClD,WAAW,gBAAgB,UAAU,cAAc,CACpD,CAAC;AACF,MAAI,UAAU,QACZ,MAAK,MAAM,MAAM,IACf,SAAQ,GAAG,IAAX;GACE,KAAK;AACH,UAAM,WAAW,QAAQ,aAAa,IAAI,GAAG,CAAC,CAAC;AAC/C;GACF,KAAK;AACH,UAAM,WAAW,QAAQ,aAAa,IAAI,GAAG,CAAC,CAAC;AAC/C;GACF,KAAK;AACH,UAAM,WAAW,QAAQ,aAAa,IAAI,GAAG,CAAC,CAAC;AAC/C;GACF,KAAK;AACH,UAAM,WAAW,QAAQ,aAAa,IAAI,GAAG,CAAC,CAAC;AAC/C;GACF,QACE,aAAY,GAAG;;;AAQzB,OAAM,cACJ,6CACE,IACA,OACA,eACA,SAAS,UACT,SAAS,GACV,CACF;AAGD,OAAM,QAAQ,IAAI,MAAM,KAAI,SAAQ,MAAM,CAAC,CAAC;;AAG9C,SAAgB,aACd,IACA,QACuC;AACvC,QAAO,EAAE,eAAe,GAAG,OAAO,UAAU,CAAC,GAAG,GAAG,OAAO,MAAM;;AAGlE,SAAgB,aACd,IACA,KACuC;CACvC,MAAM,EAAC,WAAW,YAAY,UAAS;AACvC,QAAO,EAAE;kBACO,GAAG,UAAU,CAAC,GAAG,GAAG,MAAM,CAAC;mBAC1B,GAAG,WAAW,CAAC;oBACd,GAAG,MAAM,CAAC;;;AAI9B,SAAS,aACP,IACA,QACuC;CACvC,MAAM,QAAQ,OAAO;CACrB,MAAM,EAAC,YAAY,UAAS;CAC5B,MAAM,KAAsC,EAAE;AAC9C,MAAK,MAAM,OAAO,WAChB,IAAG,OAAO,MAAQ,MAAM,MAAM,sBAAsB;AAEtD,QAAO,EAAE,UAAU,GAAG,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC,SAAS,OAAO,QAC5D,GACD,CAAC,SAAS,CAAC,KAAK,QAAQ,MACvB,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,GAAG,IAAI,CAAC,KAAK,QAAQ,GAAG,EAAE,GAAG,GAAG,IAAI,CAAC,KAAK,QAC9D;;AAGH,SAAS,aACP,IACA,UACuC;CACvC,MAAM,EAAC,WAAW,YAAY,UAAS;CAEvC,MAAM,aAAa,EAAE;AACrB,MAAK,MAAM,OAAO,YAAY;AAC5B,MAAI,WAAW,SAAS,EACtB,YAAW,KAAK,EAAE,MAAM;AAE1B,aAAW,KAAK,EAAE,GAAG,GAAG,IAAI,CAAC,KAAK,MAAM,OAAO;;AAGjD,QAAO,EAAE,eAAe,GAAG,UAAU,CAAC,SAAS;;AAGjD,eAAe,6CACb,IACA,OACA,eACA,UACA,oBACA;CACA,MAAM,CAAC,EAAC,oBAAmB,MAAM,EAA8B;kBAC/C,GAAG,eAAe,MAAM,CAAC,CAAC;;oBAExB,cAAc,IAAI,SAAS,IAAI,EAAE;;;;;AAOnD,KAAI,qBAAqB,eACvB,OAAM,IAAI,8BACR,UACA,oBACA,eACD;UACQ,qBAAqB,eAC9B,OAAM,IAAI,cAAc;EACtB,MAAM;EACN,SAAS,wCAAwC,mBAAmB,cAAc,SAAS,yBAAyB,eAAe,UAAU,CAAC;EAC9I,QAAQ;EACT,CAAC"}

package/out/zero-cache/src/services/mutagen/pusher.d.ts

@@ -1,6 +1,7 @@
 import type { LogContext } from '@rocicorp/logger';
 import type { Downstream } from '../../../../zero-protocol/src/down.ts';
-import { type MutationID, type PushBody } from '../../../../zero-protocol/src/push.ts';
+import type { MutationID } from '../../../../zero-protocol/src/mutation-id.ts';
+import { type PushBody } from '../../../../zero-protocol/src/push.ts';
 import { type ZeroConfig } from '../../config/zero-config.ts';
 import type { Source } from '../../types/streams.ts';
 import { Subscription } from '../../types/subscription.ts';
@@ -30,7 +31,7 @@ type Config = Pick<ZeroConfig, 'app' | 'shard'>;
 export declare class PusherService implements Service, Pusher {
     #private;
     readonly id: string;
-    constructor(appConfig: Config, lc: LogContext, clientGroupID: string, contextManager: ConnectionContextManager);
+    constructor(appConfig: Config, lc: LogContext, clientGroupID: string, connContextManager: ConnectionContextManager);
     initConnection(selector: ConnectionSelector): Subscription<["deleteClients", {
         readonly clientIDs?: readonly string[] | undefined;
         readonly clientGroupIDs?: readonly string[] | undefined;
@@ -647,7 +648,7 @@ export declare class PusherService implements Service, Pusher {
 }
 type PusherEntry = {
     push: PushBody;
-    context: ConnectionContext;
+    connCtx: ConnectionContext;
 };
 type PusherEntryOrStop = PusherEntry | 'stop';
 /**

package/out/zero-cache/src/services/mutagen/pusher.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pusher.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/mutagen/pusher.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAMjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,uCAAuC,CAAC;AAStE,OAAO,EAGL,KAAK,UAAU,EACf,KAAK,QAAQ,EAEd,MAAM,uCAAuC,CAAC;AAE/C,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAK5D,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AACzD,OAAO,KAAK,EAAC,aAAa,EAAE,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAC7E,OAAO,KAAK,EAAC,iBAAiB,EAAE,OAAO,EAAC,MAAM,eAAe,CAAC;AAC9D,OAAO,KAAK,EACV,iBAAiB,EACjB,wBAAwB,EACxB,kBAAkB,EACnB,MAAM,8CAA8C,CAAC;AAEtD,MAAM,WAAW,MAAO,SAAQ,iBAAiB;IAC/C,cAAc,CAAC,QAAQ,EAAE,kBAAkB,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IACjE,WAAW,CAAC,QAAQ,EAAE,kBAAkB,EAAE,IAAI,EAAE,QAAQ,GAAG,aAAa,CAAC;IACzE,oBAAoB,CAClB,SAAS,EAAE,kBAAkB,EAC7B,MAAM,EAAE,UAAU,GACjB,OAAO,CAAC,IAAI,CAAC,CAAC;IACjB,qBAAqB,CACnB,SAAS,EAAE,kBAAkB,EAC7B,SAAS,EAAE,MAAM,EAAE,GAClB,OAAO,CAAC,IAAI,CAAC,CAAC;CAClB;AAED,KAAK,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,KAAK,GAAG,OAAO,CAAC,CAAC;AAEhD;;;;;;;;;;;;GAYG;AACH,qBAAa,aAAc,YAAW,OAAO,EAAE,MAAM;;IACnD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;gBAWlB,SAAS,EAAE,MAAM,EACjB,EAAE,EAAE,UAAU,EACd,aAAa,EAAE,MAAM,EACrB,cAAc,EAAE,wBAAwB;IAe1C,cAAc,CAAC,QAAQ,EAAE,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAI3C,WAAW,CACT,QAAQ,EAAE,kBAAkB,EAC5B,IAAI,EAAE,QAAQ,GACb,OAAO,CAAC,aAAa,EAAE,YAAY,CAAC;IAWjC,oBAAoB,CACxB,SAAS,EAAE,kBAAkB,EAC7B,MAAM,EAAE,UAAU,GACjB,OAAO,CAAC,IAAI,CAAC;IA+ChB;;;;;OAKG;IACG,qBAAqB,CACzB,SAAS,EAAE,kBAAkB,EAC7B,SAAS,EAAE,MAAM,EAAE,GAClB,OAAO,CAAC,IAAI,CAAC;IAkDhB,GAAG;IAKH,KAAK;IAQL,OAAO,IAAI,OAAO;IAIlB,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAKpB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAQtB;AAED,KAAK,WAAW,GAAG;IACjB,IAAI,EAAE,QAAQ,CAAC;IACf,OAAO,EAAE,iBAAiB,CAAC;CAC5B,CAAC;AACF,KAAK,iBAAiB,GAAG,WAAW,GAAG,MAAM,CAAC;AAsU9C;;;;;GAKG;AACH,wBAAgB,aAAa,CAC3B,OAAO,EAAE,SAAS,CAAC,iBAAiB,GAAG,SAAS,CAAC,EAAE,GAClD,CAAC,WAAW,EAAE,EAAE,OAAO,CAAC,CAqC1B"}
+{"version":3,"file":"pusher.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/mutagen/pusher.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAMjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,uCAAuC,CAAC;AAYtE,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,8CAA8C,CAAC;AAG7E,OAAO,EAAC,KAAK,QAAQ,EAAC,MAAM,uCAAuC,CAAC;AAEpE,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAK5D,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AACzD,OAAO,KAAK,EAAC,aAAa,EAAE,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAC7E,OAAO,KAAK,EAAC,iBAAiB,EAAE,OAAO,EAAC,MAAM,eAAe,CAAC;AAC9D,OAAO,KAAK,EACV,iBAAiB,EACjB,wBAAwB,EACxB,kBAAkB,EACnB,MAAM,8CAA8C,CAAC;AAEtD,MAAM,WAAW,MAAO,SAAQ,iBAAiB;IAC/C,cAAc,CAAC,QAAQ,EAAE,kBAAkB,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IACjE,WAAW,CAAC,QAAQ,EAAE,kBAAkB,EAAE,IAAI,EAAE,QAAQ,GAAG,aAAa,CAAC;IACzE,oBAAoB,CAClB,SAAS,EAAE,kBAAkB,EAC7B,MAAM,EAAE,UAAU,GACjB,OAAO,CAAC,IAAI,CAAC,CAAC;IACjB,qBAAqB,CACnB,SAAS,EAAE,kBAAkB,EAC7B,SAAS,EAAE,MAAM,EAAE,GAClB,OAAO,CAAC,IAAI,CAAC,CAAC;CAClB;AAED,KAAK,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,KAAK,GAAG,OAAO,CAAC,CAAC;AAEhD;;;;;;;;;;;;GAYG;AACH,qBAAa,aAAc,YAAW,OAAO,EAAE,MAAM;;IACnD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;gBAWlB,SAAS,EAAE,MAAM,EACjB,EAAE,EAAE,UAAU,EACd,aAAa,EAAE,MAAM,EACrB,kBAAkB,EAAE,wBAAwB;IAe9C,cAAc,CAAC,QAAQ,EAAE,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAI3C,WAAW,CACT,QAAQ,EAAE,kBAAkB,EAC5B,IAAI,EAAE,QAAQ,GACb,OAAO,CAAC,aAAa,EAAE,YAAY,CAAC;IAWjC,oBAAoB,CACxB,SAAS,EAAE,kBAAkB,EAC7B,MAAM,EAAE,UAAU,GACjB,OAAO,CAAC,IAAI,CAAC;IA+ChB;;;;;OAKG;IACG,qBAAqB,CACzB,SAAS,EAAE,kBAAkB,EAC7B,SAAS,EAAE,MAAM,EAAE,GAClB,OAAO,CAAC,IAAI,CAAC;IAkDhB,GAAG;IAKH,KAAK;IAQL,OAAO,IAAI,OAAO;IAIlB,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAKpB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAQtB;AAED,KAAK,WAAW,GAAG;IACjB,IAAI,EAAE,QAAQ,CAAC;IACf,OAAO,EAAE,iBAAiB,CAAC;CAC5B,CAAC;AACF,KAAK,iBAAiB,GAAG,WAAW,GAAG,MAAM,CAAC;AAoW9C;;;;;GAKG;AACH,wBAAgB,aAAa,CAC3B,OAAO,EAAE,SAAS,CAAC,iBAAiB,GAAG,SAAS,CAAC,EAAE,GAClD,CAAC,WAAW,EAAE,EAAE,OAAO,CAAC,CAqC1B"}

package/out/zero-cache/src/services/mutagen/pusher.js

@@ -1,4 +1,4 @@
-import { assert, unreachable } from "../../../../shared/src/asserts.js";
+import { assert } from "../../../../shared/src/asserts.js";
 import { getErrorMessage } from "../../../../shared/src/error.js";
 import { groupBy } from "../../../../shared/src/arrays.js";
 import { must } from "../../../../shared/src/must.js";
@@ -7,8 +7,10 @@ import { Server, ZeroCache } from "../../../../zero-protocol/src/error-origin-en
 import { HTTP, Internal, OutOfOrderMutation, UnsupportedPushVersion } from "../../../../zero-protocol/src/error-reason-enum.js";
 import { isProtocolError } from "../../../../zero-protocol/src/error.js";
 import { Custom } from "../../../../zero-protocol/src/mutation-type-enum.js";
-import { CLEANUP_RESULTS_MUTATION_NAME, pushResponseSchema } from "../../../../zero-protocol/src/push.js";
+import { CLEANUP_RESULTS_MUTATION_NAME } from "../../../../zero-protocol/src/mutation.js";
+import "../../../../zero-protocol/src/push.js";
 import "../../config/zero-config.js";
+import { mutateResponseSchema } from "../../../../zero-protocol/src/mutate-server.js";
 import { getOrCreateCounter } from "../../observability/metrics.js";
 import { Queue } from "../../../../shared/src/queue.js";
 import { Subscription } from "../../types/subscription.js";
@@ -33,7 +35,7 @@ import { ROOT_CONTEXT, context, propagation } from "@opentelemetry/api";
 */
 var PusherService = class {
 	id;
-	#contextManager;
+	#connContextManager;
 	#pusher;
 	#queue;
 	#config;
@@ -41,24 +43,24 @@ var PusherService = class {
 	#stopped;
 	#refCount = 0;
 	#isStopped = false;
-	constructor(appConfig, lc, clientGroupID, contextManager) {
-		this.#contextManager = contextManager;
+	constructor(appConfig, lc, clientGroupID, connContextManager) {
+		this.#connContextManager = connContextManager;
 		this.#config = appConfig;
 		this.#lc = lc.withContext("component", "pusherService");
 		this.#queue = new Queue();
-		this.#pusher = new PushWorker(appConfig, lc, this.#contextManager, this.#queue);
+		this.#pusher = new PushWorker(appConfig, lc, this.#connContextManager, this.#queue);
 		this.id = clientGroupID;
 	}
 	initConnection(selector) {
 		return this.#pusher.initConnection(selector);
 	}
 	enqueuePush(selector, push) {
-		this.#pusher.enqueuePush(this.#contextManager.mustGetConnectionContext(selector), push);
+		this.#pusher.enqueuePush(this.#connContextManager.mustGetConnectionContext(selector), push);
 		return { type: "ok" };
 	}
 	async ackMutationResponses(requester, upToID) {
-		const ctx = this.#contextManager.getConnectionContext(requester);
-		if (!ctx?.pushContext?.url) return;
+		const connCtx = this.#connContextManager.getConnectionContext(requester);
+		if (!connCtx?.mutateContext?.url) return;
 		const cleanupBody = {
 			clientGroupID: this.id,
 			mutations: [{
@@ -79,7 +81,7 @@ var PusherService = class {
 			requestID: `cleanup-${this.id}-${upToID.clientID}-${upToID.id}`
 		};
 		try {
-			await fetchFromAPIServer(pushResponseSchema, "push", this.#lc, ctx, {
+			await fetchFromAPIServer(mutateResponseSchema, "push", this.#lc, connCtx, {
 				appID: this.#config.app.id,
 				shardNum: this.#config.shard.num
 			}, cleanupBody);
@@ -95,8 +97,8 @@ var PusherService = class {
 	*/
 	async deleteClientMutations(requester, clientIDs) {
 		if (clientIDs.length === 0) return;
-		const ctx = this.#contextManager.getConnectionContext(requester);
-		if (!ctx?.pushContext?.url) return;
+		const connCtx = this.#connContextManager.getConnectionContext(requester);
+		if (!connCtx?.mutateContext?.url) return;
 		const cleanupBody = {
 			clientGroupID: this.id,
 			mutations: [{
@@ -116,7 +118,7 @@ var PusherService = class {
 			requestID: `cleanup-bulk-${this.id}-${Date.now()}`
 		};
 		try {
-			await fetchFromAPIServer(pushResponseSchema, "push", this.#lc, ctx, {
+			await fetchFromAPIServer(mutateResponseSchema, "push", this.#lc, connCtx, {
 				appID: this.#config.app.id,
 				shardNum: this.#config.shard.num
 			}, cleanupBody);
@@ -152,16 +154,16 @@ var PusherService = class {
 * to the user's API server.
 */
 var PushWorker = class {
-	#contextManager;
+	#connContextManager;
 	#queue;
 	#lc;
 	#config;
 	#clients;
 	#customMutations = getOrCreateCounter("mutation", "custom", "Number of custom mutations processed");
 	#pushes = getOrCreateCounter("mutation", "pushes", "Number of pushes processed by the pusher");
-	constructor(config, lc, contextManager, queue) {
+	constructor(config, lc, connContextManager, queue) {
 		this.#lc = lc.withContext("component", "pusher");
-		this.#contextManager = contextManager;
+		this.#connContextManager = connContextManager;
 		this.#queue = queue;
 		this.#config = config;
 		this.#clients = /* @__PURE__ */ new Map();
@@ -183,10 +185,10 @@ var PushWorker = class {
 		});
 		return downstream;
 	}
-	enqueuePush(context, push) {
+	enqueuePush(connCtx, push) {
 		this.#queue.enqueue({
 			push,
-			context
+			connCtx
 		});
 	}
 	async run() {
@@ -207,7 +209,7 @@ var PushWorker = class {
 	*/
 	#fanOutResponses(response) {
 		const connectionTerminations = [];
-		if ("kind" in response || "error" in response) {
+		if ("kind" in response && response.kind === "PushFailed" || "error" in response) {
 			this.#lc.warn?.("The server behind ZERO_MUTATE_URL returned a push error.", response);
 			const groupedMutationIDs = groupBy(response.mutationIDs ?? [], (m) => m.clientID);
 			for (const [clientID, mutationIDs] of groupedMutationIDs) {
@@ -236,8 +238,7 @@ var PushWorker = class {
 						message: response.error === "zeroPusher" ? response.details : response.error === "unsupportedSchemaVersion" ? "Unsupported schema version" : "An unknown error occurred while pushing to the API server"
 					};
 					this.#failDownstream(client.downstream, pushFailedBody);
-				} else if ("kind" in response) this.#failDownstream(client.downstream, response);
-				else unreachable(response);
+				} else this.#failDownstream(client.downstream, response);
 			}
 		} else {
 			const groupedMutations = groupBy(response.mutations, (m) => m.id.clientID);
@@ -274,7 +275,7 @@ var PushWorker = class {
 		this.#customMutations.add(entry.push.mutations.length, { clientGroupID: entry.push.clientGroupID });
 		this.#pushes.add(1, { clientGroupID: entry.push.clientGroupID });
 		recordMutation("custom", entry.push.mutations.length);
-		const url = must(entry.context.pushContext.url, "ZERO_MUTATE_URL is not set");
+		const url = must(entry.connCtx.mutateContext.url, "ZERO_MUTATE_URL is not set");
 		this.#lc.debug?.("pushing to", url, "with", entry.push.mutations.length, "mutations");
 		let mutationIDs = [];
 		try {
@@ -282,21 +283,24 @@ var PushWorker = class {
 				id: m.id,
 				clientID: m.clientID
 			}));
-			const response = await fetchFromAPIServer(pushResponseSchema, "push", this.#lc, entry.context, {
+			const response = await fetchFromAPIServer(mutateResponseSchema, "push", this.#lc, entry.connCtx, {
 				appID: this.#config.app.id,
 				shardNum: this.#config.shard.num
 			}, entry.push);
-			if ("kind" in response || "error" in response) {
+			if ("kind" in response && response.kind === "PushFailed" || "error" in response) {
 				if (isAuthErrorBody(response)) {
 					this.#lc.warn?.("Push auth failed; invalidating connection", {
-						clientID: entry.context.clientID,
+						clientID: entry.connCtx.clientID,
 						response: "kind" in response ? response.message : void 0
 					});
-					this.#contextManager.failConnection(entry.context, entry.context.revision);
+					this.#connContextManager.failConnection(entry.connCtx, entry.connCtx.revision);
 				}
 				return response;
 			}
-			this.#contextManager.validateConnection(entry.context, entry.context.revision);
+			this.#connContextManager.validateConnection(entry.connCtx, entry.connCtx.revision, "kind" in response && response.kind === "MutateResponse" && response?.userID !== void 0 ? {
+				kind: "server-validated",
+				validatedUserID: response.userID
+			} : { kind: "client-fallback" });
 			return response;
 		} catch (e) {
 			if (isProtocolError(e) && e.errorBody.kind === "PushFailed") {
@@ -306,13 +310,28 @@ var PushWorker = class {
 				};
 				if (isAuthErrorBody(response)) {
 					this.#lc.warn?.("Push auth failed; invalidating connection", {
-						clientID: entry.context.clientID,
+						clientID: entry.connCtx.clientID,
 						response: "kind" in response ? response.message : void 0
 					});
-					this.#contextManager.failConnection(entry.context, entry.context.revision);
+					this.#connContextManager.failConnection(entry.connCtx, entry.connCtx.revision);
 				}
 				return response;
 			}
+			if (isProtocolError(e) && isAuthErrorBody(e.errorBody)) {
+				this.#lc.warn?.("Push validation failed; invalidating connection", {
+					clientID: entry.connCtx.clientID,
+					response: e.message
+				});
+				this.#connContextManager.failConnection(entry.connCtx, entry.connCtx.revision);
+				return {
+					kind: PushFailed,
+					origin: ZeroCache,
+					reason: HTTP,
+					message: e.message,
+					status: 401,
+					mutationIDs
+				};
+			}
 			return {
 				kind: PushFailed,
 				origin: ZeroCache,
@@ -354,7 +373,7 @@ function combinePushes(entries) {
 	}
 	for (const entry of entries) {
 		if (entry === "stop" || entry === void 0) return [collect(), true];
-		const key = `${entry.context.clientID}:${entry.context.wsID}:${entry.context.revision}`;
+		const key = `${entry.connCtx.clientID}:${entry.connCtx.wsID}:${entry.connCtx.revision}`;
 		const existing = pushesByConnection.get(key);
 		if (existing) existing.push(entry);
 		else pushesByConnection.set(key, [entry]);
@@ -362,16 +381,16 @@ function combinePushes(entries) {
 	return [collect(), false];
 }
 function assertAreCompatiblePushes(left, right) {
-	assert(left.context.clientID === right.context.clientID, "clientID must be the same for all pushes");
-	assert(left.context.wsID === right.context.wsID, "wsID must be the same for all pushes");
-	assert(left.context.revision === right.context.revision, "revision must be the same for all pushes");
-	assert(authEquals(left.context.auth, right.context.auth), "auth must be the same for all pushes with the same clientID");
+	assert(left.connCtx.clientID === right.connCtx.clientID, "clientID must be the same for all pushes");
+	assert(left.connCtx.wsID === right.connCtx.wsID, "wsID must be the same for all pushes");
+	assert(left.connCtx.revision === right.connCtx.revision, "revision must be the same for all pushes");
+	assert(authEquals(left.connCtx.auth, right.connCtx.auth), "auth must be the same for all pushes with the same clientID");
 	assert(left.push.schemaVersion === right.push.schemaVersion, "schemaVersion must be the same for all pushes with the same clientID");
 	assert(left.push.pushVersion === right.push.pushVersion, "pushVersion must be the same for all pushes with the same clientID");
-	assert(left.context.pushContext.headerOptions.cookie === right.context.pushContext.headerOptions.cookie, "httpCookie must be the same for all pushes with the same clientID");
-	assert(left.context.pushContext.headerOptions.origin === right.context.pushContext.headerOptions.origin, "origin must be the same for all pushes with the same clientID");
-	assert(left.context.userID === right.context.userID, "userID must be the same for all pushes with the same clientID");
-	assert(left.context.pushContext.url === right.context.pushContext.url, "userPushURL must be the same for all pushes with the same clientID");
+	assert(left.connCtx.mutateContext.headerOptions.cookie === right.connCtx.mutateContext.headerOptions.cookie, "httpCookie must be the same for all pushes with the same clientID");
+	assert(left.connCtx.mutateContext.headerOptions.origin === right.connCtx.mutateContext.headerOptions.origin, "origin must be the same for all pushes with the same clientID");
+	assert(left.connCtx.user.id === right.connCtx.user.id, "userID must be the same for all pushes with the same clientID");
+	assert(left.connCtx.mutateContext.url === right.connCtx.mutateContext.url, "userPushURL must be the same for all pushes with the same clientID");
 }
 //#endregion
 export { PusherService };