@rocicorp/zero 0.26.1 → 0.26.2-canary.4

package/out/zero-cache/src/services/litestream/commands.js.map

@@ -1 +1 @@
-{"version":3,"file":"commands.js","sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync} from 'node:fs';\nimport {must} from '../../../../shared/src/must.ts';\nimport {sleep} from '../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {assertNormalized} from '../../config/normalize.ts';\nimport type {ZeroConfig} from '../../config/zero-config.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getShardConfig} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {ChangeStreamerHttpClient} from '../change-streamer/change-streamer-http.ts';\nimport type {\n  SnapshotMessage,\n  SnapshotStatus,\n} from '../change-streamer/snapshot.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\n\n// Retry for up to 3 minutes (60 times with 3 second delay).\n// Beyond that, let the container runner restart the task.\nconst MAX_RETRIES = 60;\nconst RETRY_INTERVAL_MS = 3000;\n\n/**\n * @returns The time at which the last restore started\n *          (i.e. not counting failed attempts).\n */\nexport async function restoreReplica(\n  lc: LogContext,\n  config: ZeroConfig,\n): Promise<Date> {\n  const {changeStreamer} = config;\n\n  for (let i = 0; i < MAX_RETRIES; i++) {\n    if (i > 0) {\n      lc.info?.(\n        `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n      );\n      await sleep(RETRY_INTERVAL_MS);\n    }\n    const start = new Date();\n    const restored = await tryRestore(lc, config);\n    if (restored) {\n      return start;\n    }\n    if (\n      changeStreamer.mode === 'dedicated' &&\n      changeStreamer.uri === undefined\n    ) {\n      lc.info?.('no litestream backup found');\n      return start;\n    }\n  }\n  throw new Error(`max attempts exceeded restoring replica`);\n}\n\nfunction getLitestream(\n  config: ZeroConfig,\n  logLevelOverride?: LogLevel,\n  backupURLOverride?: string,\n): {\n  litestream: string;\n  env: NodeJS.ProcessEnv;\n} {\n  const {\n    executable,\n    backupURL,\n    logLevel,\n    configPath,\n    endpoint,\n    port = config.port + 2,\n    checkpointThresholdMB,\n    minCheckpointPageCount = checkpointThresholdMB * 250, // SQLite page size is 4KB\n    maxCheckpointPageCount = minCheckpointPageCount * 10,\n    incrementalBackupIntervalMinutes,\n    snapshotBackupIntervalHours,\n    multipartConcurrency,\n    multipartSize,\n  } = config.litestream;\n\n  // Set the snapshot interval to something smaller than x hours so that\n  // the hourly check triggers on the hour, rather than the hour after.\n  const snapshotBackupIntervalMinutes = snapshotBackupIntervalHours * 60 - 5;\n\n  return {\n    litestream: must(executable, `Missing --litestream-executable`),\n    env: {\n      ...process.env,\n      ['ZERO_REPLICA_FILE']: config.replica.file,\n      ['ZERO_LITESTREAM_BACKUP_URL']: must(backupURLOverride ?? backupURL),\n      ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n        minCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n        maxCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n        incrementalBackupIntervalMinutes,\n      ),\n      ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n      ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_MINUTES']: String(\n        snapshotBackupIntervalMinutes,\n      ),\n      ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n      ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n      ['ZERO_LOG_FORMAT']: config.log.format,\n      ['LITESTREAM_CONFIG']: configPath,\n      ['LITESTREAM_PORT']: String(port),\n      ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n    },\n  };\n}\n\nasync function tryRestore(lc: LogContext, config: ZeroConfig) {\n  const {changeStreamer} = config;\n\n  const isViewSyncer =\n    changeStreamer.mode === 'discover' || changeStreamer.uri !== undefined;\n\n  // Fire off a snapshot reservation to the current replication-manager\n  // (if there is one).\n  const firstMessage = reserveAndGetSnapshotStatus(lc, config, isViewSyncer);\n  let snapshotStatus: SnapshotStatus | undefined;\n  if (isViewSyncer) {\n    // The return value is required by view-syncers ...\n    snapshotStatus = await firstMessage;\n    lc.info?.(`restoring backup from ${snapshotStatus.backupURL}`);\n  } else {\n    // but it is also useful to pause change-log cleanup when a new\n    // replication-manager is starting up. In this case, the request is\n    // best-effort. In particular, there may not be a previous\n    // replication-manager running at all.\n    void firstMessage.catch(e => lc.debug?.(e));\n  }\n\n  const {litestream, env} = getLitestream(\n    config,\n    'debug', // Include all output from `litestream restore`, as it's minimal.\n    snapshotStatus?.backupURL,\n  );\n  const {restoreParallelism: parallelism} = config.litestream;\n  const proc = spawn(\n    litestream,\n    [\n      'restore',\n      '-if-db-not-exists',\n      '-if-replica-exists',\n      '-parallelism',\n      String(parallelism),\n      config.replica.file,\n    ],\n    {env, stdio: 'inherit', windowsHide: true},\n  );\n  const {promise, resolve, reject} = resolver();\n  proc.on('error', reject);\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      reject(`litestream killed with ${signal}`);\n    } else if (code !== 0) {\n      reject(`litestream exited with code ${code}`);\n    } else {\n      resolve();\n    }\n  });\n  await promise;\n  if (!existsSync(config.replica.file)) {\n    return false;\n  }\n  if (\n    snapshotStatus &&\n    !replicaIsValid(lc, config.replica.file, snapshotStatus)\n  ) {\n    lc.info?.(`Deleting local replica and retrying restore`);\n    deleteLiteDB(config.replica.file);\n    return false;\n  }\n  return true;\n}\n\nfunction replicaIsValid(\n  lc: LogContext,\n  replica: string,\n  snapshot: SnapshotStatus,\n) {\n  const db = new Database(lc, replica);\n  try {\n    const {replicaVersion, watermark} = getSubscriptionState(\n      new StatementRunner(db),\n    );\n    if (replicaVersion !== snapshot.replicaVersion) {\n      lc.warn?.(\n        `Local replica version ${replicaVersion} does not match change-streamer replicaVersion ${snapshot.replicaVersion}`,\n        snapshot,\n      );\n      return false;\n    }\n    if (watermark < snapshot.minWatermark) {\n      lc.warn?.(\n        `Local replica watermark ${watermark} is earlier than change-streamer minWatermark ${snapshot.minWatermark}`,\n      );\n      return false;\n    }\n    lc.info?.(\n      `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible with change-streamer`,\n      snapshot,\n    );\n    return true;\n  } finally {\n    db.close();\n  }\n}\n\nexport function startReplicaBackupProcess(\n  lc: LogContext,\n  config: ZeroConfig,\n): ChildProcess {\n  const {litestream, env} = getLitestream(config);\n  lc.info?.(`starting litestream backup to ${config.litestream.backupURL}`);\n  return spawn(litestream, ['replicate'], {\n    env,\n    stdio: 'inherit',\n    windowsHide: true,\n  });\n}\n\nfunction reserveAndGetSnapshotStatus(\n  lc: LogContext,\n  config: ZeroConfig,\n  isViewSyncer: boolean,\n): Promise<SnapshotStatus> {\n  const {promise: status, resolve, reject} = resolver<SnapshotStatus>();\n\n  void (async function () {\n    const abort = new AbortController();\n    process.on('SIGINT', () => abort.abort());\n    process.on('SIGTERM', () => abort.abort());\n\n    for (let i = 0; ; i++) {\n      let err: unknown | string = '';\n      try {\n        let resolved = false;\n        const stream = await reserveSnapshot(lc, config);\n        for await (const msg of stream) {\n          // Capture the value of the status message that the change-streamer\n          // (i.e. BackupMonitor) returns, and hold the connection open to\n          // \"reserve\" the snapshot and prevent change log cleanup.\n          resolve(msg[1]);\n          resolved = true;\n        }\n        // The change-streamer itself closes the connection when the\n        // subscription is started (or the reservation retried).\n        if (resolved) {\n          break;\n        }\n      } catch (e) {\n        err = e;\n      }\n      if (!isViewSyncer) {\n        return reject(err);\n      }\n      // Retry in the view-syncer since it cannot proceed until it connects\n      // to a (compatible) replication-manager. In particular, a\n      // replication-manager that does not support the view-syncer's\n      // change-streamer protocol will close the stream with an error; this\n      // retry logic essentially delays the startup of a view-syncer until\n      // a compatible replication-manager has been rolled out, allowing\n      // replication-manager and view-syncer services to be updated in\n      // parallel.\n      lc.warn?.(\n        `Unable to reserve snapshot (attempt ${i + 1}). Retrying in 5 seconds.`,\n        String(err),\n      );\n      try {\n        await sleep(5000, abort.signal);\n      } catch (e) {\n        return reject(e);\n      }\n    }\n  })();\n\n  return status;\n}\n\nfunction reserveSnapshot(\n  lc: LogContext,\n  config: ZeroConfig,\n): Promise<Source<SnapshotMessage>> {\n  assertNormalized(config);\n  const {taskID, change, changeStreamer} = config;\n  const shardID = getShardConfig(config);\n\n  const changeStreamerClient = new ChangeStreamerHttpClient(\n    lc,\n    shardID,\n    change.db,\n    changeStreamer.uri,\n  );\n\n  return changeStreamerClient.reserveSnapshot(taskID);\n}\n"],"names":[],"mappings":";;;;;;;;;;;;AAuBA,MAAM,cAAc;AACpB,MAAM,oBAAoB;AAM1B,eAAsB,eACpB,IACA,QACe;AACf,QAAM,EAAC,mBAAkB;AAEzB,WAAS,IAAI,GAAG,IAAI,aAAa,KAAK;AACpC,QAAI,IAAI,GAAG;AACT,SAAG;AAAA,QACD,kCAAkC,oBAAoB,GAAI;AAAA,MAAA;AAE5D,YAAM,MAAM,iBAAiB;AAAA,IAC/B;AACA,UAAM,4BAAY,KAAA;AAClB,UAAM,WAAW,MAAM,WAAW,IAAI,MAAM;AAC5C,QAAI,UAAU;AACZ,aAAO;AAAA,IACT;AACA,QACE,eAAe,SAAS,eACxB,eAAe,QAAQ,QACvB;AACA,SAAG,OAAO,4BAA4B;AACtC,aAAO;AAAA,IACT;AAAA,EACF;AACA,QAAM,IAAI,MAAM,yCAAyC;AAC3D;AAEA,SAAS,cACP,QACA,kBACA,mBAIA;AACA,QAAM;AAAA,IACJ;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,OAAO,OAAO,OAAO;AAAA,IACrB;AAAA,IACA,yBAAyB,wBAAwB;AAAA;AAAA,IACjD,yBAAyB,yBAAyB;AAAA,IAClD;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA,IACE,OAAO;AAIX,QAAM,gCAAgC,8BAA8B,KAAK;AAEzE,SAAO;AAAA,IACL,YAAY,KAAK,YAAY,iCAAiC;AAAA,IAC9D,KAAK;AAAA,MACH,GAAG,QAAQ;AAAA,MACX,CAAC,mBAAmB,GAAG,OAAO,QAAQ;AAAA,MACtC,CAAC,4BAA4B,GAAG,KAAK,qBAAqB,SAAS;AAAA,MACnE,CAAC,2CAA2C,GAAG;AAAA,QAC7C;AAAA,MAAA;AAAA,MAEF,CAAC,2CAA2C,GAAG;AAAA,QAC7C;AAAA,MAAA;AAAA,MAEF,CAAC,qDAAqD,GAAG;AAAA,QACvD;AAAA,MAAA;AAAA,MAEF,CAAC,2BAA2B,GAAG,oBAAoB;AAAA,MACnD,CAAC,kDAAkD,GAAG;AAAA,QACpD;AAAA,MAAA;AAAA,MAEF,CAAC,uCAAuC,GAAG,OAAO,oBAAoB;AAAA,MACtE,CAAC,gCAAgC,GAAG,OAAO,aAAa;AAAA,MACxD,CAAC,iBAAiB,GAAG,OAAO,IAAI;AAAA,MAChC,CAAC,mBAAmB,GAAG;AAAA,MACvB,CAAC,iBAAiB,GAAG,OAAO,IAAI;AAAA,MAChC,GAAI,WAAW,EAAC,CAAC,0BAA0B,GAAG,SAAA,IAAY,CAAA;AAAA,IAAC;AAAA,EAC7D;AAEJ;AAEA,eAAe,WAAW,IAAgB,QAAoB;AAC5D,QAAM,EAAC,mBAAkB;AAEzB,QAAM,eACJ,eAAe,SAAS,cAAc,eAAe,QAAQ;AAI/D,QAAM,eAAe,4BAA4B,IAAI,QAAQ,YAAY;AACzE,MAAI;AACJ,MAAI,cAAc;AAEhB,qBAAiB,MAAM;AACvB,OAAG,OAAO,yBAAyB,eAAe,SAAS,EAAE;AAAA,EAC/D,OAAO;AAKL,SAAK,aAAa,MAAM,CAAA,MAAK,GAAG,QAAQ,CAAC,CAAC;AAAA,EAC5C;AAEA,QAAM,EAAC,YAAY,IAAA,IAAO;AAAA,IACxB;AAAA,IACA;AAAA;AAAA,IACA,gBAAgB;AAAA,EAAA;AAElB,QAAM,EAAC,oBAAoB,YAAA,IAAe,OAAO;AACjD,QAAM,OAAO;AAAA,IACX;AAAA,IACA;AAAA,MACE;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,OAAO,WAAW;AAAA,MAClB,OAAO,QAAQ;AAAA,IAAA;AAAA,IAEjB,EAAC,KAAK,OAAO,WAAW,aAAa,KAAA;AAAA,EAAI;AAE3C,QAAM,EAAC,SAAS,SAAS,OAAA,IAAU,SAAA;AACnC,OAAK,GAAG,SAAS,MAAM;AACvB,OAAK,GAAG,SAAS,CAAC,MAAM,WAAW;AACjC,QAAI,QAAQ;AACV,aAAO,0BAA0B,MAAM,EAAE;AAAA,IAC3C,WAAW,SAAS,GAAG;AACrB,aAAO,+BAA+B,IAAI,EAAE;AAAA,IAC9C,OAAO;AACL,cAAA;AAAA,IACF;AAAA,EACF,CAAC;AACD,QAAM;AACN,MAAI,CAAC,WAAW,OAAO,QAAQ,IAAI,GAAG;AACpC,WAAO;AAAA,EACT;AACA,MACE,kBACA,CAAC,eAAe,IAAI,OAAO,QAAQ,MAAM,cAAc,GACvD;AACA,OAAG,OAAO,6CAA6C;AACvD,iBAAa,OAAO,QAAQ,IAAI;AAChC,WAAO;AAAA,EACT;AACA,SAAO;AACT;AAEA,SAAS,eACP,IACA,SACA,UACA;AACA,QAAM,KAAK,IAAI,SAAS,IAAI,OAAO;AACnC,MAAI;AACF,UAAM,EAAC,gBAAgB,UAAA,IAAa;AAAA,MAClC,IAAI,gBAAgB,EAAE;AAAA,IAAA;AAExB,QAAI,mBAAmB,SAAS,gBAAgB;AAC9C,SAAG;AAAA,QACD,yBAAyB,cAAc,kDAAkD,SAAS,cAAc;AAAA,QAChH;AAAA,MAAA;AAEF,aAAO;AAAA,IACT;AACA,QAAI,YAAY,SAAS,cAAc;AACrC,SAAG;AAAA,QACD,2BAA2B,SAAS,iDAAiD,SAAS,YAAY;AAAA,MAAA;AAE5G,aAAO;AAAA,IACT;AACA,OAAG;AAAA,MACD,4BAA4B,cAAc,kBAAkB,SAAS;AAAA,MACrE;AAAA,IAAA;AAEF,WAAO;AAAA,EACT,UAAA;AACE,OAAG,MAAA;AAAA,EACL;AACF;AAEO,SAAS,0BACd,IACA,QACc;AACd,QAAM,EAAC,YAAY,QAAO,cAAc,MAAM;AAC9C,KAAG,OAAO,iCAAiC,OAAO,WAAW,SAAS,EAAE;AACxE,SAAO,MAAM,YAAY,CAAC,WAAW,GAAG;AAAA,IACtC;AAAA,IACA,OAAO;AAAA,IACP,aAAa;AAAA,EAAA,CACd;AACH;AAEA,SAAS,4BACP,IACA,QACA,cACyB;AACzB,QAAM,EAAC,SAAS,QAAQ,SAAS,OAAA,IAAU,SAAA;AAE3C,QAAM,iBAAkB;AACtB,UAAM,QAAQ,IAAI,gBAAA;AAClB,YAAQ,GAAG,UAAU,MAAM,MAAM,OAAO;AACxC,YAAQ,GAAG,WAAW,MAAM,MAAM,OAAO;AAEzC,aAAS,IAAI,KAAK,KAAK;AACrB,UAAI,MAAwB;AAC5B,UAAI;AACF,YAAI,WAAW;AACf,cAAM,SAAS,MAAM,gBAAgB,IAAI,MAAM;AAC/C,yBAAiB,OAAO,QAAQ;AAI9B,kBAAQ,IAAI,CAAC,CAAC;AACd,qBAAW;AAAA,QACb;AAGA,YAAI,UAAU;AACZ;AAAA,QACF;AAAA,MACF,SAAS,GAAG;AACV,cAAM;AAAA,MACR;AACA,UAAI,CAAC,cAAc;AACjB,eAAO,OAAO,GAAG;AAAA,MACnB;AASA,SAAG;AAAA,QACD,uCAAuC,IAAI,CAAC;AAAA,QAC5C,OAAO,GAAG;AAAA,MAAA;AAEZ,UAAI;AACF,cAAM,MAAM,KAAM,MAAM,MAAM;AAAA,MAChC,SAAS,GAAG;AACV,eAAO,OAAO,CAAC;AAAA,MACjB;AAAA,IACF;AAAA,EACF,GAAA;AAEA,SAAO;AACT;AAEA,SAAS,gBACP,IACA,QACkC;AAClC,mBAAiB,MAAM;AACvB,QAAM,EAAC,QAAQ,QAAQ,eAAA,IAAkB;AACzC,QAAM,UAAU,eAAe,MAAM;AAErC,QAAM,uBAAuB,IAAI;AAAA,IAC/B;AAAA,IACA;AAAA,IACA,OAAO;AAAA,IACP,eAAe;AAAA,EAAA;AAGjB,SAAO,qBAAqB,gBAAgB,MAAM;AACpD;"}
+{"version":3,"file":"commands.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync} from 'node:fs';\nimport {must} from '../../../../shared/src/must.ts';\nimport {sleep} from '../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {assertNormalized} from '../../config/normalize.ts';\nimport type {ZeroConfig} from '../../config/zero-config.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getShardConfig} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {ChangeStreamerHttpClient} from '../change-streamer/change-streamer-http.ts';\nimport type {\n  SnapshotMessage,\n  SnapshotStatus,\n} from '../change-streamer/snapshot.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\n\n// Retry for up to 3 minutes (60 times with 3 second delay).\n// Beyond that, let the container runner restart the task.\nconst MAX_RETRIES = 60;\nconst RETRY_INTERVAL_MS = 3000;\n\n/**\n * @returns The time at which the last restore started\n *          (i.e. not counting failed attempts).\n */\nexport async function restoreReplica(\n  lc: LogContext,\n  config: ZeroConfig,\n): Promise<Date> {\n  const {changeStreamer} = config;\n\n  for (let i = 0; i < MAX_RETRIES; i++) {\n    if (i > 0) {\n      lc.info?.(\n        `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n      );\n      await sleep(RETRY_INTERVAL_MS);\n    }\n    const start = new Date();\n    const restored = await tryRestore(lc, config);\n    if (restored) {\n      return start;\n    }\n    if (\n      changeStreamer.mode === 'dedicated' &&\n      changeStreamer.uri === undefined\n    ) {\n      lc.info?.('no litestream backup found');\n      return start;\n    }\n  }\n  throw new Error(`max attempts exceeded restoring replica`);\n}\n\nfunction getLitestream(\n  config: ZeroConfig,\n  logLevelOverride?: LogLevel,\n  backupURLOverride?: string,\n): {\n  litestream: string;\n  env: NodeJS.ProcessEnv;\n} {\n  const {\n    executable,\n    backupURL,\n    logLevel,\n    configPath,\n    endpoint,\n    port = config.port + 2,\n    checkpointThresholdMB,\n    minCheckpointPageCount = checkpointThresholdMB * 250, // SQLite page size is 4KB\n    maxCheckpointPageCount = minCheckpointPageCount * 10,\n    incrementalBackupIntervalMinutes,\n    snapshotBackupIntervalHours,\n    multipartConcurrency,\n    multipartSize,\n  } = config.litestream;\n\n  // Set the snapshot interval to something smaller than x hours so that\n  // the hourly check triggers on the hour, rather than the hour after.\n  const snapshotBackupIntervalMinutes = snapshotBackupIntervalHours * 60 - 5;\n\n  return {\n    litestream: must(executable, `Missing --litestream-executable`),\n    env: {\n      ...process.env,\n      ['ZERO_REPLICA_FILE']: config.replica.file,\n      ['ZERO_LITESTREAM_BACKUP_URL']: must(backupURLOverride ?? backupURL),\n      ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n        minCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n        maxCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n        incrementalBackupIntervalMinutes,\n      ),\n      ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n      ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_MINUTES']: String(\n        snapshotBackupIntervalMinutes,\n      ),\n      ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n      ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n      ['ZERO_LOG_FORMAT']: config.log.format,\n      ['LITESTREAM_CONFIG']: configPath,\n      ['LITESTREAM_PORT']: String(port),\n      ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n    },\n  };\n}\n\nasync function tryRestore(lc: LogContext, config: ZeroConfig) {\n  const {changeStreamer} = config;\n\n  const isViewSyncer =\n    changeStreamer.mode === 'discover' || changeStreamer.uri !== undefined;\n\n  // Fire off a snapshot reservation to the current replication-manager\n  // (if there is one).\n  const firstMessage = reserveAndGetSnapshotStatus(lc, config, isViewSyncer);\n  let snapshotStatus: SnapshotStatus | undefined;\n  if (isViewSyncer) {\n    // The return value is required by view-syncers ...\n    snapshotStatus = await firstMessage;\n    lc.info?.(`restoring backup from ${snapshotStatus.backupURL}`);\n  } else {\n    // but it is also useful to pause change-log cleanup when a new\n    // replication-manager is starting up. In this case, the request is\n    // best-effort. In particular, there may not be a previous\n    // replication-manager running at all.\n    void firstMessage.catch(e => lc.debug?.(e));\n  }\n\n  const {litestream, env} = getLitestream(\n    config,\n    'debug', // Include all output from `litestream restore`, as it's minimal.\n    snapshotStatus?.backupURL,\n  );\n  const {restoreParallelism: parallelism} = config.litestream;\n  const proc = spawn(\n    litestream,\n    [\n      'restore',\n      '-if-db-not-exists',\n      '-if-replica-exists',\n      '-parallelism',\n      String(parallelism),\n      config.replica.file,\n    ],\n    {env, stdio: 'inherit', windowsHide: true},\n  );\n  const {promise, resolve, reject} = resolver();\n  proc.on('error', reject);\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      reject(`litestream killed with ${signal}`);\n    } else if (code !== 0) {\n      reject(`litestream exited with code ${code}`);\n    } else {\n      resolve();\n    }\n  });\n  await promise;\n  if (!existsSync(config.replica.file)) {\n    return false;\n  }\n  if (\n    snapshotStatus &&\n    !replicaIsValid(lc, config.replica.file, snapshotStatus)\n  ) {\n    lc.info?.(`Deleting local replica and retrying restore`);\n    deleteLiteDB(config.replica.file);\n    return false;\n  }\n  return true;\n}\n\nfunction replicaIsValid(\n  lc: LogContext,\n  replica: string,\n  snapshot: SnapshotStatus,\n) {\n  const db = new Database(lc, replica);\n  try {\n    const {replicaVersion, watermark} = getSubscriptionState(\n      new StatementRunner(db),\n    );\n    if (replicaVersion !== snapshot.replicaVersion) {\n      lc.warn?.(\n        `Local replica version ${replicaVersion} does not match change-streamer replicaVersion ${snapshot.replicaVersion}`,\n        snapshot,\n      );\n      return false;\n    }\n    if (watermark < snapshot.minWatermark) {\n      lc.warn?.(\n        `Local replica watermark ${watermark} is earlier than change-streamer minWatermark ${snapshot.minWatermark}`,\n      );\n      return false;\n    }\n    lc.info?.(\n      `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible with change-streamer`,\n      snapshot,\n    );\n    return true;\n  } finally {\n    db.close();\n  }\n}\n\nexport function startReplicaBackupProcess(\n  lc: LogContext,\n  config: ZeroConfig,\n): ChildProcess {\n  const {litestream, env} = getLitestream(config);\n  lc.info?.(`starting litestream backup to ${config.litestream.backupURL}`);\n  return spawn(litestream, ['replicate'], {\n    env,\n    stdio: 'inherit',\n    windowsHide: true,\n  });\n}\n\nfunction reserveAndGetSnapshotStatus(\n  lc: LogContext,\n  config: ZeroConfig,\n  isViewSyncer: boolean,\n): Promise<SnapshotStatus> {\n  const {promise: status, resolve, reject} = resolver<SnapshotStatus>();\n\n  void (async function () {\n    const abort = new AbortController();\n    process.on('SIGINT', () => abort.abort());\n    process.on('SIGTERM', () => abort.abort());\n\n    for (let i = 0; ; i++) {\n      let err: unknown | string = '';\n      try {\n        let resolved = false;\n        const stream = await reserveSnapshot(lc, config);\n        for await (const msg of stream) {\n          // Capture the value of the status message that the change-streamer\n          // (i.e. BackupMonitor) returns, and hold the connection open to\n          // \"reserve\" the snapshot and prevent change log cleanup.\n          resolve(msg[1]);\n          resolved = true;\n        }\n        // The change-streamer itself closes the connection when the\n        // subscription is started (or the reservation retried).\n        if (resolved) {\n          break;\n        }\n      } catch (e) {\n        err = e;\n      }\n      if (!isViewSyncer) {\n        return reject(err);\n      }\n      // Retry in the view-syncer since it cannot proceed until it connects\n      // to a (compatible) replication-manager. In particular, a\n      // replication-manager that does not support the view-syncer's\n      // change-streamer protocol will close the stream with an error; this\n      // retry logic essentially delays the startup of a view-syncer until\n      // a compatible replication-manager has been rolled out, allowing\n      // replication-manager and view-syncer services to be updated in\n      // parallel.\n      lc.warn?.(\n        `Unable to reserve snapshot (attempt ${i + 1}). Retrying in 5 seconds.`,\n        String(err),\n      );\n      try {\n        await sleep(5000, abort.signal);\n      } catch (e) {\n        return reject(e);\n      }\n    }\n  })();\n\n  return status;\n}\n\nfunction reserveSnapshot(\n  lc: LogContext,\n  config: ZeroConfig,\n): Promise<Source<SnapshotMessage>> {\n  assertNormalized(config);\n  const {taskID, change, changeStreamer} = config;\n  const shardID = getShardConfig(config);\n\n  const changeStreamerClient = new ChangeStreamerHttpClient(\n    lc,\n    shardID,\n    change.db,\n    changeStreamer.uri,\n  );\n\n  return changeStreamerClient.reserveSnapshot(taskID);\n}\n"],"mappings":";;;;;;;;;;;;;AAuBA,IAAM,cAAc;AACpB,IAAM,oBAAoB;;;;;AAM1B,eAAsB,eACpB,IACA,QACe;CACf,MAAM,EAAC,mBAAkB;AAEzB,MAAK,IAAI,IAAI,GAAG,IAAI,aAAa,KAAK;AACpC,MAAI,IAAI,GAAG;AACT,MAAG,OACD,kCAAkC,oBAAoB,IAAK,UAC5D;AACD,SAAM,MAAM,kBAAkB;;EAEhC,MAAM,wBAAQ,IAAI,MAAM;AAExB,MADiB,MAAM,WAAW,IAAI,OAAO,CAE3C,QAAO;AAET,MACE,eAAe,SAAS,eACxB,eAAe,QAAQ,KAAA,GACvB;AACA,MAAG,OAAO,6BAA6B;AACvC,UAAO;;;AAGX,OAAM,IAAI,MAAM,0CAA0C;;AAG5D,SAAS,cACP,QACA,kBACA,mBAIA;CACA,MAAM,EACJ,YACA,WACA,UACA,YACA,UACA,OAAO,OAAO,OAAO,GACrB,uBACA,yBAAyB,wBAAwB,KACjD,yBAAyB,yBAAyB,IAClD,kCACA,6BACA,sBACA,kBACE,OAAO;CAIX,MAAM,gCAAgC,8BAA8B,KAAK;AAEzE,QAAO;EACL,YAAY,KAAK,YAAY,kCAAkC;EAC/D,KAAK;GACH,GAAG,QAAQ;IACV,sBAAsB,OAAO,QAAQ;IACrC,+BAA+B,KAAK,qBAAqB,UAAU;IACnE,8CAA8C,OAC7C,uBACD;IACA,8CAA8C,OAC7C,uBACD;IACA,wDAAwD,OACvD,iCACD;IACA,8BAA8B,oBAAoB;IAClD,qDAAqD,OACpD,8BACD;IACA,0CAA0C,OAAO,qBAAqB;IACtE,mCAAmC,OAAO,cAAc;IACxD,oBAAoB,OAAO,IAAI;IAC/B,sBAAsB;IACtB,oBAAoB,OAAO,KAAK;GACjC,GAAI,WAAW,GAAE,6BAA6B,UAAS,GAAG,EAAE;GAC7D;EACF;;AAGH,eAAe,WAAW,IAAgB,QAAoB;CAC5D,MAAM,EAAC,mBAAkB;CAEzB,MAAM,eACJ,eAAe,SAAS,cAAc,eAAe,QAAQ,KAAA;CAI/D,MAAM,eAAe,4BAA4B,IAAI,QAAQ,aAAa;CAC1E,IAAI;AACJ,KAAI,cAAc;AAEhB,mBAAiB,MAAM;AACvB,KAAG,OAAO,yBAAyB,eAAe,YAAY;OAMzD,cAAa,OAAM,MAAK,GAAG,QAAQ,EAAE,CAAC;CAG7C,MAAM,EAAC,YAAY,QAAO,cACxB,QACA,SACA,gBAAgB,UACjB;CACD,MAAM,EAAC,oBAAoB,gBAAe,OAAO;CACjD,MAAM,OAAO,MACX,YACA;EACE;EACA;EACA;EACA;EACA,OAAO,YAAY;EACnB,OAAO,QAAQ;EAChB,EACD;EAAC;EAAK,OAAO;EAAW,aAAa;EAAK,CAC3C;CACD,MAAM,EAAC,SAAS,SAAS,WAAU,UAAU;AAC7C,MAAK,GAAG,SAAS,OAAO;AACxB,MAAK,GAAG,UAAU,MAAM,WAAW;AACjC,MAAI,OACF,QAAO,0BAA0B,SAAS;WACjC,SAAS,EAClB,QAAO,+BAA+B,OAAO;MAE7C,UAAS;GAEX;AACF,OAAM;AACN,KAAI,CAAC,WAAW,OAAO,QAAQ,KAAK,CAClC,QAAO;AAET,KACE,kBACA,CAAC,eAAe,IAAI,OAAO,QAAQ,MAAM,eAAe,EACxD;AACA,KAAG,OAAO,8CAA8C;AACxD,eAAa,OAAO,QAAQ,KAAK;AACjC,SAAO;;AAET,QAAO;;AAGT,SAAS,eACP,IACA,SACA,UACA;CACA,MAAM,KAAK,IAAI,SAAS,IAAI,QAAQ;AACpC,KAAI;EACF,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,GAAG,CACxB;AACD,MAAI,mBAAmB,SAAS,gBAAgB;AAC9C,MAAG,OACD,yBAAyB,eAAe,iDAAiD,SAAS,kBAClG,SACD;AACD,UAAO;;AAET,MAAI,YAAY,SAAS,cAAc;AACrC,MAAG,OACD,2BAA2B,UAAU,gDAAgD,SAAS,eAC/F;AACD,UAAO;;AAET,KAAG,OACD,4BAA4B,eAAe,iBAAiB,UAAU,sCACtE,SACD;AACD,SAAO;WACC;AACR,KAAG,OAAO;;;AAId,SAAgB,0BACd,IACA,QACc;CACd,MAAM,EAAC,YAAY,QAAO,cAAc,OAAO;AAC/C,IAAG,OAAO,iCAAiC,OAAO,WAAW,YAAY;AACzE,QAAO,MAAM,YAAY,CAAC,YAAY,EAAE;EACtC;EACA,OAAO;EACP,aAAa;EACd,CAAC;;AAGJ,SAAS,4BACP,IACA,QACA,cACyB;CACzB,MAAM,EAAC,SAAS,QAAQ,SAAS,WAAU,UAA0B;AAErE,EAAM,iBAAkB;EACtB,MAAM,QAAQ,IAAI,iBAAiB;AACnC,UAAQ,GAAG,gBAAgB,MAAM,OAAO,CAAC;AACzC,UAAQ,GAAG,iBAAiB,MAAM,OAAO,CAAC;AAE1C,OAAK,IAAI,IAAI,IAAK,KAAK;GACrB,IAAI,MAAwB;AAC5B,OAAI;IACF,IAAI,WAAW;IACf,MAAM,SAAS,MAAM,gBAAgB,IAAI,OAAO;AAChD,eAAW,MAAM,OAAO,QAAQ;AAI9B,aAAQ,IAAI,GAAG;AACf,gBAAW;;AAIb,QAAI,SACF;YAEK,GAAG;AACV,UAAM;;AAER,OAAI,CAAC,aACH,QAAO,OAAO,IAAI;AAUpB,MAAG,OACD,uCAAuC,IAAI,EAAE,4BAC7C,OAAO,IAAI,CACZ;AACD,OAAI;AACF,UAAM,MAAM,KAAM,MAAM,OAAO;YACxB,GAAG;AACV,WAAO,OAAO,EAAE;;;KAGlB;AAEJ,QAAO;;AAGT,SAAS,gBACP,IACA,QACkC;AAClC,kBAAiB,OAAO;CACxB,MAAM,EAAC,QAAQ,QAAQ,mBAAkB;AAUzC,QAP6B,IAAI,yBAC/B,IAHc,eAAe,OAAO,EAKpC,OAAO,IACP,eAAe,IAChB,CAE2B,gBAAgB,OAAO"}

package/out/zero-cache/src/services/mutagen/error.js

@@ -1,16 +1,12 @@
 import { assert } from "../../../../shared/src/asserts.js";
-class MutationAlreadyProcessedError extends Error {
-  constructor(clientID, received, actual) {
-    super(
-      `Ignoring mutation from ${clientID} with ID ${received} as it was already processed. Expected: ${actual}`
-    );
-    assert(
-      received < actual,
-      () => `MutationAlreadyProcessedError: received (${received}) must be < actual (${actual})`
-    );
-  }
-}
-export {
-  MutationAlreadyProcessedError
+//#region ../zero-cache/src/services/mutagen/error.ts
+var MutationAlreadyProcessedError = class extends Error {
+	constructor(clientID, received, actual) {
+		super(`Ignoring mutation from ${clientID} with ID ${received} as it was already processed. Expected: ${actual}`);
+		assert(received < actual, () => `MutationAlreadyProcessedError: received (${received}) must be < actual (${actual})`);
+	}
 };
-//# sourceMappingURL=error.js.map
+//#endregion
+export { MutationAlreadyProcessedError };
+
+//# sourceMappingURL=error.js.map

package/out/zero-cache/src/services/mutagen/error.js.map

@@ -1 +1 @@
-{"version":3,"file":"error.js","sources":["../../../../../../zero-cache/src/services/mutagen/error.ts"],"sourcesContent":["import {assert} from '../../../../shared/src/asserts.ts';\n\nexport class MutationAlreadyProcessedError extends Error {\n  constructor(clientID: string, received: number, actual: number | bigint) {\n    super(\n      `Ignoring mutation from ${clientID} with ID ${received} as it was already processed. Expected: ${actual}`,\n    );\n    assert(\n      received < actual,\n      () =>\n        `MutationAlreadyProcessedError: received (${received}) must be < actual (${actual})`,\n    );\n  }\n}\n"],"names":[],"mappings":";AAEO,MAAM,sCAAsC,MAAM;AAAA,EACvD,YAAY,UAAkB,UAAkB,QAAyB;AACvE;AAAA,MACE,0BAA0B,QAAQ,YAAY,QAAQ,2CAA2C,MAAM;AAAA,IAAA;AAEzG;AAAA,MACE,WAAW;AAAA,MACX,MACE,4CAA4C,QAAQ,uBAAuB,MAAM;AAAA,IAAA;AAAA,EAEvF;AACF;"}
+{"version":3,"file":"error.js","names":[],"sources":["../../../../../../zero-cache/src/services/mutagen/error.ts"],"sourcesContent":["import {assert} from '../../../../shared/src/asserts.ts';\n\nexport class MutationAlreadyProcessedError extends Error {\n  constructor(clientID: string, received: number, actual: number | bigint) {\n    super(\n      `Ignoring mutation from ${clientID} with ID ${received} as it was already processed. Expected: ${actual}`,\n    );\n    assert(\n      received < actual,\n      () =>\n        `MutationAlreadyProcessedError: received (${received}) must be < actual (${actual})`,\n    );\n  }\n}\n"],"mappings":";;AAEA,IAAa,gCAAb,cAAmD,MAAM;CACvD,YAAY,UAAkB,UAAkB,QAAyB;AACvE,QACE,0BAA0B,SAAS,WAAW,SAAS,0CAA0C,SAClG;AACD,SACE,WAAW,cAET,4CAA4C,SAAS,sBAAsB,OAAO,GACrF"}

package/out/zero-cache/src/services/mutagen/mutagen.js

@@ -1,276 +1,189 @@
-import { PG_SERIALIZATION_FAILURE } from "@drdgvhbh/postgres-error-codes";
-import { resolver } from "@rocicorp/resolver";
-import postgres from "postgres";
 import { assert, unreachable } from "../../../../shared/src/asserts.js";
 import { parse } from "../../../../shared/src/valita.js";
-import { MutationRateLimited, InvalidPush, MutationFailed } from "../../../../zero-protocol/src/error-kind-enum.js";
+import { InvalidPush, MutationFailed, MutationRateLimited } from "../../../../zero-protocol/src/error-kind-enum.js";
 import { ZeroCache } from "../../../../zero-protocol/src/error-origin-enum.js";
-import { isProtocolError, ProtocolError } from "../../../../zero-protocol/src/error.js";
+import { ProtocolError, isProtocolError } from "../../../../zero-protocol/src/error.js";
 import { CRUD } from "../../../../zero-protocol/src/mutation-type-enum.js";
 import { primaryKeyValueSchema } from "../../../../zero-protocol/src/primary-key.js";
 import "../../../../zero-protocol/src/push.js";
+import { MutationAlreadyProcessedError } from "./error.js";
 import { Database } from "../../../../zqlite/src/db.js";
-import { WriteAuthorizerImpl } from "../../auth/write-authorizer.js";
+import { upstreamSchema } from "../../types/shards.js";
 import "../../config/zero-config.js";
 import { SERIALIZABLE } from "../../db/mode-enum.js";
 import { runTx } from "../../db/run-transaction.js";
 import { getOrCreateCounter } from "../../observability/metrics.js";
 import { recordMutation } from "../../server/anonymous-otel-start.js";
-import { upstreamSchema } from "../../types/shards.js";
+import { WriteAuthorizerImpl } from "../../auth/write-authorizer.js";
 import { SlidingWindowLimiter } from "../limiter/sliding-window-limiter.js";
-import { MutationAlreadyProcessedError } from "./error.js";
-class MutagenService {
-  id;
-  #lc;
-  #upstream;
-  #shard;
-  #stopped = resolver();
-  #replica;
-  #writeAuthorizer;
-  #limiter;
-  #refCount = 0;
-  #isStopped = false;
-  #crudMutations = getOrCreateCounter(
-    "mutation",
-    "crud",
-    "Number of CRUD mutations processed"
-  );
-  constructor(lc, shard, clientGroupID, upstream, config, writeAuthzStorage) {
-    this.id = clientGroupID;
-    this.#lc = lc;
-    this.#upstream = upstream;
-    this.#shard = shard;
-    this.#replica = new Database(this.#lc, config.replica.file, {
-      fileMustExist: true
-    });
-    this.#writeAuthorizer = new WriteAuthorizerImpl(
-      this.#lc,
-      config,
-      this.#replica,
-      shard.appID,
-      clientGroupID,
-      writeAuthzStorage
-    );
-    if (config.perUserMutationLimit.max !== void 0) {
-      this.#limiter = new SlidingWindowLimiter(
-        config.perUserMutationLimit.windowMs,
-        config.perUserMutationLimit.max
-      );
-    }
-  }
-  ref() {
-    assert(!this.#isStopped, "MutagenService is already stopped");
-    ++this.#refCount;
-  }
-  unref() {
-    assert(!this.#isStopped, "MutagenService is already stopped");
-    --this.#refCount;
-    if (this.#refCount <= 0) {
-      void this.stop();
-    }
-  }
-  hasRefs() {
-    return this.#refCount > 0;
-  }
-  processMutation(mutation, authData, customMutatorsEnabled = false) {
-    if (this.#limiter?.canDo() === false) {
-      return Promise.resolve([
-        MutationRateLimited,
-        "Rate limit exceeded"
-      ]);
-    }
-    this.#crudMutations.add(1, {
-      clientGroupID: this.id
-    });
-    return processMutation(
-      this.#lc,
-      authData,
-      this.#upstream,
-      this.#shard,
-      this.id,
-      mutation,
-      this.#writeAuthorizer,
-      void 0,
-      customMutatorsEnabled
-    );
-  }
-  run() {
-    return this.#stopped.promise;
-  }
-  stop() {
-    if (this.#isStopped) {
-      return this.#stopped.promise;
-    }
-    this.#writeAuthorizer.destroy();
-    this.#isStopped = true;
-    this.#stopped.resolve();
-    return this.#stopped.promise;
-  }
-}
-const MAX_SERIALIZATION_ATTEMPTS = 10;
+import { resolver } from "@rocicorp/resolver";
+import postgres from "postgres";
+import { PG_SERIALIZATION_FAILURE } from "@drdgvhbh/postgres-error-codes";
+//#region ../zero-cache/src/services/mutagen/mutagen.ts
+var MutagenService = class {
+	id;
+	#lc;
+	#upstream;
+	#shard;
+	#stopped = resolver();
+	#replica;
+	#writeAuthorizer;
+	#limiter;
+	#refCount = 0;
+	#isStopped = false;
+	#crudMutations = getOrCreateCounter("mutation", "crud", "Number of CRUD mutations processed");
+	constructor(lc, shard, clientGroupID, upstream, config, writeAuthzStorage) {
+		this.id = clientGroupID;
+		this.#lc = lc;
+		this.#upstream = upstream;
+		this.#shard = shard;
+		this.#replica = new Database(this.#lc, config.replica.file, { fileMustExist: true });
+		this.#writeAuthorizer = new WriteAuthorizerImpl(this.#lc, config, this.#replica, shard.appID, clientGroupID, writeAuthzStorage);
+		if (config.perUserMutationLimit.max !== void 0) this.#limiter = new SlidingWindowLimiter(config.perUserMutationLimit.windowMs, config.perUserMutationLimit.max);
+	}
+	ref() {
+		assert(!this.#isStopped, "MutagenService is already stopped");
+		++this.#refCount;
+	}
+	unref() {
+		assert(!this.#isStopped, "MutagenService is already stopped");
+		--this.#refCount;
+		if (this.#refCount <= 0) this.stop();
+	}
+	hasRefs() {
+		return this.#refCount > 0;
+	}
+	processMutation(mutation, authData, customMutatorsEnabled = false) {
+		if (this.#limiter?.canDo() === false) return Promise.resolve([MutationRateLimited, "Rate limit exceeded"]);
+		this.#crudMutations.add(1, { clientGroupID: this.id });
+		return processMutation(this.#lc, authData, this.#upstream, this.#shard, this.id, mutation, this.#writeAuthorizer, void 0, customMutatorsEnabled);
+	}
+	run() {
+		return this.#stopped.promise;
+	}
+	stop() {
+		if (this.#isStopped) return this.#stopped.promise;
+		this.#writeAuthorizer.destroy();
+		this.#isStopped = true;
+		this.#stopped.resolve();
+		return this.#stopped.promise;
+	}
+};
+var MAX_SERIALIZATION_ATTEMPTS = 10;
 async function processMutation(lc, authData, db, shard, clientGroupID, mutation, writeAuthorizer, onTxStart, customMutatorsEnabled = false) {
-  assert(
-    mutation.type === CRUD,
-    "Only CRUD mutations are supported"
-  );
-  lc = lc.withContext("mutationID", mutation.id);
-  lc = lc.withContext("processMutation");
-  lc.debug?.("Process mutation start", mutation);
-  recordMutation("crud");
-  let result;
-  const start = Date.now();
-  try {
-    let errorMode = false;
-    for (let i = 0; i < MAX_SERIALIZATION_ATTEMPTS; i++) {
-      try {
-        await runTx(
-          db,
-          async (tx) => {
-            const done = onTxStart?.();
-            try {
-              return await processMutationWithTx(
-                lc,
-                tx,
-                authData,
-                shard,
-                clientGroupID,
-                mutation,
-                errorMode,
-                writeAuthorizer
-              );
-            } finally {
-              await done;
-            }
-          },
-          { mode: SERIALIZABLE }
-        );
-        if (errorMode) {
-          lc.debug?.("Ran mutation successfully in error mode");
-        }
-        break;
-      } catch (e) {
-        if (e instanceof MutationAlreadyProcessedError) {
-          lc.debug?.(e.message);
-          return void 0;
-        }
-        if (isProtocolError(e) && !errorMode && e.kind === InvalidPush && customMutatorsEnabled && i < 2) {
-          lc.info?.(
-            "Both CRUD and Custom mutators are being used at once. This is supported for now but IS NOT RECOMMENDED. Migrate completely to custom mutators.",
-            e
-          );
-          await new Promise((resolve) => setTimeout(resolve, 100));
-          continue;
-        }
-        if (isProtocolError(e) || errorMode) {
-          lc.error?.("Process mutation error", e);
-          throw e;
-        }
-        if (e instanceof postgres.PostgresError && e.code === PG_SERIALIZATION_FAILURE) {
-          lc.info?.(`attempt ${i + 1}: ${String(e)}`, e);
-          continue;
-        }
-        result = [MutationFailed, String(e)];
-        if (errorMode) {
-          break;
-        }
-        lc.error?.("Got error running mutation, re-running in error mode", e);
-        errorMode = true;
-        i--;
-      }
-    }
-  } finally {
-    lc.debug?.("Process mutation complete in", Date.now() - start);
-  }
-  return result;
+	assert(mutation.type === CRUD, "Only CRUD mutations are supported");
+	lc = lc.withContext("mutationID", mutation.id);
+	lc = lc.withContext("processMutation");
+	lc.debug?.("Process mutation start", mutation);
+	recordMutation("crud");
+	let result;
+	const start = Date.now();
+	try {
+		let errorMode = false;
+		for (let i = 0; i < MAX_SERIALIZATION_ATTEMPTS; i++) try {
+			await runTx(db, async (tx) => {
+				const done = onTxStart?.();
+				try {
+					return await processMutationWithTx(lc, tx, authData, shard, clientGroupID, mutation, errorMode, writeAuthorizer);
+				} finally {
+					await done;
+				}
+			}, { mode: SERIALIZABLE });
+			if (errorMode) lc.debug?.("Ran mutation successfully in error mode");
+			break;
+		} catch (e) {
+			if (e instanceof MutationAlreadyProcessedError) {
+				lc.debug?.(e.message);
+				return;
+			}
+			if (isProtocolError(e) && !errorMode && e.kind === "InvalidPush" && customMutatorsEnabled && i < 2) {
+				lc.info?.("Both CRUD and Custom mutators are being used at once. This is supported for now but IS NOT RECOMMENDED. Migrate completely to custom mutators.", e);
+				await new Promise((resolve) => setTimeout(resolve, 100));
+				continue;
+			}
+			if (isProtocolError(e) || errorMode) {
+				lc.error?.("Process mutation error", e);
+				throw e;
+			}
+			if (e instanceof postgres.PostgresError && e.code === PG_SERIALIZATION_FAILURE) {
+				lc.info?.(`attempt ${i + 1}: ${String(e)}`, e);
+				continue;
+			}
+			result = [MutationFailed, String(e)];
+			if (errorMode) break;
+			lc.error?.("Got error running mutation, re-running in error mode", e);
+			errorMode = true;
+			i--;
+		}
+	} finally {
+		lc.debug?.("Process mutation complete in", Date.now() - start);
+	}
+	return result;
 }
 async function processMutationWithTx(lc, tx, authData, shard, clientGroupID, mutation, errorMode, authorizer) {
-  const tasks = [];
-  async function execute(stmt) {
-    try {
-      return await stmt.execute();
-    } finally {
-      const q = stmt;
-      lc.debug?.(`${q.string}: ${JSON.stringify(q.parameters)}`);
-    }
-  }
-  authorizer.reloadPermissions();
-  if (!errorMode) {
-    const { ops } = mutation.args[0];
-    authorizer.validateTableNames(ops);
-    const normalizedOps = authorizer.normalizeOps(ops);
-    const [canPre, canPost] = await Promise.all([
-      authorizer.canPreMutation(authData, normalizedOps),
-      authorizer.canPostMutation(authData, normalizedOps)
-    ]);
-    if (canPre && canPost) {
-      for (const op of ops) {
-        switch (op.op) {
-          case "insert":
-            tasks.push(() => execute(getInsertSQL(tx, op)));
-            break;
-          case "upsert":
-            tasks.push(() => execute(getUpsertSQL(tx, op)));
-            break;
-          case "update":
-            tasks.push(() => execute(getUpdateSQL(tx, op)));
-            break;
-          case "delete":
-            tasks.push(() => execute(getDeleteSQL(tx, op)));
-            break;
-          default:
-            unreachable();
-        }
-      }
-    }
-  }
-  tasks.unshift(
-    () => checkSchemaVersionAndIncrementLastMutationID(
-      tx,
-      shard,
-      clientGroupID,
-      mutation.clientID,
-      mutation.id
-    )
-  );
-  await Promise.all(tasks.map((task) => task()));
+	const tasks = [];
+	async function execute(stmt) {
+		try {
+			return await stmt.execute();
+		} finally {
+			const q = stmt;
+			lc.debug?.(`${q.string}: ${JSON.stringify(q.parameters)}`);
+		}
+	}
+	authorizer.reloadPermissions();
+	if (!errorMode) {
+		const { ops } = mutation.args[0];
+		authorizer.validateTableNames(ops);
+		const normalizedOps = authorizer.normalizeOps(ops);
+		const [canPre, canPost] = await Promise.all([authorizer.canPreMutation(authData, normalizedOps), authorizer.canPostMutation(authData, normalizedOps)]);
+		if (canPre && canPost) for (const op of ops) switch (op.op) {
+			case "insert":
+				tasks.push(() => execute(getInsertSQL(tx, op)));
+				break;
+			case "upsert":
+				tasks.push(() => execute(getUpsertSQL(tx, op)));
+				break;
+			case "update":
+				tasks.push(() => execute(getUpdateSQL(tx, op)));
+				break;
+			case "delete":
+				tasks.push(() => execute(getDeleteSQL(tx, op)));
+				break;
+			default: unreachable(op);
+		}
+	}
+	tasks.unshift(() => checkSchemaVersionAndIncrementLastMutationID(tx, shard, clientGroupID, mutation.clientID, mutation.id));
+	await Promise.all(tasks.map((task) => task()));
 }
 function getInsertSQL(tx, create) {
-  return tx`INSERT INTO ${tx(create.tableName)} ${tx(create.value)}`;
+	return tx`INSERT INTO ${tx(create.tableName)} ${tx(create.value)}`;
 }
 function getUpsertSQL(tx, set) {
-  const { tableName, primaryKey, value } = set;
-  return tx`
+	const { tableName, primaryKey, value } = set;
+	return tx`
     INSERT INTO ${tx(tableName)} ${tx(value)}
     ON CONFLICT (${tx(primaryKey)})
     DO UPDATE SET ${tx(value)}
   `;
 }
 function getUpdateSQL(tx, update) {
-  const table = update.tableName;
-  const { primaryKey, value } = update;
-  const id = {};
-  for (const key of primaryKey) {
-    id[key] = parse(value[key], primaryKeyValueSchema);
-  }
-  return tx`UPDATE ${tx(table)} SET ${tx(value)} WHERE ${Object.entries(
-    id
-  ).flatMap(
-    ([key, value2], i) => i ? [tx`AND`, tx`${tx(key)} = ${value2}`] : tx`${tx(key)} = ${value2}`
-  )}`;
+	const table = update.tableName;
+	const { primaryKey, value } = update;
+	const id = {};
+	for (const key of primaryKey) id[key] = parse(value[key], primaryKeyValueSchema);
+	return tx`UPDATE ${tx(table)} SET ${tx(value)} WHERE ${Object.entries(id).flatMap(([key, value], i) => i ? [tx`AND`, tx`${tx(key)} = ${value}`] : tx`${tx(key)} = ${value}`)}`;
 }
 function getDeleteSQL(tx, deleteOp) {
-  const { tableName, primaryKey, value } = deleteOp;
-  const conditions = [];
-  for (const key of primaryKey) {
-    if (conditions.length > 0) {
-      conditions.push(tx`AND`);
-    }
-    conditions.push(tx`${tx(key)} = ${value[key]}`);
-  }
-  return tx`DELETE FROM ${tx(tableName)} WHERE ${conditions}`;
+	const { tableName, primaryKey, value } = deleteOp;
+	const conditions = [];
+	for (const key of primaryKey) {
+		if (conditions.length > 0) conditions.push(tx`AND`);
+		conditions.push(tx`${tx(key)} = ${value[key]}`);
+	}
+	return tx`DELETE FROM ${tx(tableName)} WHERE ${conditions}`;
 }
 async function checkSchemaVersionAndIncrementLastMutationID(tx, shard, clientGroupID, clientID, receivedMutationID) {
-  const [{ lastMutationID }] = await tx`
+	const [{ lastMutationID }] = await tx`
     INSERT INTO ${tx(upstreamSchema(shard))}.clients 
       as current ("clientGroupID", "clientID", "lastMutationID")
           VALUES (${clientGroupID}, ${clientID}, ${1})
@@ -278,25 +191,14 @@ async function checkSchemaVersionAndIncrementLastMutationID(tx, shard, clientGro
       DO UPDATE SET "lastMutationID" = current."lastMutationID" + 1
       RETURNING "lastMutationID"
   `;
-  if (receivedMutationID < lastMutationID) {
-    throw new MutationAlreadyProcessedError(
-      clientID,
-      receivedMutationID,
-      lastMutationID
-    );
-  } else if (receivedMutationID > lastMutationID) {
-    throw new ProtocolError({
-      kind: InvalidPush,
-      message: `Push contains unexpected mutation id ${receivedMutationID} for client ${clientID}. Expected mutation id ${lastMutationID.toString()}.`,
-      origin: ZeroCache
-    });
-  }
+	if (receivedMutationID < lastMutationID) throw new MutationAlreadyProcessedError(clientID, receivedMutationID, lastMutationID);
+	else if (receivedMutationID > lastMutationID) throw new ProtocolError({
+		kind: InvalidPush,
+		message: `Push contains unexpected mutation id ${receivedMutationID} for client ${clientID}. Expected mutation id ${lastMutationID.toString()}.`,
+		origin: ZeroCache
+	});
 }
-export {
-  MutagenService,
-  getInsertSQL,
-  getUpsertSQL,
-  processMutation,
-  processMutationWithTx
-};
-//# sourceMappingURL=mutagen.js.map
+//#endregion
+export { MutagenService };
+
+//# sourceMappingURL=mutagen.js.map